diff --git a/.cargo/config.toml b/.cargo/config.toml
index c65cd3ac1a4fa..c9165c404ef92 100644
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -32,6 +32,3 @@ rustflags = [
 # Flags for all targets.
 [target.'cfg(all())']
 rustflags = ["--cfg", "tokio_unstable"]
-
-[unstable]
-lints = true
diff --git a/.github/workflows/auto-update-helm-and-operator-version-by-release.yml b/.github/workflows/auto-update-helm-and-operator-version-by-release.yml
new file mode 100644
index 0000000000000..1e4e6a8c9c1e9
--- /dev/null
+++ b/.github/workflows/auto-update-helm-and-operator-version-by-release.yml
@@ -0,0 +1,79 @@
+name: Update Helm Charts and Risingwave Operator on New Release
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'release version'
+        required: true
+
+env:
+  NEW_APP_VERSION: ${{ github.event.inputs.version || github.event.release.tag_name }}
+
+jobs:
+  update-helm-charts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Helm Charts Repository
+        uses: actions/checkout@v3
+        with:
+          repository: 'risingwavelabs/helm-charts'
+          token: ${{ secrets.PR_TOKEN }}
+          path: 'helm-charts'
+
+      - name: Update values.yaml
+        run: |
+          sed -i "s/^  tag:.*/  tag: \"${{ env.NEW_APP_VERSION }}\"/" helm-charts/charts/risingwave/values.yaml
+
+      - name: Update Chart.yaml
+        run: |
+          cd helm-charts/charts/risingwave
+          CURRENT_VERSION=$(grep 'version:' Chart.yaml | awk '{print $2}' | head -n 1)
+          NEW_VERSION=$(echo $CURRENT_VERSION | awk -F. -v OFS='.' '{$NF++; print}')
+          sed -i "/type: application/,/version:/!b; /version:/s/version: .*/version: $NEW_VERSION/" Chart.yaml
+          sed -i "s/^appVersion: .*/appVersion: \"${{ env.NEW_APP_VERSION }}\"/" Chart.yaml
+          echo "NEW_CHART_VERSION=$NEW_VERSION" >> $GITHUB_ENV
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.PR_TOKEN }}
+          commit-message: 'chore: bump risingwave to ${{ env.NEW_APP_VERSION }}, release chart ${{ env.NEW_CHART_VERSION }}'
+          title: 'chore: bump risingwave to ${{ env.NEW_APP_VERSION }}, release chart ${{ env.NEW_CHART_VERSION }}'
+          body: 'This is an automated pull request to update the chart versions'
+          branch: 'auto-update-${{ env.NEW_APP_VERSION }}'
+          path: 'helm-charts'
+          reviewers: arkbriar
+          delete-branch: true
+          signoff: true
+
+  update-risingwave-operator:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Risingwave Operator Repository
+        uses: actions/checkout@v3
+        with:
+          repository: 'risingwavelabs/risingwave-operator'
+          token: ${{ secrets.PR_TOKEN }}
+          path: 'risingwave-operator'
+
+      - name: Update risingwave-operator image tags
+        run: |
+          cd risingwave-operator
+          PREV_VERSION=$(grep -roh "risingwavelabs/risingwave:v[0-9\.]*" * | head -n 1 | cut -d':' -f2)
+          grep -rl "risingwavelabs/risingwave:$PREV_VERSION" . | xargs sed -i "s|risingwavelabs/risingwave:$PREV_VERSION|risingwavelabs/risingwave:${{ env.NEW_APP_VERSION }}|g"
+
+      - name: Create Pull Request for risingwave-operator
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.PR_TOKEN }}
+          commit-message: 'chore: bump risingwave image tags to ${{ env.NEW_APP_VERSION }}'
+          title: 'chore: bump risingwave image tags to ${{ env.NEW_APP_VERSION }}'
+          body: 'This is an automated pull request to update the risingwave image tags'
+          branch: 'auto-update-${{ env.NEW_APP_VERSION }}'
+          path: 'risingwave-operator'
+          reviewers: arkbriar
+          delete-branch: true
+          signoff: true
diff --git a/.github/workflows/cherry-pick-to-release-branch.yml b/.github/workflows/cherry-pick-to-release-branch.yml
index f54e9fc722e6e..4e8fd60117731 100644
--- a/.github/workflows/cherry-pick-to-release-branch.yml
+++ b/.github/workflows/cherry-pick-to-release-branch.yml
@@ -6,8 +6,8 @@ on:
     types: ["closed", "labeled"]
 
 jobs:
-  release_pull_request_1_8:
-    if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-release-1.8')  &&  github.event.pull_request.merged == true"
+  release_pull_request_1_9:
+    if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-release-1.9')  &&  github.event.pull_request.merged == true"
     runs-on: ubuntu-latest
     name: release_pull_request
     steps:
@@ -16,14 +16,14 @@ jobs:
       - name: Create PR to branch
         uses: risingwavelabs/github-action-cherry-pick@master
         with:
-          pr_branch: 'release-1.8'
+          pr_branch: 'release-1.9'
           pr_labels: 'cherry-pick'
-          pr_body: ${{ format('Cherry picking \#{0} onto branch release-1.8', github.event.number) }}
+          pr_body: ${{ format('Cherry picking \#{0} onto branch release-1.9', github.event.number) }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  release_pull_request_1_7_standalone:
-    if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-release-1.7-standalone')  &&  github.event.pull_request.merged == true"
+  release_pull_request_1_8:
+    if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-release-1.8')  &&  github.event.pull_request.merged == true"
     runs-on: ubuntu-latest
     name: release_pull_request
     steps:
@@ -32,24 +32,9 @@ jobs:
       - name: Create PR to branch
         uses: risingwavelabs/github-action-cherry-pick@master
         with:
-          pr_branch: 'release-1.7.0-standalone'
+          pr_branch: 'release-1.8'
           pr_labels: 'cherry-pick'
-          pr_body: ${{ format('Cherry picking \#{0} onto branch release-1.7.0-standalone', github.event.number) }}
+          pr_body: ${{ format('Cherry picking \#{0} onto branch release-1.8', github.event.number) }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  release_pull_request_1_7:
-    if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-release-1.7')  &&  github.event.pull_request.merged == true"
-    runs-on: ubuntu-latest
-    name: release_pull_request
-    steps:
-      - name: checkout
-        uses: actions/checkout@v1
-      - name: Create PR to branch
-        uses: risingwavelabs/github-action-cherry-pick@master
-        with:
-          pr_branch: 'release-1.7'
-          pr_labels: 'cherry-pick'
-          pr_body: ${{ format('Cherry picking \#{0} onto branch release-1.7', github.event.number) }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/connector-node-integration.yml b/.github/workflows/connector-node-integration.yml
index f20f209a3953d..abc1942055fb5 100644
--- a/.github/workflows/connector-node-integration.yml
+++ b/.github/workflows/connector-node-integration.yml
@@ -2,11 +2,11 @@ name: Connector Node Integration Tests
 
 on:
   push:
-    branches: [main]
+    branches: [ main ]
   pull_request:
-    branches: [main]
+    branches: [ main ]
   merge_group:
-    types: [checks_requested]
+    types: [ checks_requested ]
 
 jobs:
   build:
@@ -42,4 +42,8 @@ jobs:
           echo "--- build connector node"
           cd ${RISINGWAVE_ROOT}/java
           # run unit test
-          mvn --batch-mode --update-snapshots clean package -Dno-build-rust
+          # WARN: `testOnNext_writeValidation` is skipped because it relies on Rust code to decode message,
+          #       while we don't build Rust code (`-Dno-build-rust`) here to save time
+          mvn --batch-mode --update-snapshots clean package -Dno-build-rust \
+            '-Dtest=!com.risingwave.connector.sink.SinkStreamObserverTest#testOnNext_writeValidation' \
+            -Dsurefire.failIfNoSpecifiedTests=false
diff --git a/.github/workflows/dashboard_main.yml b/.github/workflows/dashboard.yml
similarity index 77%
rename from .github/workflows/dashboard_main.yml
rename to .github/workflows/dashboard.yml
index 0b9571b35154d..c722d4704e00e 100644
--- a/.github/workflows/dashboard_main.yml
+++ b/.github/workflows/dashboard.yml
@@ -1,8 +1,11 @@
-name: Dashboard (main)
+name: Dashboard
 on:
   push:
     branches: [main]
     paths: [dashboard/**, proto/**]
+  pull_request:
+    branches: [main]
+    paths: [dashboard/**, proto/**]
   workflow_dispatch:
 jobs:
   dashboard-ui-deploy:
@@ -19,11 +22,14 @@ jobs:
       - name: build
         working-directory: ./dashboard
         run: |
+          echo "::group::npm install"
           npm install
+          echo "::endgroup::"
           npm run lint
           npm run build
       - name: Deploy
         uses: s0/git-publish-subdir-action@develop
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         env:
           REPO: self
           BRANCH: dashboard-artifact
diff --git a/.github/workflows/dashboard_pr.yml b/.github/workflows/dashboard_pr.yml
deleted file mode 100644
index c7245a2119d99..0000000000000
--- a/.github/workflows/dashboard_pr.yml
+++ /dev/null
@@ -1,63 +0,0 @@
-name: Dashboard
-on:
-  pull_request:
-    branches: [main]
-    paths: [dashboard/**, proto/**]
-
-concurrency:
-  group: dashboard-build-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  dashboard-ui-build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout forked repo
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.ref }}
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-      - uses: actions/setup-node@v2
-        with:
-          node-version: "18"
-      - uses: arduino/setup-protoc@v1
-        with:
-          version: "3.x"
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - name: build
-        working-directory: ./dashboard
-        run: |
-          echo "::group::npm install"
-          npm install
-          echo "::endgroup::"
-          npm run gen-proto
-
-          git add -A
-          if ! git diff --quiet --staged; then
-            git reset
-            echo "::error::Generated proto files are not up to date."
-
-            echo "::group::check if the branch is out-of-date"
-            git pull origin $GITHUB_HEAD_REF --unshallow
-            git remote add upstream https://github.com/risingwavelabs/risingwave.git
-            git fetch upstream $GITHUB_BASE_REF:$GITHUB_BASE_REF
-
-            echo "origin/$GITHUB_HEAD_REF: $(git rev-parse origin/$GITHUB_HEAD_REF)"
-            echo "upstream/$GITHUB_BASE_REF: $(git rev-parse upstream/$GITHUB_BASE_REF)"
-
-            git merge-base origin/$GITHUB_HEAD_REF upstream/$GITHUB_BASE_REF
-
-            FORK_POINT=$(git merge-base origin/$GITHUB_HEAD_REF upstream/$GITHUB_BASE_REF)
-            echo "FORK_POINT: $FORK_POINT"
-            echo "::endgroup::"
-
-            if ! git diff --quiet $FORK_POINT..upstream/$GITHUB_BASE_REF -- ../proto; then
-              echo "::error::Your branch is out-of-date. Please update your branch first and then run 'npm i && npm run gen-proto' at dashboard/ and commit the changes."
-            else
-              echo "::error::Please run 'npm i && npm run gen-proto' at dashboard/ and commit the changes."
-            fi
-            exit 1
-          fi
-
-          npm run lint
-          npm run build
diff --git a/.github/workflows/nightly-rust.yml b/.github/workflows/nightly-rust.yml
index eab964fafa177..5fa90ae1138c8 100644
--- a/.github/workflows/nightly-rust.yml
+++ b/.github/workflows/nightly-rust.yml
@@ -5,22 +5,25 @@ name: Build with Latest Nightly Rust
 on:
   schedule:
     - cron: "0 0 * * *"
+  push:
+    branches:
+      - xxchan/latest-nightly-rust
   workflow_dispatch:
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      -  name: Maximize build space
-         uses: easimon/maximize-build-space@master
-         with:
-           remove-dotnet: 'true'
-           remove-android: 'true'
-           remove-haskell: 'true'
-           remove-codeql: 'true'
-           remove-docker-images: 'true'
-           root-reserve-mb: 2048
-           swap-size-mb: 8192
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+          root-reserve-mb: 10240
+          temp-reserve-mb: 10240
       - uses: actions/checkout@v3
         if: ${{ github.event_name == 'schedule' }}
         with:
@@ -40,3 +43,5 @@ jobs:
           export CARGO_INCREMENTAL=0
           export CARGO_PROFILE_DEV_DEBUG=false
           cargo check
+      - name: Show available storage
+        run: df -h
diff --git a/.github/workflows/package_version_check.yml b/.github/workflows/package_version_check.yml
new file mode 100644
index 0000000000000..e0bbd04eef12a
--- /dev/null
+++ b/.github/workflows/package_version_check.yml
@@ -0,0 +1,57 @@
+name: Package Version Checker
+
+on:
+  pull_request:
+    branches:
+      - 'main'
+
+jobs:
+  compare-package-version-with-latest-release-version:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: List branches
+        run: |
+          git fetch --all
+          release_branches=$(git branch -r | grep -E 'origin/release-[0-9]+\.[0-9]+' | sed 's/origin\///')
+          echo "Release branches:"
+          echo "$release_branches"
+          echo "$release_branches" > release_branches.txt
+
+      - name: Pick latest release branch
+        run: |
+          release_branches=$(cat release_branches.txt)
+          latest_branch=$(echo "$release_branches" | sort -t. -k1,1 -k2,2 -Vr | head -n 1)
+          echo "Latest release branch: $latest_branch"
+          latest_version=$(echo "$latest_branch" | sed -E 's/release-([0-9]+\.[0-9]+)/\1/' | sed 's/^[ \t]*//')
+          echo "Latest release version: $latest_version"
+          echo "$latest_version" > latest_release_version.txt
+
+      - name: Read Cargo.toml version
+        run: |
+          cargo_version=$(grep -oP '(?<=^version = ")[0-9]+\.[0-9]+' Cargo.toml)
+          echo "Cargo.toml version: $cargo_version"
+          echo "$cargo_version" > cargo_version.txt
+
+      - name: Compare versions
+        run: |
+          latest_version=$(cat latest_release_version.txt)
+          cargo_version=$(cat cargo_version.txt)
+
+          latest_major=$(echo $latest_version | cut -d. -f1)
+          latest_minor=$(echo $latest_version | cut -d. -f2)
+
+          cargo_major=$(echo $cargo_version | cut -d. -f1)
+          cargo_minor=$(echo $cargo_version | cut -d. -f2)
+
+          if [ "$cargo_major" -lt "$latest_major" ] || { [ "$cargo_major" -eq "$latest_major" ] && [ "$cargo_minor" -le "$latest_minor" ]; }; then
+            echo "Error: Cargo.toml package version $cargo_version is not larger than $latest_version"
+            exit 1
+          else
+            echo "Cargo.toml version $cargo_version is larger than or equal to $latest_version"
+          fi
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index eb5173e599e7d..765befde5a752 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -3,25 +3,50 @@ name: Mark stale issues and pull requests
 on:
   schedule:
   - cron: '30 1 * * *'
+  workflow_dispatch:
+    inputs:
+      # https://github.com/marketplace/actions/close-stale-issues#operations-per-run
+      operationsPerRun:
+        description: 'Max number of operations per run'
+        required: true
+        default: 30
 
 jobs:
   stale:
-
     runs-on: ubuntu-latest
     permissions:
       issues: write
       pull-requests: write
 
     steps:
-    - uses: actions/stale@v5
+    - uses: actions/stale@v9
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: >
           This issue has been open for 60 days with no activity.
-          Could you please update the status? Feel free to continue discussion or close as not planned.
+
+          If you think it is still relevant today, and needs to be done *in the near future*, you can comment to update the status, or just manually remove the `no-issue-activity` label.
+
+          You can also confidently close this issue as not planned to keep our backlog clean.
+          Don't worry if you think the issue is still valuable to continue in the future.
+          It's searchable and can be reopened when it's time. 😄
         stale-pr-message: >
           This PR has been open for 60 days with no activity.
-          Could you please update the status? Feel free to ping a reviewer if you are waiting for review.
+
+          If it's blocked by code review, feel free to ping a reviewer or ask someone else to review it.
+
+          If you think it is still relevant today, and have time to work on it *in the near future*, you can comment to update the status, or just manually remove the `no-pr-activity` label.
+
+          You can also confidently close this PR to keep our backlog clean. (If no further action taken, the PR will be automatically closed after 7 days. Sorry!  🙏)
+          Don't worry if you think the PR is still valuable to continue in the future.
+          It's searchable and can be reopened when it's time. 😄
+        close-pr-message: >
+          Close this PR as there's no further actions taken after it is marked as stale for 7 days. Sorry! 🙏
+
+          You can reopen it when you have time to continue working on it.
         stale-issue-label: 'no-issue-activity'
         stale-pr-label: 'no-pr-activity'
         days-before-close: -1
+        days-before-pr-close: 7
+        operations-per-run: ${{ github.event.inputs.operationsPerRun }}
+        enable-statistics: true
diff --git a/.gitignore b/.gitignore
index b74124ca9c157..855bf21820af8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -78,4 +78,6 @@ simulation-it-test.tar.zst
 # spark binary
 e2e_test/iceberg/spark-*-bin*
 
-**/poetry.lock
\ No newline at end of file
+**/poetry.lock
+
+*.slt.temp
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000000000..ab8ba3d9d7eb9
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,26 @@
+# Usage: install pre-commit, and then run `pre-commit install` to install git hooks
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: local
+    hooks:
+    -   id: rustfmt
+        name: rustfmt
+        entry: rustfmt --edition 2021
+        language: system
+        types: [rust]
+    -   id: typos
+        name: typos
+        entry: typos -w
+        language: system
+    -   id: cargo sort
+        name: cargo sort
+        entry: cargo sort -g -w
+        language: system
+        files: 'Cargo.toml'
+        pass_filenames: false
diff --git a/.typos.toml b/.typos.toml
index c062e9de44d2d..567904f5c319b 100644
--- a/.typos.toml
+++ b/.typos.toml
@@ -6,9 +6,10 @@ inout = "inout"         # This is a SQL keyword!
 numer = "numer"         # numerator
 nd = "nd"               # N-dimentional / 2nd
 steam = "stream"        # You played with Steam games too much.
+ser = "ser"             # Serialization
 # Some weird short variable names
 ot = "ot"
-bui = "bui" # BackwardUserIterator
+bui = "bui"             # BackwardUserIterator
 mosquitto = "mosquitto" # This is a MQTT broker.
 abd = "abd"
 iy = "iy"
@@ -21,6 +22,7 @@ extend-exclude = [
     "e2e_test",
     "**/*.svg",
     "scripts",
+    "src/sqlparser/tests/testdata/",
     "src/frontend/planner_test/tests/testdata",
     "src/tests/sqlsmith/tests/freeze",
     "Cargo.lock",
diff --git a/Cargo.lock b/Cargo.lock
index f8848f93726bd..96d6d1d670dd0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -77,11 +77,27 @@ dependencies = [
  "aes",
 ]
 
+[[package]]
+name = "aes-siv"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e08d0cdb774acd1e4dac11478b1a0c0d203134b2aab0ba25eb430de9b18f8b9"
+dependencies = [
+ "aead",
+ "aes",
+ "cipher",
+ "cmac",
+ "ctr",
+ "dbl",
+ "digest",
+ "zeroize",
+]
+
 [[package]]
 name = "ahash"
-version = "0.7.7"
+version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd"
+checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9"
 dependencies = [
  "getrandom",
  "once_cell",
@@ -90,9 +106,9 @@ dependencies = [
 
 [[package]]
 name = "ahash"
-version = "0.8.6"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
+checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
  "cfg-if",
  "const-random",
@@ -215,9 +231,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.81"
+version = "1.0.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247"
+checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
 dependencies = [
  "backtrace",
 ]
@@ -296,6 +312,15 @@ version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b3d0060af21e8d11a926981cc00c6c1541aa91dd64b9f881985c3da1094425f"
 
+[[package]]
+name = "array-util"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c4dd5139f13c1a8b0e9f52197dcda016bbcd4e877055f93fb9ecd0f6c6136a7"
+dependencies = [
+ "arrayvec",
+]
+
 [[package]]
 name = "arrayref"
 version = "0.3.7"
@@ -314,7 +339,7 @@ version = "48.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8919668503a4f2d8b6da96fa7c16e93046bfb3412ffcfa1e5dc7d2e3adcb378"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "arrow-arith 48.0.1",
  "arrow-array 48.0.1",
  "arrow-buffer 48.0.1",
@@ -327,7 +352,26 @@ dependencies = [
  "arrow-row 48.0.1",
  "arrow-schema 48.0.1",
  "arrow-select 48.0.1",
- "arrow-string",
+ "arrow-string 48.0.1",
+]
+
+[[package]]
+name = "arrow"
+version = "50.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa285343fba4d829d49985bdc541e3789cf6000ed0e84be7c039438df4a4e78c"
+dependencies = [
+ "arrow-arith 50.0.0",
+ "arrow-array 50.0.0",
+ "arrow-buffer 50.0.0",
+ "arrow-cast 50.0.0",
+ "arrow-data 50.0.0",
+ "arrow-ipc 50.0.0",
+ "arrow-ord 50.0.0",
+ "arrow-row 50.0.0",
+ "arrow-schema 50.0.0",
+ "arrow-select 50.0.0",
+ "arrow-string 50.0.0",
 ]
 
 [[package]]
@@ -366,12 +410,12 @@ version = "48.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6eaf89041fa5937940ae390294ece29e1db584f46d995608d6e5fe65a2e0e9b"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "arrow-buffer 48.0.1",
  "arrow-data 48.0.1",
  "arrow-schema 48.0.1",
  "chrono",
- "chrono-tz",
+ "chrono-tz 0.8.6",
  "half 2.3.1",
  "hashbrown 0.14.3",
  "num",
@@ -383,7 +427,7 @@ version = "50.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "arrow-buffer 50.0.0",
  "arrow-data 50.0.0",
  "arrow-schema 50.0.0",
@@ -598,7 +642,7 @@ version = "48.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a5d664318bc05f930559fc088888f0f7174d3c5bc888c0f4f9ae8f23aa398ba3"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "arrow-array 48.0.1",
  "arrow-buffer 48.0.1",
  "arrow-data 48.0.1",
@@ -613,7 +657,7 @@ version = "50.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "arrow-array 50.0.0",
  "arrow-buffer 50.0.0",
  "arrow-data 50.0.0",
@@ -643,7 +687,7 @@ version = "48.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "374c4c3b812ecc2118727b892252a4a4308f87a8aca1dbf09f3ce4bc578e668a"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "arrow-array 48.0.1",
  "arrow-buffer 48.0.1",
  "arrow-data 48.0.1",
@@ -657,7 +701,7 @@ version = "50.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "arrow-array 50.0.0",
  "arrow-buffer 50.0.0",
  "arrow-data 50.0.0",
@@ -681,23 +725,57 @@ dependencies = [
  "regex-syntax 0.8.2",
 ]
 
+[[package]]
+name = "arrow-string"
+version = "50.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00f3b37f2aeece31a2636d1b037dabb69ef590e03bdc7eb68519b51ec86932a7"
+dependencies = [
+ "arrow-array 50.0.0",
+ "arrow-buffer 50.0.0",
+ "arrow-data 50.0.0",
+ "arrow-schema 50.0.0",
+ "arrow-select 50.0.0",
+ "num",
+ "regex",
+ "regex-syntax 0.8.2",
+]
+
+[[package]]
+name = "arrow-udf-flight"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4adb3a066bd22fb520bc3d040d9d59ee54f320c21faeb6df815ea20445c80c54"
+dependencies = [
+ "arrow-array 50.0.0",
+ "arrow-flight",
+ "arrow-schema 50.0.0",
+ "arrow-select 50.0.0",
+ "futures-util",
+ "thiserror",
+ "tokio",
+ "tonic 0.10.2",
+ "tracing",
+]
+
 [[package]]
 name = "arrow-udf-js"
-version = "0.1.2"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "252b6355ad1e57eb6454b705c51652de55aa22eb018cdb95be0dbf62ee3ec78f"
+checksum = "76cb6d108605c5489fff1ef9c520656946ad05ed0de3ea6d26d56bcb34bdb8c5"
 dependencies = [
  "anyhow",
  "arrow-array 50.0.0",
  "arrow-buffer 50.0.0",
  "arrow-schema 50.0.0",
+ "atomic-time",
  "rquickjs",
 ]
 
 [[package]]
 name = "arrow-udf-js-deno"
 version = "0.0.1"
-source = "git+https://github.com/risingwavelabs/arrow-udf.git?rev=23fe0dd#23fe0dd41616f4646f9139e22a335518e6cc9a47"
+source = "git+https://github.com/risingwavelabs/arrow-udf.git?rev=fa36365#fa3636559de986aa592da6e8b3fbfac7bdd4bb78"
 dependencies = [
  "anyhow",
  "arrow-array 50.0.0",
@@ -719,7 +797,7 @@ dependencies = [
 [[package]]
 name = "arrow-udf-js-deno-runtime"
 version = "0.0.1"
-source = "git+https://github.com/risingwavelabs/arrow-udf.git?rev=23fe0dd#23fe0dd41616f4646f9139e22a335518e6cc9a47"
+source = "git+https://github.com/risingwavelabs/arrow-udf.git?rev=fa36365#fa3636559de986aa592da6e8b3fbfac7bdd4bb78"
 dependencies = [
  "anyhow",
  "deno_ast",
@@ -746,8 +824,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-udf-python"
-version = "0.1.0"
-source = "git+https://github.com/risingwavelabs/arrow-udf.git?rev=6c32f71#6c32f710b5948147f8214797fc334a4a3cadef0d"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4506efc6fbc200c083add2a7ed4e3616a859941a745e922320ae7051d90d12ec"
 dependencies = [
  "anyhow",
  "arrow-array 50.0.0",
@@ -761,9 +840,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-udf-wasm"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59a51355b8ca4de8ae028e5efb45c248dad4568cde6707f23b89f9b86a907f36"
+checksum = "eb829e25925161d93617d4b053bae03fe51e708f2cce088d85df856011d4f369"
 dependencies = [
  "anyhow",
  "arrow-array 50.0.0",
@@ -813,10 +892,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35"
 dependencies = [
  "concurrent-queue",
- "event-listener",
+ "event-listener 2.5.3",
  "futures-core",
 ]
 
+[[package]]
+name = "async-channel"
+version = "2.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "136d4d23bcc79e27423727b36823d86233aad06dfea531837b038394d11e9928"
+dependencies = [
+ "concurrent-queue",
+ "event-listener 5.2.0",
+ "event-listener-strategy",
+ "futures-core",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "async-compression"
 version = "0.4.5"
@@ -856,7 +948,7 @@ version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776"
 dependencies = [
- "async-channel",
+ "async-channel 1.9.0",
  "async-executor",
  "async-io",
  "async-lock",
@@ -892,14 +984,14 @@ version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b"
 dependencies = [
- "event-listener",
+ "event-listener 2.5.3",
 ]
 
 [[package]]
 name = "async-nats"
-version = "0.34.0"
+version = "0.35.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eea7b126ebfa4db78e9e788b2a792b6329f35b4f2fdd56dbc646dedc2beec7a5"
+checksum = "d5e47d2f7305524258908449aff6c86db36697a9b4219bfb1777e0ca1945358d"
 dependencies = [
  "base64 0.22.0",
  "bytes",
@@ -922,7 +1014,7 @@ dependencies = [
  "thiserror",
  "time",
  "tokio",
- "tokio-rustls 0.25.0",
+ "tokio-rustls 0.26.0",
  "tracing",
  "tryhard",
  "url",
@@ -946,7 +1038,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d"
 dependencies = [
  "async-attributes",
- "async-channel",
+ "async-channel 1.9.0",
  "async-global-executor",
  "async-io",
  "async-lock",
@@ -1005,6 +1097,19 @@ dependencies = [
  "syn 2.0.57",
 ]
 
+[[package]]
+name = "asynchronous-codec"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4057f2c32adbb2fc158e22fb38433c8e9bbf76b75a4732c7c0cbaf695fb65568"
+dependencies = [
+ "bytes",
+ "futures-sink",
+ "futures-util",
+ "memchr",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "atoi"
 version = "2.0.0"
@@ -1020,6 +1125,15 @@ version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba"
 
+[[package]]
+name = "atomic-time"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3424654267706036b8c23c0abadc4e0412416b9d0208d7ebe1e6978c8c31fec0"
+dependencies = [
+ "portable-atomic",
+]
+
 [[package]]
 name = "atomic-waker"
 version = "1.1.1"
@@ -1110,9 +1224,9 @@ dependencies = [
 
 [[package]]
 name = "aws-credential-types"
-version = "1.1.3"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d67c6836a1009b23e3f4cd1457c83e0aa49a490d9c3033b53c3f7b8cf2facc0f"
+checksum = "e16838e6c9e12125face1c1eff1343c75e3ff540de98ff7ebd61874a89bcfeb9"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-runtime-api",
@@ -1162,6 +1276,24 @@ dependencies = [
  "paste",
 ]
 
+[[package]]
+name = "aws-msk-iam-sasl-signer"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7036b8409ffe698dfdc5ae08722999d960092aeb738026ea99c3071c94831668"
+dependencies = [
+ "aws-config",
+ "aws-credential-types",
+ "aws-sdk-sts",
+ "aws-sigv4",
+ "aws-types",
+ "base64 0.22.0",
+ "chrono",
+ "futures",
+ "thiserror",
+ "url",
+]
+
 [[package]]
 name = "aws-runtime"
 version = "1.0.1"
@@ -1184,6 +1316,29 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "aws-sdk-dynamodb"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23c4ed3708df2778c0c49b16e8235e52eb8f2133ae6752c40eea1376e2563fec"
+dependencies = [
+ "aws-credential-types",
+ "aws-http",
+ "aws-runtime",
+ "aws-smithy-async",
+ "aws-smithy-http",
+ "aws-smithy-json",
+ "aws-smithy-runtime",
+ "aws-smithy-runtime-api",
+ "aws-smithy-types",
+ "aws-types",
+ "bytes",
+ "fastrand 2.0.1",
+ "http 0.2.9",
+ "regex",
+ "tracing",
+]
+
 [[package]]
 name = "aws-sdk-kinesis"
 version = "1.3.0"
@@ -1289,9 +1444,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-async"
-version = "1.1.3"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2eac0bb78e9e2765699999a02d7bfb4e6ad8f13e0962ebb9f5202b1d8cd76006"
+checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c"
 dependencies = [
  "futures-util",
  "pin-project-lite",
@@ -1390,31 +1545,33 @@ dependencies = [
  "once_cell",
  "pin-project-lite",
  "pin-utils",
- "rustls 0.21.10",
+ "rustls 0.21.11",
  "tokio",
  "tracing",
 ]
 
 [[package]]
 name = "aws-smithy-runtime-api"
-version = "1.1.3"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02ca2da7619517310bfead6d18abcdde90f1439224d887d608503cfacff46dff"
+checksum = "5b7d790d553d163c7d80a4e06e2906bf24b9172c9ebe045fc3a274e9358ab7bb"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-types",
  "bytes",
  "http 0.2.9",
+ "http 1.0.0",
  "pin-project-lite",
  "tokio",
  "tracing",
+ "zeroize",
 ]
 
 [[package]]
 name = "aws-smithy-types"
-version = "1.1.3"
+version = "1.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d4bb944488536cd2fef43212d829bc7e9a8bfc4afa079d21170441e7be8d2d0"
+checksum = "5b6764ba7e1c5ede1c9f9e4046645534f06c2581402461c559b481a420330a83"
 dependencies = [
  "base64-simd 0.8.0",
  "bytes",
@@ -1455,9 +1612,9 @@ dependencies = [
 
 [[package]]
 name = "aws-types"
-version = "1.1.3"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee2739d97d47f47cdf0d27982019a405dcc736df25925d1a75049f1faa79df88"
+checksum = "02fa328e19c849b20ef7ada4c9b581dd12351ff35ecc7642d06e69de4f98407c"
 dependencies = [
  "aws-credential-types",
  "aws-smithy-async",
@@ -1584,11 +1741,11 @@ dependencies = [
 
 [[package]]
 name = "backon"
-version = "0.4.1"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9"
+checksum = "d67782c3f868daa71d3533538e98a8e13713231969def7536e8039606fc46bf0"
 dependencies = [
- "fastrand 1.9.0",
+ "fastrand 2.0.1",
  "futures-core",
  "pin-project",
  "tokio",
@@ -1605,7 +1762,7 @@ dependencies = [
  "cfg-if",
  "libc",
  "miniz_oxide",
- "object",
+ "object 0.32.1",
  "rustc-demangle",
 ]
 
@@ -1673,11 +1830,11 @@ dependencies = [
 
 [[package]]
 name = "base64-url"
-version = "2.0.0"
+version = "3.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c5b0a88aa36e9f095ee2e2b13fb8c5e4313e022783aedacc123328c0084916d"
+checksum = "38e2b6c78c06f7288d5e3c3d683bde35a79531127c83b087e5d0d77c974b4b28"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.22.0",
 ]
 
 [[package]]
@@ -1737,6 +1894,7 @@ dependencies = [
  "num-bigint",
  "num-integer",
  "num-traits",
+ "serde",
 ]
 
 [[package]]
@@ -1820,12 +1978,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "bitmaps"
-version = "3.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "703642b98a00b3b90513279a8ede3fcfa479c126c5fb46e78f3051522f021403"
-
 [[package]]
 name = "bitvec"
 version = "1.0.1"
@@ -1894,7 +2046,7 @@ version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77231a1c8f801696fc0123ec6150ce92cffb8e164a02afb9c8ddee0e9b65ad65"
 dependencies = [
- "async-channel",
+ "async-channel 1.9.0",
  "async-lock",
  "async-task",
  "atomic-waker",
@@ -1905,9 +2057,9 @@ dependencies = [
 
 [[package]]
 name = "borsh"
-version = "1.2.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf617fabf5cdbdc92f774bfe5062d870f228b80056d41180797abf48bed4056e"
+checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed"
 dependencies = [
  "borsh-derive",
  "cfg_aliases",
@@ -1915,12 +2067,12 @@ dependencies = [
 
 [[package]]
 name = "borsh-derive"
-version = "1.2.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f404657a7ea7b5249e36808dff544bc88a28f26e0ac40009f674b7a009d14be3"
+checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b"
 dependencies = [
  "once_cell",
- "proc-macro-crate 2.0.0",
+ "proc-macro-crate 3.1.0",
  "proc-macro2",
  "quote",
  "syn 2.0.57",
@@ -1929,9 +2081,9 @@ dependencies = [
 
 [[package]]
 name = "brotli"
-version = "3.3.4"
+version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68"
+checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -1940,9 +2092,9 @@ dependencies = [
 
 [[package]]
 name = "brotli-decompressor"
-version = "2.3.4"
+version = "2.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744"
+checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -2228,9 +2380,9 @@ dependencies = [
 
 [[package]]
 name = "cfg_aliases"
-version = "0.1.1"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
 [[package]]
 name = "chrono"
@@ -2249,21 +2401,43 @@ dependencies = [
 
 [[package]]
 name = "chrono-tz"
-version = "0.8.3"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1369bc6b9e9a7dfdae2055f6ec151fe9c554a9d23d357c0237cee2e25eaabb7"
+checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e"
 dependencies = [
  "chrono",
- "chrono-tz-build",
+ "chrono-tz-build 0.2.1",
+ "phf",
+]
+
+[[package]]
+name = "chrono-tz"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb"
+dependencies = [
+ "chrono",
+ "chrono-tz-build 0.3.0",
  "phf",
  "uncased",
 ]
 
 [[package]]
 name = "chrono-tz-build"
-version = "0.2.0"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f"
+dependencies = [
+ "parse-zoneinfo",
+ "phf",
+ "phf_codegen",
+]
+
+[[package]]
+name = "chrono-tz-build"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2f5ebdc942f57ed96d560a6d1a459bae5851102a25d5bf89dc04ae453e31ecf"
+checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1"
 dependencies = [
  "parse-zoneinfo",
  "phf",
@@ -2372,7 +2546,7 @@ dependencies = [
  "hyper 0.14.27",
  "hyper-tls 0.5.0",
  "lz4",
- "sealed",
+ "sealed 0.4.0",
  "serde",
  "static_assertions",
  "thiserror",
@@ -2402,21 +2576,23 @@ dependencies = [
 ]
 
 [[package]]
-name = "cmake"
-version = "0.1.50"
+name = "cmac"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
+checksum = "8543454e3c3f5126effff9cd44d562af4e31fb8ce1cc0d3dcd8f084515dbc1aa"
 dependencies = [
- "cc",
+ "cipher",
+ "dbl",
+ "digest",
 ]
 
 [[package]]
-name = "cmsketch"
-version = "0.1.5"
+name = "cmake"
+version = "0.1.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93710598b87c37ea250ab17a36f9f79dbaf3bd20e55806cf09345103bc26d60e"
+checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
 dependencies = [
- "paste",
+ "cc",
 ]
 
 [[package]]
@@ -2491,6 +2667,12 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "connection-string"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "510ca239cf13b7f8d16a2b48f263de7b4f8c566f0af58d901031473c76afb1e3"
+
 [[package]]
 name = "console"
 version = "0.15.7"
@@ -2563,23 +2745,21 @@ checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f"
 
 [[package]]
 name = "const-random"
-version = "0.1.15"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e"
+checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
 dependencies = [
  "const-random-macro",
- "proc-macro-hack",
 ]
 
 [[package]]
 name = "const-random-macro"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb"
+checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
 dependencies = [
  "getrandom",
  "once_cell",
- "proc-macro-hack",
  "tiny-keccak",
 ]
 
@@ -2661,18 +2841,18 @@ dependencies = [
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.106.1"
+version = "0.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b3775cc6cc00c90d29eebea55feedb2b0168e23f5415bab7859c4004d7323d1"
+checksum = "79b27922a6879b5b5361d0a084cb0b1941bf109a98540addcb932da13b68bed4"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.106.1"
+version = "0.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "637f3184ba5bfa48d425bad1d2e4faf5fcf619f5e0ca107edc6dc02f589d4d74"
+checksum = "304c455b28bf56372729acb356afbb55d622f2b0f2f7837aa5e57c138acaac4d"
 dependencies = [
  "bumpalo",
  "cranelift-bforest",
@@ -2691,33 +2871,33 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.106.1"
+version = "0.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4b35b8240462341d94d31aab807cad704683988708261aecae3d57db48b7212"
+checksum = "1653c56b99591d07f67c5ca7f9f25888948af3f4b97186bff838d687d666f613"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.106.1"
+version = "0.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f3cd1555aa9df1d6d8375732de41b4cb0d787006948d55b6d004d521e9efeb0"
+checksum = "f5b6a9cf6b6eb820ee3f973a0db313c05dc12d370f37b4fe9630286e1672573f"
 
 [[package]]
 name = "cranelift-control"
-version = "0.106.1"
+version = "0.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14b31a562a10e98ab148fa146801e20665c5f9eda4fce9b2c5a3836575887d74"
+checksum = "d9d06e6bf30075fb6bed9e034ec046475093392eea1aff90eb5c44c4a033d19a"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.106.1"
+version = "0.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af1e0467700a3f4fccf5feddbaebdf8b0eb82535b06a9600c4bc5df40872e75d"
+checksum = "29be04f931b73cdb9694874a295027471817f26f26d2f0ebe5454153176b6e3a"
 dependencies = [
  "serde",
  "serde_derive",
@@ -2725,9 +2905,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.106.1"
+version = "0.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6cb918ee2c23939262efd1b99d76a21212ac7bd35129582133e21a22a6ff0467"
+checksum = "a07fd7393041d7faa2f37426f5dc7fc04003b70988810e8c063beefeff1cd8f9"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -2737,15 +2917,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.106.1"
+version = "0.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "966e4cfb23cf6d7f1d285d53a912baaffc5f06bcd9c9b0a2d8c66a184fae534b"
+checksum = "f341d7938caa6dff8149dac05bb2b53fc680323826b83b4cf175ab9f5139a3c9"
 
 [[package]]
 name = "cranelift-native"
-version = "0.106.1"
+version = "0.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bea803aadfc4aabdfae7c3870f1b1f6dd4332f4091859e9758ef5fca6bf8cc87"
+checksum = "82af6066e6448d26eeabb7aa26a43f7ff79f8217b06bade4ee6ef230aecc8880"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -2754,9 +2934,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-wasm"
-version = "0.106.1"
+version = "0.107.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11d18a3572cd897555bba3621e568029417d8f5cc26aeede2d7cb0bad6afd916"
+checksum = "2766fab7284a914a7f17f90ebe865c86453225fb8637ac31f123f5028fee69cd"
 dependencies = [
  "cranelift-codegen",
  "cranelift-entity",
@@ -2791,9 +2971,9 @@ checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff"
 
 [[package]]
 name = "crc32c"
-version = "0.6.4"
+version = "0.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74"
+checksum = "89254598aa9b9fa608de44b3ae54c810f0f06d755e24c50177f1f8f31ff50ce2"
 dependencies = [
  "rustc_version 0.4.0",
 ]
@@ -3036,9 +3216,9 @@ dependencies = [
 
 [[package]]
 name = "curve25519-dalek"
-version = "4.1.1"
+version = "4.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e89b8c6a2e4b1f45971ad09761aafb85514a84744b67a95e32c3cc1352d1f65c"
+checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -3263,8 +3443,8 @@ version = "33.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "676796427e638d85e9eadf13765705212be60b34f8fc5d3934d95184c63ca1b4"
 dependencies = [
- "ahash 0.8.6",
- "arrow",
+ "ahash 0.8.11",
+ "arrow 48.0.1",
  "arrow-array 48.0.1",
  "arrow-schema 48.0.1",
  "async-compression",
@@ -3310,8 +3490,8 @@ version = "33.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "31e23b3d21a6531259d291bd20ce59282ea794bda1018b0a1e278c13cd52e50c"
 dependencies = [
- "ahash 0.8.6",
- "arrow",
+ "ahash 0.8.11",
+ "arrow 48.0.1",
  "arrow-array 48.0.1",
  "arrow-buffer 48.0.1",
  "arrow-schema 48.0.1",
@@ -3329,7 +3509,7 @@ version = "33.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4de1fd0d8db0f2b8e4f4121bfa1c7c09d3a5c08a0a65c2229cd849eb65cff855"
 dependencies = [
- "arrow",
+ "arrow 48.0.1",
  "chrono",
  "dashmap",
  "datafusion-common",
@@ -3350,8 +3530,8 @@ version = "33.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "18e227fe88bf6730cab378d0cd8fc4c6b2ea42bc7e414a8ea9feba7225932735"
 dependencies = [
- "ahash 0.8.6",
- "arrow",
+ "ahash 0.8.11",
+ "arrow 48.0.1",
  "arrow-array 48.0.1",
  "datafusion-common",
  "sqlparser",
@@ -3365,7 +3545,7 @@ version = "33.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5c6648e62ea7605b9bfcd87fdc9d67e579c3b9ac563a87734ae5fe6d79ee4547"
 dependencies = [
- "arrow",
+ "arrow 48.0.1",
  "async-trait",
  "chrono",
  "datafusion-common",
@@ -3383,8 +3563,8 @@ version = "33.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f32b8574add16a32411a9b3fb3844ac1fc09ab4e7be289f86fd56d620e4f2508"
 dependencies = [
- "ahash 0.8.6",
- "arrow",
+ "ahash 0.8.11",
+ "arrow 48.0.1",
  "arrow-array 48.0.1",
  "arrow-buffer 48.0.1",
  "arrow-ord 48.0.1",
@@ -3418,8 +3598,8 @@ version = "33.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "796abd77d5bfecd9e5275a99daf0ec45f5b3a793ec431349ce8211a67826fd22"
 dependencies = [
- "ahash 0.8.6",
- "arrow",
+ "ahash 0.8.11",
+ "arrow 48.0.1",
  "arrow-array 48.0.1",
  "arrow-buffer 48.0.1",
  "arrow-schema 48.0.1",
@@ -3449,7 +3629,7 @@ version = "33.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26de2592417beb20f73f29b131a04d7de14e2a6336c631554d611584b4306236"
 dependencies = [
- "arrow",
+ "arrow 48.0.1",
  "chrono",
  "datafusion",
  "datafusion-common",
@@ -3464,7 +3644,7 @@ version = "33.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ced70b8a5648ba7b95c61fc512183c33287ffe2c9f22ffe22700619d7d48c84f"
 dependencies = [
- "arrow",
+ "arrow 48.0.1",
  "arrow-schema 48.0.1",
  "datafusion-common",
  "datafusion-expr",
@@ -3490,6 +3670,15 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "dbl"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd2735a791158376708f9347fe8faba9667589d82427ef3aed6794a8981de3d9"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "debugid"
 version = "0.8.0"
@@ -3502,9 +3691,9 @@ dependencies = [
 
 [[package]]
 name = "delta_btree_map"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
- "educe 0.5.7",
+ "educe",
  "enum-as-inner 0.6.0",
 ]
 
@@ -3521,7 +3710,7 @@ name = "deltalake-core"
 version = "0.17.0"
 source = "git+https://github.com/risingwavelabs/delta-rs?rev=5c2dccd4640490202ffe98adbd13b09cef8e007b#5c2dccd4640490202ffe98adbd13b09cef8e007b"
 dependencies = [
- "arrow",
+ "arrow 48.0.1",
  "arrow-array 48.0.1",
  "arrow-buffer 48.0.1",
  "arrow-cast 48.0.1",
@@ -3703,7 +3892,7 @@ dependencies = [
  "dyn-clone",
  "http 1.0.0",
  "pin-project",
- "reqwest 0.12.2",
+ "reqwest 0.12.4",
  "serde",
  "serde_json",
  "tokio",
@@ -3837,7 +4026,7 @@ dependencies = [
  "deno_core",
  "deno_native_certs",
  "once_cell",
- "rustls 0.22.3",
+ "rustls 0.22.4",
  "rustls-pemfile 1.0.4",
  "rustls-tokio-stream",
  "rustls-webpki 0.101.7",
@@ -4227,16 +4416,16 @@ checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b"
 
 [[package]]
 name = "duration-str"
-version = "0.7.1"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8bb6a301a95ba86fa0ebaf71d49ae4838c51f8b84cb88ed140dfb66452bb3c4"
+checksum = "7c1a2e028bbf7921549873b291ddc0cfe08b673d9489da81ac28898cd5a0e6e0"
 dependencies = [
  "chrono",
- "nom",
  "rust_decimal",
  "serde",
  "thiserror",
  "time",
+ "winnow 0.6.11",
 ]
 
 [[package]]
@@ -4304,23 +4493,11 @@ dependencies = [
 
 [[package]]
 name = "educe"
-version = "0.4.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f0042ff8246a363dbe77d2ceedb073339e85a804b9a47636c6e016a9a32c05f"
-dependencies = [
- "enum-ordinalize 3.1.13",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "educe"
-version = "0.5.7"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "597b6c9e278fefec3b25dfbb86195aeafe270dc6f13e27a3a20a95ba449f1bfb"
+checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417"
 dependencies = [
- "enum-ordinalize 4.3.0",
+ "enum-ordinalize",
  "proc-macro2",
  "quote",
  "syn 2.0.57",
@@ -4328,9 +4505,9 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.11.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2"
+checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b"
 dependencies = [
  "serde",
 ]
@@ -4382,6 +4559,70 @@ version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
 
+[[package]]
+name = "encoding"
+version = "0.2.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
+dependencies = [
+ "encoding-index-japanese",
+ "encoding-index-korean",
+ "encoding-index-simpchinese",
+ "encoding-index-singlebyte",
+ "encoding-index-tradchinese",
+]
+
+[[package]]
+name = "encoding-index-japanese"
+version = "1.20141219.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
+dependencies = [
+ "encoding_index_tests",
+]
+
+[[package]]
+name = "encoding-index-korean"
+version = "1.20141219.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
+dependencies = [
+ "encoding_index_tests",
+]
+
+[[package]]
+name = "encoding-index-simpchinese"
+version = "1.20141219.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7"
+dependencies = [
+ "encoding_index_tests",
+]
+
+[[package]]
+name = "encoding-index-singlebyte"
+version = "1.20141219.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a"
+dependencies = [
+ "encoding_index_tests",
+]
+
+[[package]]
+name = "encoding-index-tradchinese"
+version = "1.20141219.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18"
+dependencies = [
+ "encoding_index_tests",
+]
+
+[[package]]
+name = "encoding_index_tests"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.33"
@@ -4437,32 +4678,19 @@ dependencies = [
 
 [[package]]
 name = "enum-iterator"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "600536cfe9e2da0820aa498e570f6b2b9223eec3ce2f835c8ae4861304fa4794"
-dependencies = [
- "enum-iterator-derive",
-]
-
-[[package]]
-name = "enum-iterator-derive"
-version = "1.3.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8"
+checksum = "c280b9e6b3ae19e152d8e31cf47f18389781e119d4013a2a2bb0180e5facc635"
 dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.57",
+ "enum-iterator-derive",
 ]
 
 [[package]]
-name = "enum-ordinalize"
-version = "3.1.13"
+name = "enum-iterator-derive"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4f76552f53cefc9a7f64987c3701b99d982f7690606fd67de1d09712fbf52f1"
+checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b"
 dependencies = [
- "num-bigint",
- "num-traits",
  "proc-macro2",
  "quote",
  "syn 2.0.57",
@@ -4596,6 +4824,27 @@ version = "2.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
 
+[[package]]
+name = "event-listener"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b5fb89194fa3cad959b833185b3063ba881dbfc7030680b314250779fb4cc91"
+dependencies = [
+ "concurrent-queue",
+ "parking",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "event-listener-strategy"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "332f51cb23d20b0de8458b86580878211da09bcd4503cb579c225b3d124cabb3"
+dependencies = [
+ "event-listener 5.2.0",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "expect-test"
 version = "1.5.0"
@@ -4891,130 +5140,102 @@ dependencies = [
 
 [[package]]
 name = "foyer"
-version = "0.6.0"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15240094bab62dfeb59bb974e2fb7bff18727c6dc7bd1f5f6fc82a9e6fda5a38"
+checksum = "0720e9cc504174e952382345ea25fe887fbd4c1761dccdd4c5276c6d1a93dbc5"
 dependencies = [
+ "ahash 0.8.11",
+ "anyhow",
  "foyer-common",
- "foyer-intrusive",
  "foyer-memory",
  "foyer-storage",
- "foyer-workspace-hack",
+ "madsim-tokio",
+ "tracing",
 ]
 
 [[package]]
 name = "foyer-common"
-version = "0.4.0"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad95f985cfbd5a8bf9f115d963918e742dfbb75350febff509b09cf194842b0c"
+checksum = "8a823da3228992cc6392ddac757e0ed44fa4923588b03fcbbba4bad1ebfea7b2"
 dependencies = [
- "anyhow",
  "bytes",
- "foyer-workspace-hack",
- "itertools 0.12.1",
+ "cfg-if",
+ "crossbeam",
+ "hashbrown 0.14.3",
+ "itertools 0.13.0",
  "madsim-tokio",
+ "metrics",
+ "nix 0.29.0",
  "parking_lot 0.12.1",
- "paste",
- "tracing",
+ "rustversion",
+ "serde",
 ]
 
 [[package]]
 name = "foyer-intrusive"
-version = "0.3.0"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acf22ed0dfa6315714099046504711d5563d191a442afcd87bd4d0bf5010bb9a"
+checksum = "5168b34c3e6369a4c6055f08c6b37e416910652bfdd9a84de55a602f2aa3dcea"
 dependencies = [
- "bytes",
- "cmsketch 0.1.5",
  "foyer-common",
- "foyer-workspace-hack",
- "itertools 0.12.1",
- "memoffset",
- "parking_lot 0.12.1",
- "paste",
- "tracing",
- "twox-hash",
+ "itertools 0.13.0",
 ]
 
 [[package]]
 name = "foyer-memory"
-version = "0.1.3"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f59727e86709e4eab603f8a5086f3b2c7d3ddddf7c8b31d8dea2b00364b7fb95"
+checksum = "56055774fa6ed25d0affe30c04e6a3f18fc00713a42ef6a3dbd96e72fe31a3eb"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "bitflags 2.5.0",
- "cmsketch 0.2.0",
- "crossbeam",
+ "cmsketch",
+ "foyer-common",
  "foyer-intrusive",
- "foyer-workspace-hack",
  "futures",
  "hashbrown 0.14.3",
- "itertools 0.12.1",
+ "itertools 0.13.0",
  "libc",
  "madsim-tokio",
  "parking_lot 0.12.1",
+ "serde",
+ "tracing",
 ]
 
 [[package]]
 name = "foyer-storage"
-version = "0.5.1"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5e0749c569ef5f8d9f877a6a81da24e7d1b6f8ea87ec6c3933e12dac0c7632f"
+checksum = "74811aec3df193b9c304d031e9f4879c4aab86cb78ff6e4cadf1188d014e7069"
 dependencies = [
+ "ahash 0.8.11",
+ "allocator-api2",
  "anyhow",
+ "array-util",
+ "async-channel 2.2.1",
+ "bincode 1.3.3",
  "bitflags 2.5.0",
- "bitmaps",
  "bytes",
+ "either",
  "foyer-common",
- "foyer-intrusive",
- "foyer-workspace-hack",
+ "foyer-memory",
  "futures",
- "itertools 0.12.1",
+ "itertools 0.13.0",
+ "lazy_static",
  "libc",
  "lz4",
  "madsim-tokio",
- "memoffset",
- "nix 0.28.0",
  "parking_lot 0.12.1",
- "paste",
- "prometheus",
+ "pin-project",
  "rand",
+ "serde",
  "thiserror",
  "tracing",
  "twox-hash",
  "zstd 0.13.0",
 ]
 
-[[package]]
-name = "foyer-workspace-hack"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77a5ee5ab890d1893232fd44af33197c7499813816b8880649a15ee9438d5e07"
-dependencies = [
- "ahash 0.8.6",
- "cc",
- "crossbeam-channel",
- "crossbeam-utils",
- "either",
- "futures-channel",
- "futures-core",
- "futures-executor",
- "futures-sink",
- "futures-util",
- "hashbrown 0.14.3",
- "libc",
- "parking_lot 0.12.1",
- "parking_lot_core 0.9.8",
- "proc-macro2",
- "quote",
- "rand",
- "smallvec",
- "syn 2.0.57",
- "tokio",
- "tracing-core",
-]
-
 [[package]]
 name = "fragile"
 version = "2.0.0"
@@ -5437,9 +5658,9 @@ dependencies = [
 
 [[package]]
 name = "ginepro"
-version = "0.7.0"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eedbff62a689be48f58f32571dbf3d60c4a73b39740141dfe7ac942536ea27f7"
+checksum = "3b00ef897d4082727a53ea1111cd19bfa4ccdc476a5eb9f49087047113a43891"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -5472,9 +5693,9 @@ dependencies = [
 
 [[package]]
 name = "google-cloud-auth"
-version = "0.14.0"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe54dd8b6eb2bcd5390998238bcc39d1daed4dbb70df2845832532540384fc41"
+checksum = "e09ed5b2998bc8d0d3df09c859028210d4961b8fe779cfda8dc8ca4e83d5def2"
 dependencies = [
  "async-trait",
  "base64 0.21.7",
@@ -5482,7 +5703,7 @@ dependencies = [
  "google-cloud-token",
  "home",
  "jsonwebtoken",
- "reqwest 0.11.20",
+ "reqwest 0.12.4",
  "serde",
  "serde_json",
  "thiserror",
@@ -5492,6 +5713,33 @@ dependencies = [
  "urlencoding",
 ]
 
+[[package]]
+name = "google-cloud-bigquery"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e321c127945bb44a5cf5129c37530e2494b97afefe7f334a983ac754e40914e"
+dependencies = [
+ "anyhow",
+ "arrow 50.0.0",
+ "async-trait",
+ "backon",
+ "base64 0.21.7",
+ "bigdecimal 0.4.2",
+ "google-cloud-auth",
+ "google-cloud-gax",
+ "google-cloud-googleapis",
+ "google-cloud-token",
+ "num-bigint",
+ "reqwest 0.12.4",
+ "reqwest-middleware",
+ "serde",
+ "serde_json",
+ "thiserror",
+ "time",
+ "tokio",
+ "tracing",
+]
+
 [[package]]
 name = "google-cloud-gax"
 version = "0.17.0"
@@ -5510,9 +5758,9 @@ dependencies = [
 
 [[package]]
 name = "google-cloud-googleapis"
-version = "0.12.0"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db8a478015d079296167e3f08e096dc99cffc2cb50fa203dd38aaa9dd37f8354"
+checksum = "32cd184c52aa2619ac1b16ad8b5a752e91d25be88a8cf08eaec19777dfacbe54"
 dependencies = [
  "prost 0.12.1",
  "prost-types 0.12.1",
@@ -5521,22 +5769,22 @@ dependencies = [
 
 [[package]]
 name = "google-cloud-metadata"
-version = "0.4.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc279bfb50487d7bcd900e8688406475fc750fe474a835b2ab9ade9eb1fc90e2"
+checksum = "04f945a208886a13d07636f38fb978da371d0abc3e34bad338124b9f8c135a8f"
 dependencies = [
- "reqwest 0.11.20",
+ "reqwest 0.12.4",
  "thiserror",
  "tokio",
 ]
 
 [[package]]
 name = "google-cloud-pubsub"
-version = "0.24.0"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b2184a5c70b994e6d77eb1c140e193e7f5fe6015e9115322fac24f7e33f003c"
+checksum = "0a35e4a008db5cf01a5c03d3c67bd90b3cad77427ca949f3c8eddd90c4a3c932"
 dependencies = [
- "async-channel",
+ "async-channel 1.9.0",
  "async-stream",
  "google-cloud-auth",
  "google-cloud-gax",
@@ -5677,7 +5925,7 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 dependencies = [
- "ahash 0.7.7",
+ "ahash 0.7.8",
 ]
 
 [[package]]
@@ -5686,7 +5934,7 @@ version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
 ]
 
 [[package]]
@@ -5695,7 +5943,7 @@ version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "allocator-api2",
 ]
 
@@ -5949,7 +6197,7 @@ dependencies = [
  "http 0.2.9",
  "hyper 0.14.27",
  "log",
- "rustls 0.21.10",
+ "rustls 0.21.11",
  "rustls-native-certs 0.6.3",
  "tokio",
  "tokio-rustls 0.24.1",
@@ -5966,7 +6214,7 @@ dependencies = [
  "http 1.0.0",
  "hyper 1.1.0",
  "hyper-util",
- "rustls 0.22.3",
+ "rustls 0.22.4",
  "rustls-pki-types",
  "tokio",
  "tokio-rustls 0.25.0",
@@ -6219,7 +6467,7 @@ version = "0.11.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73c0fefcb6d409a6587c07515951495d482006f89a21daa0f2f783aa4fd5e027"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "indexmap 2.0.0",
  "is-terminal",
  "itoa",
@@ -6320,7 +6568,7 @@ dependencies = [
  "socket2 0.5.6",
  "widestring",
  "windows-sys 0.48.0",
- "winreg",
+ "winreg 0.50.0",
 ]
 
 [[package]]
@@ -6379,6 +6627,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.9"
@@ -6459,9 +6716,9 @@ dependencies = [
 
 [[package]]
 name = "jsonbb"
-version = "0.1.3"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e2edfc17ad42a5ece82df036301d5ef0c3dc3d071e28aa8a62e461711c55d19"
+checksum = "91cdcbd02ee94c68803dd808bf8406e91491eaf875f09da650f5893dc56be18c"
 dependencies = [
  "bytes",
  "serde",
@@ -6623,9 +6880,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.153"
+version = "0.2.155"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
+checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
 
 [[package]]
 name = "libflate"
@@ -6689,17 +6946,6 @@ dependencies = [
  "vcpkg",
 ]
 
-[[package]]
-name = "libtest-mimic"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d8de370f98a6cb8a4606618e53e802f93b094ddec0f96988eaec2c27e6e9ce7"
-dependencies = [
- "clap",
- "termcolor",
- "threadpool",
-]
-
 [[package]]
 name = "libtest-mimic"
 version = "0.7.0"
@@ -6779,7 +7025,7 @@ checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456"
 
 [[package]]
 name = "local_stats_alloc"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "workspace-hack",
 ]
@@ -6869,32 +7115,23 @@ dependencies = [
  "pkg-config",
 ]
 
-[[package]]
-name = "mach"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "mach2"
-version = "0.4.1"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8"
+checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "madsim"
-version = "0.2.22"
+version = "0.2.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7777a8bc4e68878b6e5433ac7b9bc196d9ccdfeef1f7cb3d23193cb997a520c9"
+checksum = "f88753ddf8d3cd43b9cf71a93626dd9aad3c24086a04420beb31922e1f856d02"
 dependencies = [
- "ahash 0.7.7",
- "async-channel",
+ "ahash 0.8.11",
+ "async-channel 2.2.1",
  "async-stream",
  "async-task",
  "bincode 1.3.3",
@@ -6913,7 +7150,7 @@ dependencies = [
  "spin 0.9.8",
  "tokio",
  "tokio-util",
- "toml 0.7.8",
+ "toml 0.8.12",
  "tracing",
  "tracing-subscriber",
 ]
@@ -6970,11 +7207,11 @@ dependencies = [
 
 [[package]]
 name = "madsim-rdkafka"
-version = "0.3.4+0.34.0"
+version = "0.4.2+0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0992d46c19414bda24bad935ea223ad025dcd7e64f38f0acee388efa8ff5319b"
+checksum = "6038ceb55ecf0d80eeae01a85dc9db052f0629ca2d82732923c022185096a76c"
 dependencies = [
- "async-channel",
+ "async-channel 1.9.0",
  "async-trait",
  "futures-channel",
  "futures-util",
@@ -6994,9 +7231,9 @@ dependencies = [
 
 [[package]]
 name = "madsim-tokio"
-version = "0.2.24"
+version = "0.2.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5611fd0eb96867dd03a9fd2494d4c1bb126f413519673195065b6ea011e8c68"
+checksum = "569929c869275edc1e2c1f1381a688a6e5a4200302b58caff819e07414ccddb9"
 dependencies = [
  "madsim",
  "spin 0.9.8",
@@ -7138,6 +7375,45 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "metrics"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "884adb57038347dfbaf2d5065887b6cf4312330dc8e94bc30a1a839bd79d3261"
+dependencies = [
+ "ahash 0.8.11",
+ "portable-atomic",
+]
+
+[[package]]
+name = "metrics-prometheus"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51557a875fdbd5b953b698ecd6cd06efef47618e02d95ad912e2392e5b5617ff"
+dependencies = [
+ "arc-swap",
+ "metrics",
+ "metrics-util",
+ "once_cell",
+ "prometheus",
+ "sealed 0.5.0",
+ "smallvec",
+ "thiserror",
+]
+
+[[package]]
+name = "metrics-util"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4259040465c955f9f2f1a4a8a16dc46726169bca0f88e8fb2dbeced487c3e828"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "hashbrown 0.14.3",
+ "metrics",
+ "num_cpus",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -7420,9 +7696,9 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.28.0"
+version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4"
+checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
 dependencies = [
  "bitflags 2.5.0",
  "cfg-if",
@@ -7688,6 +7964,15 @@ name = "object"
 version = "0.32.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "object"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8dd6c0cdf9429bce006e1362bfce61fa1bfd8c898a643ed8d2b471934701d3d"
 dependencies = [
  "crc32fast",
  "hashbrown 0.14.3",
@@ -7838,15 +8123,6 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
-[[package]]
-name = "openssl-src"
-version = "300.1.6+3.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085"
-dependencies = [
- "cc",
-]
-
 [[package]]
 name = "openssl-sys"
 version = "0.9.101"
@@ -7855,7 +8131,6 @@ checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff"
 dependencies = [
  "cc",
  "libc",
- "openssl-src",
  "pkg-config",
  "vcpkg",
 ]
@@ -8049,9 +8324,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 
 [[package]]
 name = "owo-colors"
-version = "3.5.0"
+version = "4.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f"
+checksum = "caff54706df99d2a78a5a4e3455ff45448d81ef1bb63c22cd14052ca0e993a3f"
 
 [[package]]
 name = "p256"
@@ -8171,7 +8446,7 @@ version = "48.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6bfe55df96e3f02f11bf197ae37d91bb79801631f82f6195dd196ef521df3597"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "arrow-array 48.0.1",
  "arrow-buffer 48.0.1",
  "arrow-cast 48.0.1",
@@ -8205,7 +8480,7 @@ version = "50.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "arrow-array 50.0.0",
  "arrow-buffer 50.0.0",
  "arrow-cast 50.0.0",
@@ -8390,7 +8665,7 @@ dependencies = [
 
 [[package]]
 name = "pgwire"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "auto_enums",
@@ -8403,7 +8678,7 @@ dependencies = [
  "openssl",
  "panic-message",
  "parking_lot 0.12.1",
- "reqwest 0.12.2",
+ "reqwest 0.12.4",
  "risingwave_common",
  "risingwave_sqlparser",
  "serde",
@@ -8631,9 +8906,9 @@ dependencies = [
 
 [[package]]
 name = "portable-atomic"
-version = "1.4.3"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b"
+checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"
 
 [[package]]
 name = "postgres-derive"
@@ -8764,14 +9039,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "04bfa62906ce8d9badf8d1764501640ae7f0bcea3437a209315830e0f73564d1"
 
 [[package]]
-name = "prepare_ci_pubsub"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "google-cloud-googleapis",
- "google-cloud-pubsub",
- "madsim-tokio",
-]
+name = "pretty-hex"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5"
 
 [[package]]
 name = "pretty-xmlish"
@@ -8828,15 +9099,6 @@ dependencies = [
  "toml_edit 0.19.15",
 ]
 
-[[package]]
-name = "proc-macro-crate"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8"
-dependencies = [
- "toml_edit 0.20.2",
-]
-
 [[package]]
 name = "proc-macro-crate"
 version = "3.1.0"
@@ -9137,9 +9399,9 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
 
 [[package]]
 name = "protobuf-native"
-version = "0.2.1+3.19.1"
+version = "0.2.2+3.19.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86df76d0f2a6933036e8a9f28f1adc8b48081fa681dba07eaa30ac75663f7f4e"
+checksum = "577feb02952883f2348ee6f36afeeb4e0ee22be0e5b707d60bdc2f5aab038e8f"
 dependencies = [
  "cxx",
  "cxx-build",
@@ -9200,9 +9462,9 @@ dependencies = [
 
 [[package]]
 name = "pulsar"
-version = "6.1.0"
+version = "6.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d21c6a837986cf25d22ac5b951c267d95808f3c830ff009c2879fff259a0268"
+checksum = "d7f3541ff84e39da334979ac4bf171e0f277f4f782603aeae65bf5795dc7275a"
 dependencies = [
  "async-trait",
  "bit-vec",
@@ -9238,9 +9500,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3"
-version = "0.20.3"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233"
+checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8"
 dependencies = [
  "cfg-if",
  "indoc",
@@ -9256,9 +9518,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.20.3"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7"
+checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50"
 dependencies = [
  "once_cell",
  "target-lexicon",
@@ -9266,9 +9528,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.20.3"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa"
+checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -9276,9 +9538,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.20.3"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158"
+checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -9288,9 +9550,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.20.3"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185"
+checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c"
 dependencies = [
  "heck 0.4.1",
  "proc-macro2",
@@ -9621,7 +9883,7 @@ dependencies = [
 [[package]]
 name = "reqsign"
 version = "0.14.9"
-source = "git+https://github.com/wcy-fdu/reqsign.git?rev=002ee2a#002ee2a41749b08bb5336f344e31f514d8fce718"
+source = "git+https://github.com/wcy-fdu/reqsign.git?rev=c7dd668#c7dd668764ada1e7477177cfa913fec24252dd34"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -9673,7 +9935,7 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
- "rustls 0.21.10",
+ "rustls 0.21.11",
  "rustls-native-certs 0.6.3",
  "rustls-pemfile 1.0.4",
  "serde",
@@ -9690,17 +9952,17 @@ dependencies = [
  "wasm-streams 0.3.0",
  "web-sys",
  "webpki-roots 0.25.2",
- "winreg",
+ "winreg 0.50.0",
 ]
 
 [[package]]
 name = "reqwest"
-version = "0.12.2"
+version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338"
+checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10"
 dependencies = [
  "async-compression",
- "base64 0.21.7",
+ "base64 0.22.0",
  "bytes",
  "encoding_rs",
  "futures-channel",
@@ -9718,12 +9980,13 @@ dependencies = [
  "js-sys",
  "log",
  "mime",
+ "mime_guess",
  "native-tls",
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
- "rustls 0.22.3",
- "rustls-pemfile 1.0.4",
+ "rustls 0.22.4",
+ "rustls-pemfile 2.1.1",
  "rustls-pki-types",
  "serde",
  "serde_json",
@@ -9742,7 +10005,22 @@ dependencies = [
  "wasm-streams 0.4.0",
  "web-sys",
  "webpki-roots 0.26.1",
- "winreg",
+ "winreg 0.52.0",
+]
+
+[[package]]
+name = "reqwest-middleware"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a45d100244a467870f6cb763c4484d010a6bed6bd610b3676e3825d93fb4cfbd"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "http 1.0.0",
+ "reqwest 0.12.4",
+ "serde",
+ "thiserror",
+ "tower-service",
 ]
 
 [[package]]
@@ -9822,7 +10100,7 @@ dependencies = [
 
 [[package]]
 name = "risedev"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "chrono",
@@ -9837,7 +10115,7 @@ dependencies = [
  "madsim-tokio",
  "redis",
  "regex",
- "reqwest 0.12.2",
+ "reqwest 0.12.4",
  "serde",
  "serde_json",
  "serde_with",
@@ -9852,7 +10130,7 @@ dependencies = [
 
 [[package]]
 name = "risedev-config"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "clap",
@@ -9865,7 +10143,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave-fields-derive"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "expect-test",
  "indoc",
@@ -9877,7 +10155,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_backup"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -9899,7 +10177,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_batch"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "arrow-array 50.0.0",
@@ -9907,6 +10185,7 @@ dependencies = [
  "assert_matches",
  "async-recursion",
  "async-trait",
+ "bytes",
  "criterion",
  "either",
  "foyer",
@@ -9920,9 +10199,11 @@ dependencies = [
  "madsim-tokio",
  "madsim-tonic",
  "memcomparable",
+ "opendal",
  "parking_lot 0.12.1",
  "paste",
  "prometheus",
+ "prost 0.12.1",
  "rand",
  "risingwave_common",
  "risingwave_common_estimate_size",
@@ -9945,12 +10226,14 @@ dependencies = [
  "tokio-stream",
  "tokio-util",
  "tracing",
+ "twox-hash",
+ "uuid",
  "workspace-hack",
 ]
 
 [[package]]
 name = "risingwave_bench"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -9968,7 +10251,7 @@ dependencies = [
  "itertools 0.12.1",
  "libc",
  "madsim-tokio",
- "nix 0.28.0",
+ "nix 0.29.0",
  "opentelemetry",
  "parking_lot 0.12.1",
  "prometheus",
@@ -9991,7 +10274,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_cmd"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "clap",
  "madsim-tokio",
@@ -10011,7 +10294,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_cmd_all"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "clap",
@@ -10026,7 +10309,6 @@ dependencies = [
  "risingwave_compactor",
  "risingwave_compute",
  "risingwave_ctl",
- "risingwave_expr",
  "risingwave_expr_impl",
  "risingwave_frontend",
  "risingwave_meta_node",
@@ -10045,8 +10327,9 @@ dependencies = [
 
 [[package]]
 name = "risingwave_common"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
+ "ahash 0.8.11",
  "anyhow",
  "arc-swap",
  "arrow-array 48.0.1",
@@ -10064,14 +10347,15 @@ dependencies = [
  "byteorder",
  "bytes",
  "chrono",
- "chrono-tz",
+ "chrono-tz 0.9.0",
  "clap",
+ "coarsetime",
  "comfy-table",
  "crc32fast",
  "criterion",
  "darwin-libproc",
  "easy-ext",
- "educe 0.5.7",
+ "educe",
  "either",
  "enum-as-inner 0.6.0",
  "enumflags2",
@@ -10081,6 +10365,7 @@ dependencies = [
  "foyer",
  "futures",
  "governor",
+ "hashbrown 0.14.3",
  "hex",
  "http 0.2.9",
  "http-body 0.4.5",
@@ -10111,7 +10396,7 @@ dependencies = [
  "prost 0.12.1",
  "rand",
  "regex",
- "reqwest 0.12.2",
+ "reqwest 0.12.4",
  "risingwave-fields-derive",
  "risingwave_common_estimate_size",
  "risingwave_common_metrics",
@@ -10155,10 +10440,10 @@ dependencies = [
 
 [[package]]
 name = "risingwave_common_estimate_size"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "bytes",
- "educe 0.5.7",
+ "educe",
  "ethnum",
  "fixedbitset 0.5.0",
  "jsonbb",
@@ -10170,7 +10455,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_common_heap_profiling"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "chrono",
@@ -10185,7 +10470,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_common_metrics"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "bytes",
  "clap",
@@ -10217,7 +10502,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_common_proc_macro"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "bae",
  "itertools 0.12.1",
@@ -10229,7 +10514,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_common_service"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "async-trait",
  "axum 0.7.4",
@@ -10251,7 +10536,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_compaction_test"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -10279,7 +10564,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_compactor"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "async-trait",
  "await-tree",
@@ -10302,7 +10587,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_compute"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -10347,7 +10632,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_connector"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "apache-avro 0.16.0",
@@ -10360,8 +10645,11 @@ dependencies = [
  "async-trait",
  "auto_enums",
  "auto_impl",
+ "await-tree",
  "aws-config",
  "aws-credential-types",
+ "aws-msk-iam-sasl-signer",
+ "aws-sdk-dynamodb",
  "aws-sdk-kinesis",
  "aws-sdk-s3",
  "aws-smithy-http",
@@ -10372,6 +10660,7 @@ dependencies = [
  "base64 0.22.0",
  "byteorder",
  "bytes",
+ "cfg-or-panic",
  "chrono",
  "clickhouse",
  "criterion",
@@ -10385,12 +10674,16 @@ dependencies = [
  "futures-async-stream",
  "gcp-bigquery-client",
  "glob",
+ "google-cloud-bigquery",
+ "google-cloud-gax",
+ "google-cloud-googleapis",
  "google-cloud-pubsub",
  "http 0.2.9",
  "icelake",
  "indexmap 1.9.3",
  "itertools 0.12.1",
  "jni",
+ "jsonbb",
  "jsonschema-transpiler",
  "jsonwebtoken",
  "madsim-rdkafka",
@@ -10421,9 +10714,10 @@ dependencies = [
  "rand",
  "redis",
  "regex",
- "reqwest 0.12.2",
+ "reqwest 0.12.4",
  "risingwave_common",
  "risingwave_common_estimate_size",
+ "risingwave_connector_codec",
  "risingwave_jni_core",
  "risingwave_object_store",
  "risingwave_pb",
@@ -10434,18 +10728,21 @@ dependencies = [
  "rustls-pemfile 2.1.1",
  "rustls-pki-types",
  "rw_futures_util",
+ "sea-schema",
  "serde",
  "serde_derive",
  "serde_json",
  "serde_with",
  "serde_yaml",
  "simd-json",
+ "sqlx",
  "strum 0.26.1",
  "strum_macros 0.26.1",
  "syn 1.0.109",
  "tempfile",
  "thiserror",
  "thiserror-ext",
+ "tiberius",
  "time",
  "tokio-postgres",
  "tokio-retry",
@@ -10463,15 +10760,37 @@ dependencies = [
  "yup-oauth2",
 ]
 
+[[package]]
+name = "risingwave_connector_codec"
+version = "1.10.0-alpha"
+dependencies = [
+ "anyhow",
+ "apache-avro 0.16.0",
+ "chrono",
+ "easy-ext",
+ "itertools 0.12.1",
+ "jsonbb",
+ "num-bigint",
+ "risingwave_common",
+ "risingwave_pb",
+ "rust_decimal",
+ "thiserror",
+ "thiserror-ext",
+ "time",
+ "tracing",
+ "workspace-hack",
+]
+
 [[package]]
 name = "risingwave_ctl"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "bytes",
  "chrono",
  "clap",
  "comfy-table",
+ "foyer",
  "futures",
  "hex",
  "inquire",
@@ -10506,7 +10825,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_dml"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "assert_matches",
  "criterion",
@@ -10530,7 +10849,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_e2e_extended_mode_test"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "chrono",
@@ -10545,7 +10864,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_error"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "bincode 1.3.3",
@@ -10561,15 +10880,11 @@ dependencies = [
 
 [[package]]
 name = "risingwave_expr"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "arrow-array 50.0.0",
  "arrow-schema 50.0.0",
- "arrow-udf-js",
- "arrow-udf-js-deno",
- "arrow-udf-python",
- "arrow-udf-wasm",
  "async-trait",
  "auto_impl",
  "await-tree",
@@ -10578,7 +10893,7 @@ dependencies = [
  "const-currying",
  "downcast-rs",
  "easy-ext",
- "educe 0.5.7",
+ "educe",
  "either",
  "enum-as-inner 0.6.0",
  "expect-test",
@@ -10588,42 +10903,47 @@ dependencies = [
  "itertools 0.12.1",
  "linkme",
  "madsim-tokio",
- "md5",
- "moka",
  "num-traits",
- "openssl",
  "parse-display",
  "paste",
+ "prometheus",
  "risingwave_common",
  "risingwave_common_estimate_size",
  "risingwave_expr_macro",
  "risingwave_pb",
- "risingwave_udf",
  "smallvec",
  "static_assertions",
  "thiserror",
  "thiserror-ext",
  "tracing",
  "workspace-hack",
- "zstd 0.13.0",
 ]
 
 [[package]]
 name = "risingwave_expr_impl"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "aho-corasick",
  "anyhow",
+ "arrow-array 50.0.0",
+ "arrow-flight",
  "arrow-schema 50.0.0",
+ "arrow-udf-flight",
+ "arrow-udf-js",
+ "arrow-udf-js-deno",
+ "arrow-udf-python",
+ "arrow-udf-wasm",
  "async-trait",
  "auto_enums",
  "chrono",
- "chrono-tz",
+ "chrono-tz 0.9.0",
  "criterion",
+ "educe",
  "expect-test",
  "fancy-regex",
  "futures-async-stream",
  "futures-util",
+ "ginepro",
  "hex",
  "icelake",
  "itertools 0.12.1",
@@ -10631,6 +10951,7 @@ dependencies = [
  "linkme",
  "madsim-tokio",
  "md5",
+ "moka",
  "num-traits",
  "openssl",
  "regex",
@@ -10638,17 +10959,21 @@ dependencies = [
  "risingwave_common_estimate_size",
  "risingwave_expr",
  "risingwave_pb",
+ "risingwave_sqlparser",
  "rust_decimal",
  "self_cell",
  "serde",
  "serde_json",
  "sha1",
  "sha2",
+ "smallvec",
  "sql-json-path",
  "thiserror",
  "thiserror-ext",
+ "tonic 0.10.2",
  "tracing",
  "workspace-hack",
+ "zstd 0.13.0",
 ]
 
 [[package]]
@@ -10663,12 +10988,11 @@ dependencies = [
 
 [[package]]
 name = "risingwave_frontend"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "arc-swap",
  "arrow-schema 50.0.0",
- "arrow-udf-wasm",
  "assert_matches",
  "async-recursion",
  "async-trait",
@@ -10681,7 +11005,7 @@ dependencies = [
  "downcast-rs",
  "dyn-clone",
  "easy-ext",
- "educe 0.5.7",
+ "educe",
  "either",
  "enum-as-inner 0.6.0",
  "expect-test",
@@ -10727,7 +11051,6 @@ dependencies = [
  "risingwave_rpc_client",
  "risingwave_sqlparser",
  "risingwave_storage",
- "risingwave_udf",
  "risingwave_variables",
  "rw_futures_util",
  "serde",
@@ -10747,7 +11070,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_frontend_macro"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -10756,7 +11079,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_hummock_sdk"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "bytes",
  "easy-ext",
@@ -10767,13 +11090,15 @@ dependencies = [
  "risingwave_common",
  "risingwave_common_estimate_size",
  "risingwave_pb",
+ "serde",
+ "serde_bytes",
  "tracing",
  "workspace-hack",
 ]
 
 [[package]]
 name = "risingwave_hummock_test"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "async-trait",
  "bytes",
@@ -10806,7 +11131,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_hummock_trace"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "async-trait",
  "bincode 2.0.0-rc.3",
@@ -10831,14 +11156,25 @@ dependencies = [
 name = "risingwave_java_binding"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
+ "bytes",
+ "cfg-or-panic",
+ "foyer",
+ "futures",
  "jni",
+ "madsim-tokio",
  "prost 0.12.1",
  "risingwave_common",
  "risingwave_expr",
+ "risingwave_hummock_sdk",
  "risingwave_jni_core",
+ "risingwave_object_store",
  "risingwave_pb",
+ "risingwave_storage",
+ "rw_futures_util",
  "serde",
  "serde_json",
+ "tracing",
 ]
 
 [[package]]
@@ -10850,6 +11186,7 @@ dependencies = [
  "cfg-or-panic",
  "chrono",
  "expect-test",
+ "foyer",
  "fs-err",
  "futures",
  "itertools 0.12.1",
@@ -10860,9 +11197,7 @@ dependencies = [
  "risingwave_common",
  "risingwave_expr",
  "risingwave_hummock_sdk",
- "risingwave_object_store",
  "risingwave_pb",
- "risingwave_storage",
  "rw_futures_util",
  "serde",
  "serde_json",
@@ -10873,7 +11208,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_mem_table_spill_test"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "async-trait",
  "bytes",
@@ -10889,8 +11224,9 @@ dependencies = [
 
 [[package]]
 name = "risingwave_meta"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
+ "aes-siv",
  "anyhow",
  "arc-swap",
  "assert_matches",
@@ -10898,6 +11234,7 @@ dependencies = [
  "aws-config",
  "axum 0.7.4",
  "base64-url",
+ "bincode 1.3.3",
  "bytes",
  "chrono",
  "clap",
@@ -10962,7 +11299,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_meta_dashboard"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "axum 0.7.4",
@@ -10972,7 +11309,7 @@ dependencies = [
  "dircpy",
  "mime_guess",
  "npm_rs",
- "reqwest 0.12.2",
+ "reqwest 0.12.4",
  "rust-embed",
  "thiserror-ext",
  "tokio",
@@ -10983,7 +11320,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_meta_model_migration"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "async-std",
  "sea-orm-migration",
@@ -10992,7 +11329,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_meta_model_v2"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "prost 0.12.1",
  "risingwave_common",
@@ -11005,7 +11342,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_meta_node"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "clap",
@@ -11037,7 +11374,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_meta_service"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -11065,7 +11402,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_object_store"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "async-trait",
  "await-tree",
@@ -11101,7 +11438,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_pb"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "enum-as-inner 0.6.0",
  "fs-err",
@@ -11121,12 +11458,12 @@ dependencies = [
 
 [[package]]
 name = "risingwave_planner_test"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "expect-test",
  "itertools 0.12.1",
- "libtest-mimic 0.7.0",
+ "libtest-mimic",
  "madsim-tokio",
  "paste",
  "risingwave_expr_impl",
@@ -11143,7 +11480,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_regress_test"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "clap",
@@ -11157,7 +11494,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_rpc_client"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -11191,7 +11528,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_rt"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "await-tree",
  "console",
@@ -11272,42 +11609,35 @@ dependencies = [
 
 [[package]]
 name = "risingwave_sqlparser"
-version = "1.9.0-alpha"
-dependencies = [
- "itertools 0.12.1",
- "matches",
- "serde",
- "tracing",
- "tracing-subscriber",
- "workspace-hack",
-]
-
-[[package]]
-name = "risingwave_sqlparser_test_runner"
-version = "0.1.0"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "console",
- "futures",
+ "itertools 0.12.1",
+ "libtest-mimic",
  "madsim-tokio",
- "risingwave_sqlparser",
+ "matches",
  "serde",
  "serde_with",
  "serde_yaml",
+ "thiserror",
+ "tracing",
+ "tracing-subscriber",
  "walkdir",
+ "winnow 0.6.11",
  "workspace-hack",
 ]
 
 [[package]]
 name = "risingwave_sqlsmith"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "chrono",
  "clap",
  "expect-test",
  "itertools 0.12.1",
- "libtest-mimic 0.7.0",
+ "libtest-mimic",
  "madsim-tokio",
  "rand",
  "rand_chacha",
@@ -11328,7 +11658,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_state_cleaning_test"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "clap",
@@ -11348,14 +11678,15 @@ dependencies = [
 
 [[package]]
 name = "risingwave_storage"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "anyhow",
  "arc-swap",
  "async-trait",
  "auto_enums",
  "await-tree",
+ "bincode 1.3.3",
  "bytes",
  "criterion",
  "crossbeam",
@@ -11378,9 +11709,10 @@ dependencies = [
  "madsim-tokio",
  "madsim-tonic",
  "memcomparable",
+ "metrics-prometheus",
  "moka",
  "more-asserts",
- "nix 0.28.0",
+ "nix 0.29.0",
  "num-integer",
  "parking_lot 0.12.1",
  "procfs 0.16.0",
@@ -11399,6 +11731,8 @@ dependencies = [
  "risingwave_rpc_client",
  "risingwave_test_runner",
  "scopeguard",
+ "serde",
+ "serde_bytes",
  "sled",
  "spin 0.9.8",
  "sync-point",
@@ -11416,7 +11750,7 @@ dependencies = [
 
 [[package]]
 name = "risingwave_stream"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "anyhow",
  "assert_matches",
@@ -11429,7 +11763,7 @@ dependencies = [
  "cfg-if",
  "criterion",
  "delta_btree_map",
- "educe 0.5.7",
+ "educe",
  "either",
  "enum-as-inner 0.6.0",
  "expect-test",
@@ -11484,38 +11818,16 @@ dependencies = [
 
 [[package]]
 name = "risingwave_test_runner"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "fail",
  "sync-point",
  "workspace-hack",
 ]
 
-[[package]]
-name = "risingwave_udf"
-version = "0.1.0"
-dependencies = [
- "arrow-array 50.0.0",
- "arrow-flight",
- "arrow-schema 50.0.0",
- "arrow-select 50.0.0",
- "cfg-or-panic",
- "futures",
- "futures-util",
- "ginepro",
- "madsim-tokio",
- "madsim-tonic",
- "prometheus",
- "risingwave_common",
- "static_assertions",
- "thiserror",
- "thiserror-ext",
- "tracing",
-]
-
 [[package]]
 name = "risingwave_variables"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "chrono",
  "workspace-hack",
@@ -11577,27 +11889,27 @@ dependencies = [
 
 [[package]]
 name = "rquickjs"
-version = "0.5.1"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad7f63201fa6f2ff8173e4758ea552549d687d8f63003361a8b5c50f7c446ded"
+checksum = "9cbd33e0b668aea0ab238b9164523aca929096f9f40834700d71d91dd4888882"
 dependencies = [
  "rquickjs-core",
 ]
 
 [[package]]
 name = "rquickjs-core"
-version = "0.5.1"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cad00eeddc0f88af54ee202c8385fb214fe0423897c056a7df8369fb482e3695"
+checksum = "2e9129d69b7b8f7ee8ad1da5b12c7f4a8a8acd45f2e6dd9cb2ee1bc5a1f2fa3d"
 dependencies = [
  "rquickjs-sys",
 ]
 
 [[package]]
 name = "rquickjs-sys"
-version = "0.5.1"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "120dbbc3296de9b96de8890091635d46f3506cd38b4e8f21800c386c035d64fa"
+checksum = "bf6f2288d8e7fbb5130f62cf720451641e99d55f6fde9db86aa2914ecb553fd2"
 dependencies = [
  "cc",
 ]
@@ -11644,9 +11956,9 @@ dependencies = [
 
 [[package]]
 name = "rust-embed"
-version = "8.3.0"
+version = "8.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb78f46d0066053d16d4ca7b898e9343bc3530f71c61d5ad84cd404ada068745"
+checksum = "19549741604902eb99a7ed0ee177a0663ee1eda51a29f71401f166e47e77806a"
 dependencies = [
  "rust-embed-impl",
  "rust-embed-utils",
@@ -11655,9 +11967,9 @@ dependencies = [
 
 [[package]]
 name = "rust-embed-impl"
-version = "8.3.0"
+version = "8.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b91ac2a3c6c0520a3fb3dd89321177c3c692937c4eb21893378219da10c44fc8"
+checksum = "cb9f96e283ec64401f30d3df8ee2aaeb2561f34c824381efa24a35f79bf40ee4"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -11669,9 +11981,9 @@ dependencies = [
 
 [[package]]
 name = "rust-embed-utils"
-version = "8.3.0"
+version = "8.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f69089032567ffff4eada41c573fc43ff466c7db7c5688b2e7969584345581"
+checksum = "38c74a686185620830701348de757fd36bef4aa9680fd23c49fc539ddcc1af32"
 dependencies = [
  "mime_guess",
  "sha2",
@@ -11780,9 +12092,21 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.21.10"
+version = "0.20.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99"
+dependencies = [
+ "log",
+ "ring 0.16.20",
+ "sct",
+ "webpki",
+]
+
+[[package]]
+name = "rustls"
+version = "0.21.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba"
+checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4"
 dependencies = [
  "log",
  "ring 0.17.5",
@@ -11792,9 +12116,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.22.3"
+version = "0.22.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99008d7ad0bbbea527ec27bddbc0e432c5b87d8175178cee68d2eec9c4a1813c"
+checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432"
 dependencies = [
  "log",
  "ring 0.17.5",
@@ -11813,6 +12137,7 @@ dependencies = [
  "aws-lc-rs",
  "log",
  "once_cell",
+ "ring 0.17.5",
  "rustls-pki-types",
  "rustls-webpki 0.102.2",
  "subtle",
@@ -11865,9 +12190,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.5.0"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54"
+checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"
 
 [[package]]
 name = "rustls-tokio-stream"
@@ -11876,7 +12201,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "696a389edb0b54b9bb888c8318404d9a0c0b9091fb03ca3d9c64731511db03f6"
 dependencies = [
  "futures",
- "rustls 0.22.3",
+ "rustls 0.22.4",
  "socket2 0.5.6",
  "tokio",
 ]
@@ -12004,9 +12329,9 @@ dependencies = [
 
 [[package]]
 name = "sasl2-sys"
-version = "0.1.20+2.1.28"
+version = "0.1.22+2.1.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e645bd98535fc8fd251c43ba7c7c1f9be1e0369c99b6a5ea719052a773e655c"
+checksum = "05f2a7f7efd9fc98b3a9033272df10709f5ee3fa0eabbd61a527a3a1ed6bd3c6"
 dependencies = [
  "cc",
  "duct",
@@ -12021,6 +12346,15 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71"
 
+[[package]]
+name = "scc"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec96560eea317a9cc4e0bb1f6a2c93c09a19b8c4fc5cb3fcc0ec1c094cd783e2"
+dependencies = [
+ "sdd",
+]
+
 [[package]]
 name = "schannel"
 version = "0.1.22"
@@ -12036,7 +12370,7 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "772575a524feeb803e5b0fcbc6dd9f367e579488197c94c6e4023aad2305774d"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "cfg-if",
  "hashbrown 0.13.2",
 ]
@@ -12080,6 +12414,12 @@ dependencies = [
  "untrusted 0.7.1",
 ]
 
+[[package]]
+name = "sdd"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b84345e4c9bd703274a082fb80caaa99b7612be48dfaa1dd9266577ec412309d"
+
 [[package]]
 name = "sea-bae"
 version = "0.2.0"
@@ -12224,7 +12564,9 @@ checksum = "30d148608012d25222442d1ebbfafd1228dbc5221baf4ec35596494e27a2394e"
 dependencies = [
  "futures",
  "sea-query",
+ "sea-query-binder",
  "sea-schema-derive",
+ "sqlx",
 ]
 
 [[package]]
@@ -12257,6 +12599,18 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "sealed"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4a8caec23b7800fb97971a1c6ae365b6239aaeddfb934d6265f8505e795699d"
+dependencies = [
+ "heck 0.4.1",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.57",
+]
+
 [[package]]
 name = "sec1"
 version = "0.3.0"
@@ -12501,11 +12855,11 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.7.0"
+version = "3.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee80b0e361bbf88fd2f6e242ccd19cfda072cb0faa6ae694ecee08199938569a"
+checksum = "2c85f8e96d1d6857f13768fcbd895fcb06225510022a2774ed8b5150581847b0"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.22.0",
  "chrono",
  "hex",
  "indexmap 1.9.3",
@@ -12519,9 +12873,9 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.7.0"
+version = "3.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6561dc161a9224638a31d876ccdfefbc1df91d3f3a8342eddb35f055d48c7655"
+checksum = "c8b3a576c4eb2924262d5951a3b737ccaf16c931e39a2810c36f9a7e25575557"
 dependencies = [
  "darling 0.20.8",
  "proc-macro2",
@@ -12544,23 +12898,23 @@ dependencies = [
 
 [[package]]
 name = "serial_test"
-version = "3.0.0"
+version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "953ad9342b3aaca7cb43c45c097dd008d4907070394bd0751a0aa8817e5a018d"
+checksum = "adb86f9315df5df6a70eae0cc22395a44e544a0d8897586820770a35ede74449"
 dependencies = [
- "dashmap",
  "futures",
- "lazy_static",
  "log",
+ "once_cell",
  "parking_lot 0.12.1",
+ "scc",
  "serial_test_derive",
 ]
 
 [[package]]
 name = "serial_test_derive"
-version = "3.0.0"
+version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b93fb4adc70021ac1b47f7d45e8cc4169baaa7ea58483bc5b721d19a26202212"
+checksum = "a9bb72430492e9549b0c4596725c0f82729bff861c45aa8099c0a8e67fc3b721"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -13011,18 +13365,18 @@ dependencies = [
 
 [[package]]
 name = "sqllogictest"
-version = "0.20.1"
+version = "0.20.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e7c6a33098cd55e4fead1bd1f85c1d2064f02bafdb9fe004ca39fd94aee36e6"
+checksum = "f20de090b0fde4dcd53b330e7ad2772140304eb34f5e9a99c8963fc4c052f149"
 dependencies = [
  "async-trait",
- "educe 0.4.23",
+ "educe",
  "fs-err",
  "futures",
  "glob",
  "humantime",
- "itertools 0.11.0",
- "libtest-mimic 0.6.1",
+ "itertools 0.13.0",
+ "libtest-mimic",
  "md-5",
  "owo-colors",
  "regex",
@@ -13073,7 +13427,7 @@ version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "atoi",
  "bigdecimal 0.3.1",
  "byteorder",
@@ -13083,7 +13437,7 @@ dependencies = [
  "crossbeam-queue",
  "dotenvy",
  "either",
- "event-listener",
+ "event-listener 2.5.3",
  "futures-channel",
  "futures-core",
  "futures-intrusive",
@@ -13446,7 +13800,7 @@ version = "0.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "630c761c74ac8021490b78578cc2223aa4a568241e26505c27bf0e4fd4ad8ec2"
 dependencies = [
- "ahash 0.8.6",
+ "ahash 0.8.11",
  "anyhow",
  "dashmap",
  "once_cell",
@@ -13970,9 +14324,9 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.58"
+version = "1.0.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297"
+checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709"
 dependencies = [
  "thiserror-impl",
 ]
@@ -14001,9 +14355,9 @@ dependencies = [
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.58"
+version = "1.0.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
+checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -14051,6 +14405,37 @@ dependencies = [
  "ordered-float 2.10.0",
 ]
 
+[[package]]
+name = "tiberius"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc6e2bf3e4b5be181a2a2ceff4b9b12e2684010d436a6958bd564fbc8094d44d"
+dependencies = [
+ "async-trait",
+ "asynchronous-codec",
+ "bigdecimal 0.3.1",
+ "byteorder",
+ "bytes",
+ "chrono",
+ "connection-string",
+ "encoding",
+ "enumflags2",
+ "futures-util",
+ "num-traits",
+ "once_cell",
+ "pin-project-lite",
+ "pretty-hex",
+ "rust_decimal",
+ "rustls-native-certs 0.6.3",
+ "rustls-pemfile 1.0.4",
+ "thiserror",
+ "time",
+ "tokio-rustls 0.23.4",
+ "tokio-util",
+ "tracing",
+ "uuid",
+]
+
 [[package]]
 name = "tikv-jemalloc-ctl"
 version = "0.5.4"
@@ -14254,13 +14639,24 @@ dependencies = [
  "rand",
 ]
 
+[[package]]
+name = "tokio-rustls"
+version = "0.23.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
+dependencies = [
+ "rustls 0.20.9",
+ "tokio",
+ "webpki",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
 dependencies = [
- "rustls 0.21.10",
+ "rustls 0.21.11",
  "tokio",
 ]
 
@@ -14270,7 +14666,18 @@ version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f"
 dependencies = [
- "rustls 0.22.3",
+ "rustls 0.22.4",
+ "rustls-pki-types",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-rustls"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
+dependencies = [
+ "rustls 0.23.5",
  "rustls-pki-types",
  "tokio",
 ]
@@ -14305,6 +14712,7 @@ checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d"
 dependencies = [
  "bytes",
  "futures-core",
+ "futures-io",
  "futures-sink",
  "pin-project-lite",
  "tokio",
@@ -14357,17 +14765,6 @@ dependencies = [
  "winnow 0.5.15",
 ]
 
-[[package]]
-name = "toml_edit"
-version = "0.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338"
-dependencies = [
- "indexmap 2.0.0",
- "toml_datetime",
- "winnow 0.5.15",
-]
-
 [[package]]
 name = "toml_edit"
 version = "0.21.1"
@@ -14389,7 +14786,7 @@ dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "winnow 0.6.5",
+ "winnow 0.6.11",
 ]
 
 [[package]]
@@ -14440,7 +14837,7 @@ dependencies = [
  "percent-encoding",
  "pin-project",
  "prost 0.12.1",
- "rustls 0.21.10",
+ "rustls 0.21.11",
  "rustls-pemfile 1.0.4",
  "tokio",
  "tokio-rustls 0.24.1",
@@ -15175,9 +15572,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasi-common"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b53dfacdeacca15ee2a48a4aa0ec6a6d0da737676e465770c0585f79c04e638"
+checksum = "63255d85e10627b07325d7cf4e5fe5a40fa4ff183569a0a67931be26d50ede07"
 dependencies = [
  "anyhow",
  "bitflags 2.5.0",
@@ -15273,9 +15670,18 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
 
 [[package]]
 name = "wasm-encoder"
-version = "0.201.0"
+version = "0.202.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfd106365a7f5f7aa3c1916a98cbb3ad477f5ff96ddb130285a91c6e7429e67a"
+dependencies = [
+ "leb128",
+]
+
+[[package]]
+name = "wasm-encoder"
+version = "0.206.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9c7d2731df60006819b013f64ccc2019691deccf6e11a1804bc850cd6748f1a"
+checksum = "d759312e1137f199096d80a70be685899cd7d3d09c572836bb2e9b69b4dc3b1e"
 dependencies = [
  "leb128",
 ]
@@ -15308,9 +15714,9 @@ dependencies = [
 
 [[package]]
 name = "wasmparser"
-version = "0.201.0"
+version = "0.202.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84e5df6dba6c0d7fafc63a450f1738451ed7a0b52295d83e868218fa286bf708"
+checksum = "d6998515d3cf3f8b980ef7c11b29a9b1017d4cf86b99ae93b546992df9931413"
 dependencies = [
  "bitflags 2.5.0",
  "indexmap 2.0.0",
@@ -15319,9 +15725,9 @@ dependencies = [
 
 [[package]]
 name = "wasmprinter"
-version = "0.201.0"
+version = "0.202.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a67e66da702706ba08729a78e3c0079085f6bfcb1a62e4799e97bbf728c2c265"
+checksum = "ab1cc9508685eef9502e787f4d4123745f5651a1e29aec047645d3cac1e2da7a"
 dependencies = [
  "anyhow",
  "wasmparser",
@@ -15329,9 +15735,9 @@ dependencies = [
 
 [[package]]
 name = "wasmtime"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "516be5b58a8f75d39b01378516dcb0ff7b9bc39c7f1f10eec5b338d4916cf988"
+checksum = "5a5990663c28d81015ddbb02a068ac1bf396a4ea296eba7125b2dfc7c00cb52e"
 dependencies = [
  "addr2line",
  "anyhow",
@@ -15346,7 +15752,7 @@ dependencies = [
  "ittapi",
  "libc",
  "log",
- "object",
+ "object 0.33.0",
  "once_cell",
  "paste",
  "rayon",
@@ -15356,7 +15762,7 @@ dependencies = [
  "serde_derive",
  "serde_json",
  "target-lexicon",
- "wasm-encoder",
+ "wasm-encoder 0.202.0",
  "wasmparser",
  "wasmtime-cache",
  "wasmtime-component-macro",
@@ -15375,18 +15781,18 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-asm-macros"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8d22d88a92d69385f18143c946884bf6aaa9ec206ce54c85a2d320c1362b009"
+checksum = "625ee94c72004f3ea0228989c9506596e469517d7d0ed66f7300d1067bdf1ca9"
 dependencies = [
  "cfg-if",
 ]
 
 [[package]]
 name = "wasmtime-cache"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "068728a840223b56c964507550da671372e7e5c2f3a7856012b57482e3e979a7"
+checksum = "98534bf28de232299e83eab33984a7a6c40c69534d6bd0ea216150b63d41a83a"
 dependencies = [
  "anyhow",
  "base64 0.21.7",
@@ -15404,9 +15810,9 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-component-macro"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "631244bac89c57ebe7283209d86fe175ad5929328e75f61bf9141895cafbf52d"
+checksum = "64f84414a25ee3a624c8b77550f3fe7b5d8145bd3405ca58886ee6900abb6dc2"
 dependencies = [
  "anyhow",
  "proc-macro2",
@@ -15419,15 +15825,15 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-component-util"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82ad496ba0558f7602da5e9d4c201f35f7aefcca70f973ec916f3f0d0787ef74"
+checksum = "78580bdb4e04c7da3bf98088559ca1d29382668536e4d5c7f2f966d79c390307"
 
 [[package]]
 name = "wasmtime-cranelift"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "961ab5ee4b17e627001b18069ee89ef906edbbd3f84955515f6aad5ab6d82299"
+checksum = "b60df0ee08c6a536c765f69e9e8205273435b66d02dd401e938769a2622a6c1a"
 dependencies = [
  "anyhow",
  "cfg-if",
@@ -15439,36 +15845,19 @@ dependencies = [
  "cranelift-wasm",
  "gimli",
  "log",
- "object",
+ "object 0.33.0",
  "target-lexicon",
  "thiserror",
  "wasmparser",
- "wasmtime-cranelift-shared",
  "wasmtime-environ",
  "wasmtime-versioned-export-macros",
 ]
 
-[[package]]
-name = "wasmtime-cranelift-shared"
-version = "19.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc4db94596be14cd1f85844ce85470bf68acf235143098b9d9bf72b49e47b917"
-dependencies = [
- "anyhow",
- "cranelift-codegen",
- "cranelift-control",
- "cranelift-native",
- "gimli",
- "object",
- "target-lexicon",
- "wasmtime-environ",
-]
-
 [[package]]
 name = "wasmtime-environ"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "420b13858ef27dfd116f1fdb0513e9593a307a632ade2ea58334b639a3d8d24e"
+checksum = "64ffc1613db69ee47c96738861534f9a405e422a5aa00224fbf5d410b03fb445"
 dependencies = [
  "anyhow",
  "bincode 1.3.3",
@@ -15477,13 +15866,13 @@ dependencies = [
  "gimli",
  "indexmap 2.0.0",
  "log",
- "object",
+ "object 0.33.0",
  "rustc-demangle",
  "serde",
  "serde_derive",
  "target-lexicon",
  "thiserror",
- "wasm-encoder",
+ "wasm-encoder 0.202.0",
  "wasmparser",
  "wasmprinter",
  "wasmtime-component-util",
@@ -15492,9 +15881,9 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-fiber"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d37ff0e11a023019e34fe839c74a1c00880b989f4446176b6cc6da3b58e3ef2"
+checksum = "f043514a23792761c5765f8ba61a4aa7d67f260c0c37494caabceb41d8ae81de"
 dependencies = [
  "anyhow",
  "cc",
@@ -15507,11 +15896,11 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-jit-debug"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b849f19ad1d4a8133ff05b82c438144f17fb49b08e5f7995f8c1e25cf35f390"
+checksum = "9c0ca2ad8f5d2b37f507ef1c935687a690e84e9f325f5a2af9639440b43c1f0e"
 dependencies = [
- "object",
+ "object 0.33.0",
  "once_cell",
  "rustix 0.38.31",
  "wasmtime-versioned-export-macros",
@@ -15519,9 +15908,9 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59c48eb4223d6556ffbf3decb146d0da124f1fd043f41c98b705252cb6a5c186"
+checksum = "7a9f93a3289057b26dc75eb84d6e60d7694f7d169c7c09597495de6e016a13ff"
 dependencies = [
  "cfg-if",
  "libc",
@@ -15530,9 +15919,9 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-runtime"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fefac2cb5f5a6f365234a3584bf40bd2e45e7f6cd90a689d9b2afbb9881978f"
+checksum = "c6332a2b0af4224c3ea57c857ad39acd2780ccc2b0c99ba1baa01864d90d7c94"
 dependencies = [
  "anyhow",
  "cc",
@@ -15541,34 +15930,34 @@ dependencies = [
  "indexmap 2.0.0",
  "libc",
  "log",
- "mach",
+ "mach2",
  "memfd",
  "memoffset",
  "paste",
  "psm",
  "rustix 0.38.31",
  "sptr",
- "wasm-encoder",
+ "wasm-encoder 0.202.0",
  "wasmtime-asm-macros",
  "wasmtime-environ",
  "wasmtime-fiber",
  "wasmtime-jit-debug",
+ "wasmtime-slab",
  "wasmtime-versioned-export-macros",
- "wasmtime-wmemcheck",
  "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "wasmtime-slab"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52d7b97b92df126fdbe994a53d2215828ec5ed5087535e6d4703b1fbd299f0e3"
+checksum = "8b3655075824a374c536a2b2cc9283bb765fcdf3d58b58587862c48571ad81ef"
 
 [[package]]
 name = "wasmtime-types"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "509c88abb830819b259c49e2d4e4f22b555db066ba08ded0b76b071a2aa53ddf"
+checksum = "b98cf64a242b0b9257604181ca28b28a5fcaa4c9ea1d396f76d1d2d1c5b40eef"
 dependencies = [
  "cranelift-entity",
  "serde",
@@ -15579,9 +15968,9 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-versioned-export-macros"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1d81c092a61ca1667013e2eb08fed7c6c53e496dbbaa32d5685dc5152b0a772"
+checksum = "8561d9e2920db2a175213d557d71c2ac7695831ab472bbfafb9060cd1034684f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -15590,26 +15979,26 @@ dependencies = [
 
 [[package]]
 name = "wasmtime-winch"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0958907880e37a2d3974f5b3574c23bf70aaf1fc6c1f716625bb50dac776f1a"
+checksum = "a06b573d14ac846a0fb8c541d8fca6a64acf9a1d176176982472274ab1d2fa5d"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
  "gimli",
- "object",
+ "object 0.33.0",
  "target-lexicon",
  "wasmparser",
- "wasmtime-cranelift-shared",
+ "wasmtime-cranelift",
  "wasmtime-environ",
  "winch-codegen",
 ]
 
 [[package]]
 name = "wasmtime-wit-bindgen"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a593ddefd2f80617df6bea084b2e422d8969e924bc209642a794d57518f59587"
+checksum = "595bc7bb3b0ff4aa00fab718c323ea552c3034d77abc821a35112552f2ea487a"
 dependencies = [
  "anyhow",
  "heck 0.4.1",
@@ -15617,12 +16006,6 @@ dependencies = [
  "wit-parser",
 ]
 
-[[package]]
-name = "wasmtime-wmemcheck"
-version = "19.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b77212b6874bbc86d220bb1d28632d0c11c6afe996c3e1ddcf746b1a6b4919b9"
-
 [[package]]
 name = "wast"
 version = "35.0.2"
@@ -15634,24 +16017,24 @@ dependencies = [
 
 [[package]]
 name = "wast"
-version = "201.0.0"
+version = "206.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ef6e1ef34d7da3e2b374fd2b1a9c0227aff6cad596e1b24df9b58d0f6222faa"
+checksum = "68586953ee4960b1f5d84ebf26df3b628b17e6173bc088e0acfbce431469795a"
 dependencies = [
  "bumpalo",
  "leb128",
  "memchr",
  "unicode-width",
- "wasm-encoder",
+ "wasm-encoder 0.206.0",
 ]
 
 [[package]]
 name = "wat"
-version = "1.201.0"
+version = "1.206.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "453d5b37a45b98dee4f4cb68015fc73634d7883bbef1c65e6e9c78d454cf3f32"
+checksum = "da4c6f2606276c6e991aebf441b2fc92c517807393f039992a3e0ad873efe4ad"
 dependencies = [
- "wast 201.0.0",
+ "wast 206.0.0",
 ]
 
 [[package]]
@@ -15680,6 +16063,16 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "webpki"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53"
+dependencies = [
+ "ring 0.17.5",
+ "untrusted 0.9.0",
+]
+
 [[package]]
 name = "webpki-roots"
 version = "0.25.2"
@@ -15739,9 +16132,9 @@ checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8"
 
 [[package]]
 name = "wiggle"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f093d8afdb09efaf2ed1037468bd4614308a762d215b6cafd60a7712993a8ffa"
+checksum = "1b6552dda951239e219c329e5a768393664e8d120c5e0818487ac2633f173b1f"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -15754,9 +16147,9 @@ dependencies = [
 
 [[package]]
 name = "wiggle-generate"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47c7bccd5172ce8d853242f723e42c84b8c131b24fb07a1570f9045d99258616"
+checksum = "da64cb31e0bfe8b1d2d13956ef9fd5c77545756a1a6ef0e6cfd44e8f1f207aed"
 dependencies = [
  "anyhow",
  "heck 0.4.1",
@@ -15769,9 +16162,9 @@ dependencies = [
 
 [[package]]
 name = "wiggle-macro"
-version = "19.0.1"
+version = "20.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a69d087dee85991096fc0c6eaf4dcf4e17cd16a0594c33b8ab9e2d345234ef75"
+checksum = "900b2416ef2ff2903ded6cf55d4a941fed601bf56a8c4874856d7a77c1891994"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -15812,9 +16205,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "winch-codegen"
-version = "0.17.1"
+version = "0.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e72a6a7034793b874b85e428fd6d7b3ccccb98c326e33af3aa40cdf50d0c33da"
+checksum = "fb23450977f9d4a23c02439cf6899340b2d68887b19465c5682740d9cc37d52e"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -15823,6 +16216,7 @@ dependencies = [
  "smallvec",
  "target-lexicon",
  "wasmparser",
+ "wasmtime-cranelift",
  "wasmtime-environ",
 ]
 
@@ -16082,9 +16476,9 @@ dependencies = [
 
 [[package]]
 name = "winnow"
-version = "0.6.5"
+version = "0.6.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8"
+checksum = "56c52728401e1dc672a56e81e593e912aa54c78f40246869f78359a2bf24d29d"
 dependencies = [
  "memchr",
 ]
@@ -16099,6 +16493,16 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "winreg"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5"
+dependencies = [
+ "cfg-if",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "winx"
 version = "0.36.3"
@@ -16111,9 +16515,9 @@ dependencies = [
 
 [[package]]
 name = "wit-parser"
-version = "0.201.0"
+version = "0.202.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "196d3ecfc4b759a8573bf86a9b3f8996b304b3732e4c7de81655f875f6efdca6"
+checksum = "744237b488352f4f27bca05a10acb79474415951c450e52ebd0da784c1df2bcc"
 dependencies = [
  "anyhow",
  "id-arena",
@@ -16129,7 +16533,7 @@ dependencies = [
 
 [[package]]
 name = "with_options"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -16150,12 +16554,11 @@ dependencies = [
 
 [[package]]
 name = "workspace-config"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 dependencies = [
  "libz-sys",
  "log",
  "lzma-sys",
- "openssl-sys",
  "sasl2-sys",
  "tracing",
  "zstd-sys",
@@ -16163,7 +16566,7 @@ dependencies = [
 
 [[package]]
 name = "workspace-hack"
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 
 [[package]]
 name = "wyz"
@@ -16248,7 +16651,7 @@ dependencies = [
  "itertools 0.10.5",
  "log",
  "percent-encoding",
- "rustls 0.21.10",
+ "rustls 0.21.11",
  "rustls-pemfile 1.0.4",
  "seahash",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index ce1a66c94bdaa..091b3c113ed78 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,5 @@
 [workspace]
 members = [
-    "scripts/source/prepare_ci_pubsub",
     "src/batch",
     "src/bench",
     "src/cmd",
@@ -13,6 +12,7 @@ members = [
     "src/common/metrics",
     "src/compute",
     "src/connector",
+    "src/connector/codec",
     "src/connector/with_options",
     "src/ctl",
     "src/dml",
@@ -20,7 +20,6 @@ members = [
     "src/expr/core",
     "src/expr/impl",
     "src/expr/macro",
-    "src/expr/udf",
     "src/frontend",
     "src/frontend/macro",
     "src/frontend/planner_test",
@@ -39,7 +38,6 @@ members = [
     "src/risedevtool/config",
     "src/rpc_client",
     "src/sqlparser",
-    "src/sqlparser/test_runner",
     "src/storage",
     "src/storage/backup",
     "src/storage/compactor",
@@ -70,7 +68,7 @@ exclude = ["e2e_test/udf/wasm", "lints"]
 resolver = "2"
 
 [workspace.package]
-version = "1.9.0-alpha"
+version = "1.10.0-alpha"
 edition = "2021"
 homepage = "https://github.com/risingwavelabs/risingwave"
 keywords = ["sql", "database", "streaming"]
@@ -78,7 +76,7 @@ license = "Apache-2.0"
 repository = "https://github.com/risingwavelabs/risingwave"
 
 [workspace.dependencies]
-foyer = "0.6"
+foyer = { version = "0.9.4", features = ["nightly"] }
 auto_enums = { version = "0.8", features = ["futures03", "tokio1"] }
 await-tree = "0.2.1"
 aws-config = { version = "1", default-features = false, features = [
@@ -118,7 +116,7 @@ axum = "=0.7.4" # TODO: 0.7.5+ does not work with current toolchain
 etcd-client = { package = "madsim-etcd-client", version = "0.4" }
 futures-async-stream = "0.2.9"
 hytra = "0.1"
-rdkafka = { package = "madsim-rdkafka", version = "0.3.4", features = [
+rdkafka = { package = "madsim-rdkafka", version = "0.4.1", features = [
     "cmake-build",
 ] }
 hashbrown = { version = "0.14", features = ["ahash", "inline-more", "nightly"] }
@@ -139,10 +137,11 @@ arrow-flight = "50"
 arrow-select = "50"
 arrow-ord = "50"
 arrow-row = "50"
-arrow-udf-js = "0.1"
-arrow-udf-js-deno = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "23fe0dd" }
-arrow-udf-wasm = { version = "0.2.1", features = ["build"] }
-arrow-udf-python = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "6c32f71" }
+arrow-udf-js = "0.3.1"
+arrow-udf-js-deno = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "fa36365" }
+arrow-udf-wasm = { version = "0.2.2", features = ["build"] }
+arrow-udf-python = "0.2"
+arrow-udf-flight = "0.1"
 arrow-array-deltalake = { package = "arrow-array", version = "48.0.1" }
 arrow-buffer-deltalake = { package = "arrow-buffer", version = "48.0.1" }
 arrow-cast-deltalake = { package = "arrow-cast", version = "48.0.1" }
@@ -155,6 +154,7 @@ deltalake = { git = "https://github.com/risingwavelabs/delta-rs", rev = "5c2dccd
     "gcs",
 ] }
 itertools = "0.12.0"
+jsonbb = "0.1.4"
 lru = { git = "https://github.com/risingwavelabs/lru-rs.git", rev = "2682b85" }
 parquet = "50"
 thiserror-ext = "0.1.2"
@@ -177,6 +177,7 @@ sea-orm = { version = "0.12.14", features = [
     "sqlx-sqlite",
     "runtime-tokio-native-tls",
 ] }
+sqlx = "0.7"
 tokio-util = "0.7"
 tracing-opentelemetry = "0.22"
 rand = { version = "0.8", features = ["small_rng"] }
@@ -193,6 +194,7 @@ risingwave_compactor = { path = "./src/storage/compactor" }
 risingwave_compute = { path = "./src/compute" }
 risingwave_ctl = { path = "./src/ctl" }
 risingwave_connector = { path = "./src/connector" }
+risingwave_connector_codec = { path = "./src/connector/codec" }
 risingwave_dml = { path = "./src/dml" }
 risingwave_error = { path = "./src/error" }
 risingwave_expr = { path = "./src/expr/core" }
@@ -251,8 +253,11 @@ await_holding_lock = "warn"
 ptr_arg = "allow"
 # a little pedantic
 get_first = "allow"
-# https://github.com/rust-lang/rust-clippy/issues/12016
-blocks_in_conditions = "allow"
+new_without_default = "allow"
+# TODO: remove later https://github.com/rust-lang/rust-clippy/issues/12537
+# duplicated_attributes = "allow"
+# TODO: remove later https://github.com/rust-lang/rust-clippy/issues/12436
+mixed_attributes_style = "allow"
 
 [workspace.lints.rustdoc]
 private_intra_doc_links = "allow"
@@ -315,7 +320,7 @@ futures-timer = { git = "https://github.com/madsim-rs/futures-timer.git", rev =
 # patch: unlimit 4MB message size for grpc client
 etcd-client = { git = "https://github.com/risingwavelabs/etcd-client.git", rev = "4e84d40" }
 # todo(wcy-fdu): remove this patch fork after opendal release a new version to apply azure workload identity change.
-reqsign = { git = "https://github.com/wcy-fdu/reqsign.git", rev = "002ee2a" }
+reqsign = { git = "https://github.com/wcy-fdu/reqsign.git", rev = "c7dd668" }
 # patch to remove preserve_order from serde_json
 deno_core = { git = "https://github.com/bakjos/deno_core", rev = "9b241c6" }
 # patch to user reqwest 0.12.2
diff --git a/Makefile.toml b/Makefile.toml
index 504ff88a33d5a..1e618b1aed6ba 100644
--- a/Makefile.toml
+++ b/Makefile.toml
@@ -1,103 +1,121 @@
 extend = [
-  { path = "src/risedevtool/grafana.toml" },
-  { path = "src/risedevtool/prometheus.toml" },
-  { path = "src/risedevtool/minio.toml" },
-  { path = "src/risedevtool/etcd.toml" },
-  { path = "src/risedevtool/tempo.toml" },
-  { path = "src/risedevtool/kafka.toml" },
-  { path = "src/risedevtool/gcloud-pubsub.toml" },
-  { path = "src/risedevtool/redis.toml" },
-  { path = "src/risedevtool/connector.toml" },
-  { path = "src/risedevtool/risedev-components.toml" },
-  { path = "src/sqlparser/test_runner/sqlparser_test.toml" },
-  { path = "src/frontend/planner_test/planner_test.toml" },
-  { path = "src/tests/compaction_test/Makefile.toml" },
-  { path = "src/storage/backup/integration_tests/Makefile.toml" },
-  { path = "src/java_binding/make-java-binding.toml" },
-  { path = "src/stream/tests/integration_tests/integration_test.toml" },
-  { path = "e2e_test/source_inline/commands.toml" },
+    { path = "src/risedevtool/grafana.toml" },
+    { path = "src/risedevtool/prometheus.toml" },
+    { path = "src/risedevtool/minio.toml" },
+    { path = "src/risedevtool/etcd.toml" },
+    { path = "src/risedevtool/tempo.toml" },
+    { path = "src/risedevtool/gcloud-pubsub.toml" },
+    { path = "src/risedevtool/redis.toml" },
+    { path = "src/risedevtool/connector.toml" },
+    { path = "src/risedevtool/risedev-components.toml" },
+    { path = "src/sqlparser/sqlparser_test.toml" },
+    { path = "src/frontend/planner_test/planner_test.toml" },
+    { path = "src/tests/compaction_test/Makefile.toml" },
+    { path = "src/storage/backup/integration_tests/Makefile.toml" },
+    { path = "src/java_binding/make-java-binding.toml" },
+    { path = "src/stream/tests/integration_tests/integration_test.toml" },
+    { path = "e2e_test/source_inline/commands.toml" },
 ]
 
 env_files = ["./risedev-components.user.env"]
 
 env_scripts = [
-  '''
-#!@duckscript
-
-# only duckscript can modify env variables in cargo-make
-# duckscript doc: https://github.com/sagiegurari/duckscript/blob/master/docs/sdk.md
+    '''
+  #!@duckscript
+
+  # only duckscript can modify env variables in cargo-make
+  # duckscript doc: https://github.com/sagiegurari/duckscript/blob/master/docs/sdk.md
+
+  set_env ENABLE_TELEMETRY "false"
+  set_env RW_TELEMETRY_TYPE "test"
+
+  is_sanitizer_enabled = get_env ENABLE_SANITIZER
+  is_hdfs_backend = get_env ENABLE_HDFS
+  is_release = get_env ENABLE_RELEASE_PROFILE
+  is_not_release = not ${is_release}
+  is_dynamic_linking = get_env ENABLE_DYNAMIC_LINKING
+  is_hummock_trace = get_env ENABLE_HUMMOCK_TRACE
+  is_external_udf_enabled = get_env ENABLE_EXTERNAL_UDF
+  is_wasm_udf_enabled = get_env ENABLE_WASM_UDF
+  is_js_udf_enabled = get_env ENABLE_JS_UDF
+  is_deno_udf_enabled = get_env ENABLE_DENO_UDF
+  is_python_udf_enabled = get_env ENABLE_PYTHON_UDF
+
+  if ${is_sanitizer_enabled}
+    set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS "--timings -Zbuild-std --target ${CARGO_MAKE_RUST_TARGET_TRIPLE}"
+    set_env RISEDEV_BUILD_TARGET_DIR "${CARGO_MAKE_RUST_TARGET_TRIPLE}/"
+    set_env RISEDEV_RUSTFLAGS "-Ctarget-cpu=native --cfg tokio_unstable -Zsanitizer=thread"
+  else
+    set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS "--timings"
+    set_env RISEDEV_BUILD_TARGET_DIR ""
+  end
 
-set_env ENABLE_TELEMETRY "false"
+  if ${is_hdfs_backend}
+    set_env BUILD_HDFS_BACKEND_CMD "-p risingwave_object_store --features hdfs-backend"
+  else
+    set_env BUILD_HDFS_BACKEND_CMD ""
+  end
 
-is_sanitizer_enabled = get_env ENABLE_SANITIZER
-is_hdfs_backend = get_env ENABLE_HDFS
-is_release = get_env ENABLE_RELEASE_PROFILE
-is_not_release = not ${is_release}
-is_dynamic_linking = get_env ENABLE_DYNAMIC_LINKING
-is_hummock_trace = get_env ENABLE_HUMMOCK_TRACE
-is_deno_udf_enabled = get_env ENABLE_DENO_UDF
-is_python_udf_enabled = get_env ENABLE_PYTHON_UDF
+  if ${is_not_release} and ${is_dynamic_linking}
+    set_env RISINGWAVE_FEATURE_FLAGS "--features rw-dynamic-link --no-default-features"
+  else
+    set_env RISINGWAVE_FEATURE_FLAGS "--features rw-static-link"
+  end
 
-if ${is_sanitizer_enabled}
-  set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS "-Zbuild-std --target ${CARGO_MAKE_RUST_TARGET_TRIPLE}"
-  set_env RISEDEV_BUILD_TARGET_DIR "${CARGO_MAKE_RUST_TARGET_TRIPLE}/"
-  set_env RISEDEV_RUSTFLAGS "-Ctarget-cpu=native --cfg tokio_unstable -Zsanitizer=thread"
-else
-  set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS ""
-  set_env RISEDEV_BUILD_TARGET_DIR ""
-end
+  if ${is_external_udf_enabled}
+    flags = get_env RISINGWAVE_FEATURE_FLAGS
+    set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features external-udf"
+  end
 
-if ${is_hdfs_backend}
-  set_env BUILD_HDFS_BACKEND_CMD "-p risingwave_object_store --features hdfs-backend"
-else
-  set_env BUILD_HDFS_BACKEND_CMD ""
-end
+  if ${is_wasm_udf_enabled}
+    flags = get_env RISINGWAVE_FEATURE_FLAGS
+    set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features wasm-udf"
+  end
 
-if ${is_not_release} and ${is_dynamic_linking}
-  set_env RISINGWAVE_FEATURE_FLAGS "--features rw-dynamic-link --no-default-features"
-else
-  set_env RISINGWAVE_FEATURE_FLAGS "--features rw-static-link"
-end
+  if ${is_js_udf_enabled}
+    flags = get_env RISINGWAVE_FEATURE_FLAGS
+    set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features js-udf"
+  end
 
-if ${is_deno_udf_enabled}
-  flags = get_env RISINGWAVE_FEATURE_FLAGS
-  set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features embedded-deno-udf"
-end
+  if ${is_deno_udf_enabled}
+    flags = get_env RISINGWAVE_FEATURE_FLAGS
+    set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features deno-udf"
+  end
 
-if ${is_python_udf_enabled}
-  flags = get_env RISINGWAVE_FEATURE_FLAGS
-  set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features embedded-python-udf"
-end
+  if ${is_python_udf_enabled}
+    flags = get_env RISINGWAVE_FEATURE_FLAGS
+    set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features python-udf"
+  end
 
-if ${is_hummock_trace}
-  set_env BUILD_HUMMOCK_TRACE_CMD "-p risingwave_storage --features hm-trace"
-else
-  set_env BUILD_HUMMOCK_TRACE_CMD ""
-end
+  if ${is_hummock_trace}
+    set_env BUILD_HUMMOCK_TRACE_CMD "-p risingwave_storage --features hm-trace"
+  else
+    set_env BUILD_HUMMOCK_TRACE_CMD ""
+  end
 
-is_ci = get_env RISINGWAVE_CI
-is_not_ci = not ${is_ci}
+  is_ci = get_env RISINGWAVE_CI
+  is_not_ci = not ${is_ci}
 
-if ${is_not_ci}
-  query_log_path = get_env RW_QUERY_LOG_PATH
-  no_query_log_path = not ${query_log_path}
+  if ${is_not_ci}
+    query_log_path = get_env RW_QUERY_LOG_PATH
+    no_query_log_path = not ${query_log_path}
 
-  if ${no_query_log_path}
-    set_env RW_QUERY_LOG_PATH "${PREFIX_LOG}"
-  fi
+    if ${no_query_log_path}
+      set_env RW_QUERY_LOG_PATH "${PREFIX_LOG}"
+    fi
 
-  rust_log = get_env RUST_LOG
-  no_rust_log = not ${rust_log}
+    rust_log = get_env RUST_LOG
+    no_rust_log = not ${rust_log}
 
-  if ${no_rust_log}
-    set_env RUST_LOG "pgwire_query_log=info"
-  else
-    set_env RUST_LOG "pgwire_query_log=info,${rust_log}"
+    if ${no_rust_log}
+      set_env RUST_LOG "pgwire_query_log=info"
+    else
+      set_env RUST_LOG "pgwire_query_log=info,${rust_log}"
+    end
   end
-end
 
-set_env TMUX "tmux -L risedev"
-''',
+  set_env TMUX "tmux -L risedev"
+  ''',
 ]
 
 
@@ -131,8 +149,7 @@ rm -rf "${PREFIX_PROFILING}"
 [tasks.reset-rw]
 category = "RiseDev - Start/Stop"
 description = "Clean all data in the default database dev of the running RisingWave"
-dependencies = ["check-risedev-env-file"]
-env_files = ["${PREFIX_CONFIG}/risedev-env"]
+dependencies = ["check-and-load-risedev-env-file"]
 script = '''
 #!/usr/bin/env bash
 psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d dev -c "CREATE DATABASE risedev_tmp;"
@@ -306,7 +323,7 @@ description = "Codesign playground binary to support coredump"
 # codesign the binary before running.
 # https://developer.apple.com/forums/thread/694233?answerId=695943022#695943022
 condition = { env_set = [
-  "ENABLE_COREDUMP",
+    "ENABLE_COREDUMP",
 ], env = { "SYSTEM" = "darwin-arm64" } }
 script = '''
 #!/usr/bin/env bash
@@ -323,7 +340,7 @@ description = "Codesign all binaries to support coredump"
 # codesign the binary before running.
 # https://developer.apple.com/forums/thread/694233?answerId=695943022#695943022
 condition = { env_set = [
-  "ENABLE_COREDUMP",
+    "ENABLE_COREDUMP",
 ], env = { "SYSTEM" = "darwin-arm64" } }
 script = '''
 #!/usr/bin/env bash
@@ -358,9 +375,9 @@ category = "RiseDev - Build"
 description = "Copy RisngWave binaries to bin"
 condition = { env_set = ["ENABLE_BUILD_RUST"] }
 dependencies = [
-  "link-all-in-one-binaries",
-  "link-user-bin",
-  "codesign-binaries",
+    "link-all-in-one-binaries",
+    "link-user-bin",
+    "codesign-binaries",
 ]
 
 [tasks.b]
@@ -477,16 +494,15 @@ private = true
 category = "Misc"
 description = "Download all available components at once"
 dependencies = [
-  "download-maven",
-  "download-etcd",
-  "download-grafana",
-  "download-tempo",
-  "download-kafka",
-  "download-mcli",
-  "download-minio",
-  "download-prometheus",
-  "download-pubsub",
-  "download-redis",
+    "download-maven",
+    "download-etcd",
+    "download-grafana",
+    "download-tempo",
+    "download-mcli",
+    "download-minio",
+    "download-prometheus",
+    "download-pubsub",
+    "download-redis",
 ]
 
 [tasks.create-user-profiles-file]
@@ -494,7 +510,7 @@ private = true
 category = "RiseDev - Prepare"
 description = "Create user profiles file if not exists"
 condition = { files_not_exist = [
-  "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/risedev-profiles.user.yml",
+    "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/risedev-profiles.user.yml",
 ] }
 script = '''
 #!/usr/bin/env bash
@@ -521,35 +537,34 @@ EOF
 category = "RiseDev - Prepare"
 description = "Prepare dev cluster by downloading necessary tools and build required components"
 dependencies = [
-  "create-user-profiles-file",
-  "download-all",
-  "build-risingwave",
-  "build-connector-node",
-  "post-build-risingwave",
-  "prepare-config",
+    "create-user-profiles-file",
+    "download-all",
+    "build-risingwave",
+    "build-connector-node",
+    "post-build-risingwave",
+    "prepare-config",
 ]
 
 [tasks.pre-start-benchmark]
 category = "RiseDev - Prepare"
 description = "Download necessary tools to deploy a benchmark env"
 dependencies = [
-  "download-minio",
-  "download-mcli",
-  "download-etcd",
-  "download-grafana",
-  "download-prometheus",
-  "download-tempo",
-  "download-kafka",
-  "download-redis",
+    "download-minio",
+    "download-mcli",
+    "download-etcd",
+    "download-grafana",
+    "download-prometheus",
+    "download-tempo",
+    "download-redis",
 ]
 
 [tasks.pre-start-playground]
 category = "RiseDev - Prepare"
 description = "Preparation steps for playground"
 dependencies = [
-  "build-risingwave-playground",
-  "codesign-playground",
-  "build-connector-node",
+    "build-risingwave-playground",
+    "codesign-playground",
+    "build-connector-node",
 ]
 
 [tasks.check-risedev-env-file]
@@ -568,11 +583,16 @@ if [ ! -f "${RC_ENV_FILE}" ]; then
 fi
 '''
 
+[tasks.check-and-load-risedev-env-file]
+private = true
+category = "RiseDev - Prepare"
+dependencies = ["check-risedev-env-file"]
+env_files = ["${PREFIX_CONFIG}/risedev-env"]
+
 [tasks.psql-env]
 category = "RiseDev - Start/Stop"
 description = "Dump env configuration for psql"
-dependencies = ["check-risedev-env-file"]
-env_files = ["${PREFIX_CONFIG}/risedev-env"]
+dependencies = ["check-and-load-risedev-env-file"]
 script = '''
 #!/usr/bin/env bash
 cat <<EOF > "${PREFIX_CONFIG}/psql-env"
@@ -590,8 +610,7 @@ echo "   $(tput setaf 4)source ${PREFIX_CONFIG}/psql-env$(tput sgr0)"
 [tasks.psql]
 category = "RiseDev - Start/Stop"
 description = "Run local psql client with default connection parameters. You can pass extra arguments to psql."
-dependencies = ["check-risedev-env-file"]
-env_files = ["${PREFIX_CONFIG}/risedev-env"]
+dependencies = ["check-and-load-risedev-env-file"]
 script = '''
 #!/usr/bin/env bash
 psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d dev "$@"
@@ -600,8 +619,7 @@ psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root
 [tasks.ctl]
 category = "RiseDev - Start/Stop"
 description = "Start RiseCtl"
-dependencies = ["check-risedev-env-file"]
-env_files = ["${PREFIX_CONFIG}/risedev-env"]
+dependencies = ["check-and-load-risedev-env-file"]
 script = '''
 #!/usr/bin/env bash
 cargo run -p risingwave_cmd_all --profile "${RISINGWAVE_BUILD_PROFILE}" -- ctl "$@"
@@ -687,54 +705,21 @@ script = '''
 
 set -euo pipefail
 
-wait_kafka_exit() {
-  # Follow kafka-server-stop.sh
-  while [[ -n "$(ps ax | grep ' kafka\.Kafka ' | grep java | grep -v grep | awk '{print $1}')" ]]; do
-    echo "Waiting for kafka to exit"
-    sleep 1
-  done
-}
-
-wait_zookeeper_exit() {
-  # Follow zookeeper-server-stop.sh
-  while [[ -n "$(ps ax | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}')" ]]; do
-    echo "Waiting for zookeeper to exit"
-    sleep 1
-  done
-}
-
-kill_kafka() {
-  ${PREFIX_BIN}/kafka/bin/kafka-server-stop.sh
-  wait_kafka_exit
-}
-
-kill_zookeeper() {
-  ${PREFIX_BIN}/kafka/bin/zookeeper-server-stop.sh
-  wait_zookeeper_exit
-}
-
 if ! ${TMUX} ls &>/dev/null ; then
   echo "No risedev cluster to kill. Exiting..."
   exit 0
 fi
 
-# Kill other components
+# Kill other components with Ctrl+C/Ctrl+D
 ${TMUX} list-windows -F "#{window_name} #{pane_id}" \
-| grep -v 'kafka' \
-| grep -v 'zookeeper' \
 | awk '{ print $2 }' \
 | xargs -I {} ${TMUX} send-keys -t {} C-c C-d
 
-if [[ -n $(${TMUX} list-windows | grep kafka) ]];
-then
-  echo "kill kafka"
-  kill_kafka || true
-
-  echo "kill zookeeper"
-  kill_zookeeper || true
-
-  # Kill their ${TMUX} sessions
-  ${TMUX} list-windows -t risedev -F "#{pane_id}" | xargs -I {} ${TMUX} send-keys -t {} C-c C-d
+# Stop docker components
+containers=$(docker ps -a -q -f name=risedev- 2>/dev/null) || true
+if [[ -n ${containers} ]]; then
+  echo "Stopping docker components..."
+  docker stop ${containers}
 fi
 
 ${TMUX} kill-server
@@ -756,16 +741,16 @@ dependencies = ["k", "clean-data"]
 private = true
 category = "RiseDev - Check"
 install_crate = { min_version = "0.9.51", crate_name = "cargo-nextest", binary = "cargo", test_arg = [
-  "nextest",
-  "--help",
+    "nextest",
+    "--help",
 ], install_command = "binstall" }
 
 [tasks.install-llvm-cov]
 private = true
 category = "RiseDev - Check"
 install_crate = { min_version = "0.5.17", crate_name = "cargo-llvm-cov", binary = "cargo", test_arg = [
-  "llvm-cov",
-  "--help",
+    "llvm-cov",
+    "--help",
 ], install_command = "binstall" }
 
 [tasks.install-tools]
@@ -846,11 +831,11 @@ description = "🌟 Run unit tests"
 category = "RiseDev - Build"
 dependencies = ["prepare"]
 condition = { env_set = [
-  "ENABLE_BUILD_RW_CONNECTOR",
+    "ENABLE_BUILD_RW_CONNECTOR",
 ], files_modified = { input = [
-  "./java/connector-node/**/*",
+    "./java/connector-node/**/*",
 ], output = [
-  "./java/connector-node/assembly/target/**/*",
+    "./java/connector-node/assembly/target/**/*",
 ] } }
 description = "Build RisingWave Connector from source"
 script = '''
@@ -1070,8 +1055,8 @@ private = true
 category = "RiseDev - Check"
 description = "Run cargo hakari check and attempt to fix"
 install_crate = { min_version = "0.9.24", crate_name = "cargo-hakari", binary = "cargo", test_arg = [
-  "hakari",
-  "--help",
+    "hakari",
+    "--help",
 ], install_command = "binstall" }
 script = """
 #!/usr/bin/env bash
@@ -1088,8 +1073,8 @@ private = true
 category = "RiseDev - Check"
 description = "Run cargo sort check and attempt to fix"
 install_crate = { min_version = "1.0.9", crate_name = "cargo-sort", binary = "cargo", test_arg = [
-  "sort",
-  "--help",
+    "sort",
+    "--help",
 ], install_command = "binstall" }
 script = """
 #!/usr/bin/env bash
@@ -1151,7 +1136,7 @@ private = true
 category = "RiseDev - Check"
 description = "Run cargo typos-cli check"
 install_crate = { min_version = "1.20.4", crate_name = "typos-cli", binary = "typos", test_arg = [
-  "--help",
+    "--help",
 ], install_command = "binstall" }
 script = """
 #!/usr/bin/env bash
@@ -1167,8 +1152,8 @@ category = "RiseDev - Check"
 description = "Check unused dependencies"
 env = { RUSTFLAGS = "--cfg tokio_unstable" }
 install_crate = { min_version = "0.1.35", crate_name = "cargo-udeps", binary = "cargo", test_arg = [
-  "udeps",
-  "--help",
+    "udeps",
+    "--help",
 ], install_command = "binstall" }
 script = """
 #!/usr/bin/env bash
@@ -1195,13 +1180,13 @@ scripts/check/check-trailing-spaces.sh --fix
 [tasks.check-fast]
 category = "RiseDev - Check"
 dependencies = [
-  "warn-on-missing-tools",
-  # Disable hakari until we make sure it's useful
-  # "check-hakari",
-  "check-dep-sort",
-  "check-fmt",
-  "check-trailing-spaces",
-  "check-typos",
+    "warn-on-missing-tools",
+    # Disable hakari until we make sure it's useful
+    # "check-hakari",
+    "check-dep-sort",
+    "check-fmt",
+    "check-trailing-spaces",
+    "check-typos",
 ]
 description = "Perform part of the pre-CI checks that are fast to run"
 
@@ -1222,10 +1207,10 @@ alias = "check"
 [tasks.check-fix]
 category = "RiseDev - Check"
 dependencies = [
-  "warn-on-missing-tools",
-  "check-fast",
-  "check-clippy-fix",
-  "check-java-fix",
+    "warn-on-missing-tools",
+    "check-fast",
+    "check-clippy-fix",
+    "check-java-fix",
 ]
 script = """
 #!/usr/bin/env bash
@@ -1308,17 +1293,29 @@ echo "All processes has exited."
 """
 
 [tasks.slt]
-env = { SLT_HOST = "${RISEDEV_RW_FRONTEND_LISTEN_ADDRESS}", SLT_PORT = "${RISEDEV_RW_FRONTEND_PORT}", SLT_DB = "dev" }
 category = "RiseDev - Test - SQLLogicTest"
 install_crate = { version = "0.20.1", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
-  "--help",
+    "--help",
 ], install_command = "binstall" }
-dependencies = ["check-risedev-env-file"]
-env_files = ["${PREFIX_CONFIG}/risedev-env"]
+dependencies = ["check-and-load-risedev-env-file"]
 command = "sqllogictest"
 args = ["${@}"]
 description = "🌟 Run SQLLogicTest"
 
+[tasks.slt.env]
+SLT_HOST = "${RISEDEV_RW_FRONTEND_LISTEN_ADDRESS}"
+SLT_PORT = "${RISEDEV_RW_FRONTEND_PORT}"
+SLT_DB = "dev"
+PATH = "${PWD}/e2e_test/commands:${PATH}"
+RUST_BACKTRACE = "0" # slt backtrace is useless
+
+[tasks.slt-clean]
+category = "RiseDev - Test - SQLLogicTest"
+dependencies = ["clean-kafka", "reset-rw"]
+description = "Run SQLLogicTest with a clean environment"
+run_task = "slt"
+args = ["${@}"]
+
 [tasks.slt-streaming]
 category = "RiseDev - Test - SQLLogicTest"
 extend = "slt"
@@ -1420,7 +1417,7 @@ UPDATE_EXPECT=1 cargo test -p risingwave_connector tests::test_with_options_yaml
 [tasks.backwards-compat-test]
 category = "RiseDev - Test - Backwards Compatibility Test"
 description = "Run backwards compatibility test"
-script = "./backwards-compat-tests/scripts/run_local.sh"
+script = "./e2e_test/backwards-compat-tests/scripts/run_local.sh"
 
 # For debugging.
 # To show the env for a specific task, use `run_task = "show-env"` for that task.
@@ -1433,3 +1430,15 @@ script = """
 vars = dump_variables
 echo ${vars}
 """
+
+[tasks.show-risedev-env]
+description = "Show risedev-env environment variables"
+dependencies = ["check-risedev-env-file"]
+script = '''
+#!/usr/bin/env bash
+set -euo pipefail
+
+cat ${PREFIX_CONFIG}/risedev-env
+echo "Hint: To load the environment variables into the shell, you may run:"
+echo "$(tput setaf 4)\set -a; source ${PREFIX_CONFIG}/risedev-env; set +a$(tput sgr0)"
+'''
diff --git a/README.md b/README.md
index e59af1a0395b4..156b53060313d 100644
--- a/README.md
+++ b/README.md
@@ -57,7 +57,8 @@
 RisingWave is a Postgres-compatible streaming database engineered to provide the <i><b>simplest</b></i> and <i><b>most cost-efficient</b></i> approach for <b>processing</b>, <b>analyzing</b>, and <b>managing</b> real-time event streaming data.
 
 
-![RisingWave](https://github.com/risingwavelabs/risingwave-docs/blob/main/docs/images/new_archi_grey.png)
+![RisingWave](https://github.com/risingwavelabs/risingwave/assets/41638002/10c44404-f78b-43ce-bbd9-3646690acc59)
+
 
 ## Try it out in 60 seconds
 
@@ -70,6 +71,9 @@ Then follow the prompts to start and connect to RisingWave.
 
 To learn about other installation options, such as using a Docker image, see [Quick Start](https://docs.risingwave.com/docs/current/get-started/).
 
+> Please note: RisingWave uses [Scarf](https://scarf.sh/) to collect anonymized installation analytics. These analytics help support us understand and improve the distribution of our package.
+> The privacy policy of Scarf is available at [https://about.scarf.sh/privacy-policy](https://about.scarf.sh/privacy-policy).
+
 ## Production deployments
 
 [**RisingWave Cloud**](https://cloud.risingwave.com) offers the easiest way to run RisingWave in production, with a _forever-free_ developer tier.
diff --git a/ci/Dockerfile b/ci/Dockerfile
index cffa1a026be3e..1e98d02fa522d 100644
--- a/ci/Dockerfile
+++ b/ci/Dockerfile
@@ -11,8 +11,8 @@ ENV LANG en_US.utf8
 # Use AWS ubuntu mirror
 RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://us-east-2.ec2.archive.ubuntu.com/ubuntu/|g' /etc/apt/sources.list
 RUN apt-get update -yy && \
-    DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl parallel python3 python3-pip python3-venv software-properties-common \
-    openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld postgresql-client kcat netcat-openbsd mysql-client \
+    DEBIAN_FRONTEND=noninteractive apt-get -y install sudo make build-essential cmake protobuf-compiler curl parallel python3 python3-pip python3-venv software-properties-common \
+    openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config bash openjdk-17-jdk wget unzip git tmux lld postgresql-client kcat netcat-openbsd mysql-client \
     maven zstd libzstd-dev locales \
     python3.12 python3.12-dev \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/
@@ -70,13 +70,13 @@ ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \
     cargo-make@0.37.9 \
-    sqllogictest-bin@0.20.1 \
+    sqllogictest-bin@0.20.4 \
     sccache@0.7.4 \
     && cargo cache -a \
     && rm -rf "/root/.cargo/registry/index" \
     && rm -rf "/root/.cargo/registry/cache" \
     && rm -rf "/root/.cargo/git/db"
-RUN cargo install cargo-dylint@2.6.0 dylint-link@2.6.0
+RUN cargo install cargo-dylint@3.1.0 dylint-link@3.1.0
 RUN cargo uninstall cargo-binstall cargo-cache
 
 # install risedev
diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh
index 1ec12359d896c..5569cf6af3fdb 100755
--- a/ci/build-ci-image.sh
+++ b/ci/build-ci-image.sh
@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20240414_x
+export BUILD_ENV_VERSION=v20240606
 
 export BUILD_TAG="public.ecr.aws/w1p7b4n3/rw-build-env:${BUILD_ENV_VERSION}"
 
diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml
index c754dcc174ed1..b2a885a4ba2e6 100644
--- a/ci/docker-compose.yml
+++ b/ci/docker-compose.yml
@@ -1,4 +1,3 @@
-version: "3.9"
 services:
   db:
     image: postgres:15-alpine
@@ -61,7 +60,8 @@ services:
       - "29092:29092"
       - "9092:9092"
       - "9644:9644"
-      - "8081:8081"
+      # Don't use Redpanda's schema registry, use the separated service instead
+      # - "8081:8081"
     environment: {}
     container_name: message_queue
     healthcheck:
@@ -71,7 +71,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240414_x
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240606
     depends_on:
       - mysql
       - db
@@ -84,11 +84,12 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240414_x
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240606
     depends_on:
       - mysql
       - db
       - message_queue
+      - schemaregistry
       - elasticsearch
       - clickhouse-server
       - redis-server
@@ -98,17 +99,18 @@ services:
       - doris-server
       - starrocks-fe-server
       - starrocks-be-server
+      - sqlserver-server
     volumes:
       - ..:/risingwave
 
 
   rw-build-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240414_x
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240606
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240414_x
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240606
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -119,7 +121,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240414_x
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240606
     depends_on:
       db:
         condition: service_healthy
@@ -202,6 +204,16 @@ services:
       timeout: 5s
       retries: 30
 
+  sqlserver-server:
+    container_name: sqlserver-server
+    image: mcr.microsoft.com/mssql/server:2022-latest
+    hostname: sqlserver-server
+    ports:
+      - 1433:1433
+    environment:
+      ACCEPT_EULA: 'Y'
+      SA_PASSWORD: 'SomeTestOnly@SA'
+
   starrocks-fe-server:
     container_name: starrocks-fe-server
     image: starrocks/fe-ubuntu:3.1.7
@@ -238,44 +250,20 @@ services:
 # # protobuf/avro schema registry. Should be removed after the support.
 # # Related tracking issue:
 # # https://github.com/redpanda-data/redpanda/issues/1878
-  zookeeper:
-    container_name: zookeeper
-    image: confluentinc/cp-zookeeper:latest
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
-    ports:
-      - "2181:2181"
-
   schemaregistry:
     container_name: schemaregistry
     hostname: schemaregistry
     image: confluentinc/cp-schema-registry:latest
     depends_on:
-      - kafka
+      - message_queue
     ports:
       - "8082:8082"
     environment:
       SCHEMA_REGISTRY_HOST_NAME: schemaregistry
-      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
       SCHEMA_REGISTRY_LISTENERS: http://schemaregistry:8082
-      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9093,PLAINTEXT_INTERNAL://localhost:29093
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: message_queue:29092
       SCHEMA_REGISTRY_DEBUG: 'true'
 
-  kafka:
-    container_name: kafka
-    image: confluentinc/cp-kafka:latest
-    ports:
-      - "29093:29093"
-    depends_on:
-      - zookeeper
-    environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,PLAINTEXT_INTERNAL://localhost:29093
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-
   pulsar-server:
     container_name: pulsar-server
     image: apachepulsar/pulsar:latest
diff --git a/ci/plugins/docker-compose-logs/hooks/post-command b/ci/plugins/docker-compose-logs/hooks/post-command
index 268b27c19cf0f..ff0bf941026bd 100755
--- a/ci/plugins/docker-compose-logs/hooks/post-command
+++ b/ci/plugins/docker-compose-logs/hooks/post-command
@@ -7,7 +7,6 @@ source env_vars.sh
 COMPOSE_FILE_PATH="integration_tests/${INTEGRATION_TEST_CASE}/docker-compose.yml"
 
 if [ $BUILDKITE_COMMAND_EXIT_STATUS -ne 0 ]; then
-
   if docker-compose -f "$COMPOSE_FILE_PATH" ps | grep "risingwave-standalone" > /dev/null; then
       echo "--- upload risingwave standalone docker compose logs"
       docker-compose -f "$COMPOSE_FILE_PATH" logs risingwave-standalone > risingwave-standalone.log
@@ -19,4 +18,7 @@ if [ $BUILDKITE_COMMAND_EXIT_STATUS -ne 0 ]; then
       docker-compose -f "$COMPOSE_FILE_PATH" logs meta-node-0 > risingwave-meta.log
   fi
     buildkite-agent artifact upload "./risingwave-*.log"
-fi
\ No newline at end of file
+fi
+
+echo "--- Docker clean"
+python3 integration_tests/scripts/clean_demos.py --case "${INTEGRATION_TEST_CASE}"
\ No newline at end of file
diff --git a/ci/risedev-components.ci.benchmark.env b/ci/risedev-components.ci.benchmark.env
index 67171fe10bc28..5bb6c4abddb56 100644
--- a/ci/risedev-components.ci.benchmark.env
+++ b/ci/risedev-components.ci.benchmark.env
@@ -1,7 +1,7 @@
 RISEDEV_CONFIGURED=true
 ENABLE_MINIO=true
 ENABLE_ETCD=true
-ENABLE_KAFKA=true
 ENABLE_BUILD_RUST=true
 ENABLE_RELEASE_PROFILE=true
 ENABLE_PROMETHEUS_GRAFANA=true
+RW_TELEMETRY_TYPE=test
diff --git a/ci/risedev-components.ci.env b/ci/risedev-components.ci.env
index 7157083eb871a..0430207739145 100644
--- a/ci/risedev-components.ci.env
+++ b/ci/risedev-components.ci.env
@@ -1,3 +1,4 @@
 RISEDEV_CONFIGURED=true
 ENABLE_MINIO=true
 ENABLE_ETCD=true
+RW_TELEMETRY_TYPE=test
diff --git a/ci/risedev-components.ci.source.env b/ci/risedev-components.ci.source.env
index 255b873eab94d..88d76b3df8e98 100644
--- a/ci/risedev-components.ci.source.env
+++ b/ci/risedev-components.ci.source.env
@@ -1,5 +1,5 @@
 RISEDEV_CONFIGURED=true
 ENABLE_MINIO=true
 ENABLE_ETCD=true
-ENABLE_KAFKA=true
 ENABLE_PUBSUB=true
+RW_TELEMETRY_TYPE=test
diff --git a/ci/rust-toolchain b/ci/rust-toolchain
index b1f2df70a8d0f..92ef4a5ff27be 100644
--- a/ci/rust-toolchain
+++ b/ci/rust-toolchain
@@ -4,4 +4,4 @@
 # 3. (optional) **follow the instructions in lints/README.md** to update the toolchain and dependencies for lints
 
 [toolchain]
-channel = "nightly-2023-12-26"
+channel = "nightly-2024-03-12"
diff --git a/ci/scripts/backwards-compat-test.sh b/ci/scripts/backwards-compat-test.sh
index d539748a23fd9..90a01f57a51cc 100755
--- a/ci/scripts/backwards-compat-test.sh
+++ b/ci/scripts/backwards-compat-test.sh
@@ -32,15 +32,17 @@ else
     exit 1
 fi
 
-source backwards-compat-tests/scripts/utils.sh
+source e2e_test/backwards-compat-tests/scripts/utils.sh
 
 ################################### Main
 
 configure_rw() {
 VERSION="$1"
+ENABLE_BUILD="$2"
 
 echo "--- Setting up cluster config"
-cat <<EOF > risedev-profiles.user.yml
+  if version_le "$VERSION" "1.8.9"; then
+    cat <<EOF > risedev-profiles.user.yml
 full-without-monitoring:
   steps:
     - use: minio
@@ -49,19 +51,31 @@ full-without-monitoring:
     - use: compute-node
     - use: frontend
     - use: compactor
-    - use: zookeeper
-    - use: kafka
 EOF
+  else
+     # For versions >= 1.9.0, the default config will default to sql backend,
+     # breaking backwards compat, so we must specify meta-backend: etcd
+     cat <<EOF > risedev-profiles.user.yml
+full-without-monitoring:
+ steps:
+   - use: minio
+   - use: etcd
+   - use: meta-node
+     meta-backend: etcd
+   - use: compute-node
+   - use: frontend
+   - use: compactor
+EOF
+  fi
 
 cat <<EOF > risedev-components.user.env
 RISEDEV_CONFIGURED=true
 
 ENABLE_MINIO=true
 ENABLE_ETCD=true
-ENABLE_KAFKA=true
 
-# Fetch risingwave binary from release.
-ENABLE_BUILD_RUST=false
+# Whether to build or directly fetch binary from release.
+ENABLE_BUILD_RUST=$ENABLE_BUILD
 
 # Use target/debug for simplicity.
 ENABLE_RELEASE_PROFILE=false
@@ -73,36 +87,6 @@ if version_le "${VERSION:-}" "1.8.0" ; then
 fi
 }
 
-configure_rw_build() {
-echo "--- Setting up cluster config"
-cat <<EOF > risedev-profiles.user.yml
-full-without-monitoring:
-  steps:
-    - use: minio
-    - use: etcd
-    - use: meta-node
-    - use: compute-node
-    - use: frontend
-    - use: compactor
-    - use: zookeeper
-    - use: kafka
-EOF
-
-cat <<EOF > risedev-components.user.env
-RISEDEV_CONFIGURED=true
-
-ENABLE_MINIO=true
-ENABLE_ETCD=true
-ENABLE_KAFKA=true
-
-# Make sure that it builds
-ENABLE_BUILD_RUST=true
-
-# Use target/debug for simplicity.
-ENABLE_RELEASE_PROFILE=false
-EOF
-}
-
 setup_old_cluster() {
   echo "--- Build risedev for $OLD_VERSION, it may not be backwards compatible"
   git config --global --add safe.directory /risingwave
@@ -115,7 +99,7 @@ setup_old_cluster() {
   if [[ "$?" -ne 0 ]]; then
     set -e
     echo "Failed to download ${OLD_VERSION} from github releases, build from source later during \`risedev d\`"
-    configure_rw_build
+    configure_rw "$OLD_VERSION" true
   else
     set -e
     tar -xvf risingwave-v"${OLD_VERSION}"-x86_64-unknown-linux.tar.gz
@@ -123,13 +107,13 @@ setup_old_cluster() {
 
     echo "--- Start cluster on tag $OLD_VERSION"
     git config --global --add safe.directory /risingwave
-    configure_rw "$OLD_VERSION"
+    configure_rw "$OLD_VERSION" false
   fi
 }
 
 setup_new_cluster() {
   echo "--- Setup Risingwave @ $RW_COMMIT"
-  git checkout -
+  git checkout "$RW_COMMIT"
   download_and_prepare_rw "$profile" common
   # Make sure we always start w/o old config
   rm -r .risingwave/config
@@ -148,8 +132,8 @@ main() {
   # Assume we use the latest version, so we just set to some large number.
   # The current $NEW_VERSION as of this change is 1.7.0, so we can't use that.
   # See: https://github.com/risingwavelabs/risingwave/pull/15448
-  configure_rw "99.99.99"
+  configure_rw "99.99.99" false
   validate_new_cluster "$NEW_VERSION"
 }
 
-main
\ No newline at end of file
+main
diff --git a/ci/scripts/build-other.sh b/ci/scripts/build-other.sh
index 2311e5164fe74..65c50462f97a0 100755
--- a/ci/scripts/build-other.sh
+++ b/ci/scripts/build-other.sh
@@ -16,9 +16,13 @@ cd java
 mvn -B package -Dmaven.test.skip=true
 mvn -B install -Dmaven.test.skip=true --pl java-binding-integration-test --am
 mvn dependency:copy-dependencies --no-transfer-progress --pl java-binding-integration-test
-mvn -B test --pl udf
 cd ..
 
+echo "--- Build Java UDF"
+cd e2e_test/udf/java
+mvn -B package
+cd ../../..
+
 echo "--- Build rust binary for java binding integration test"
 cargo build -p risingwave_java_binding --bin data-chunk-payload-generator --bin data-chunk-payload-convert-generator
 
@@ -30,9 +34,9 @@ tar --zstd -cf java-binding-integration-test.tar.zst bin java/java-binding-integ
 
 echo "--- Upload Java artifacts"
 cp java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz ./risingwave-connector.tar.gz
-cp java/udf-example/target/risingwave-udf-example.jar ./risingwave-udf-example.jar
+cp e2e_test/udf/java/target/risingwave-udf-example.jar ./udf.jar
 cp e2e_test/udf/wasm/target/wasm32-wasi/release/udf.wasm udf.wasm
 buildkite-agent artifact upload ./risingwave-connector.tar.gz
-buildkite-agent artifact upload ./risingwave-udf-example.jar
 buildkite-agent artifact upload ./java-binding-integration-test.tar.zst
+buildkite-agent artifact upload ./udf.jar
 buildkite-agent artifact upload ./udf.wasm
diff --git a/ci/scripts/build-simulation.sh b/ci/scripts/build-simulation.sh
index c2af5b0f3b7bc..7d8f09c5ca923 100755
--- a/ci/scripts/build-simulation.sh
+++ b/ci/scripts/build-simulation.sh
@@ -9,7 +9,7 @@ echo "--- Generate RiseDev CI config"
 cp ci/risedev-components.ci.env risedev-components.user.env
 
 echo "--- Build deterministic simulation e2e test runner"
-risedev sslt-build-all --profile ci-sim
+risedev sslt-build-all --profile ci-sim --timings
 
 echo "--- Show sccache stats"
 sccache --show-stats
@@ -25,6 +25,7 @@ sccache --zero-stats
 echo "--- Upload artifacts"
 mv target/sim/ci-sim/risingwave_simulation ./risingwave_simulation
 tar --zstd -cvf risingwave_simulation.tar.zst risingwave_simulation
+buildkite-agent artifact upload target/sim/cargo-timings/cargo-timing.html
 
 artifacts=(risingwave_simulation.tar.zst simulation-it-test.tar.zst)
 echo -n "${artifacts[*]}" | parallel -d ' ' "buildkite-agent artifact upload ./{}"
diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh
index f50c5a794ec8d..f595bede10d24 100755
--- a/ci/scripts/build.sh
+++ b/ci/scripts/build.sh
@@ -44,6 +44,7 @@ if [[ "$profile" == "ci-dev" ]]; then
     RISINGWAVE_FEATURE_FLAGS=(--features rw-dynamic-link --no-default-features)
 else
     RISINGWAVE_FEATURE_FLAGS=(--features rw-static-link)
+    export OPENSSL_STATIC=1
 fi
 
 cargo build \
@@ -54,9 +55,9 @@ cargo build \
     -p risingwave_compaction_test \
     -p risingwave_e2e_extended_mode_test \
     "${RISINGWAVE_FEATURE_FLAGS[@]}" \
-    --features embedded-deno-udf \
-    --features embedded-python-udf \
-    --profile "$profile"
+    --features all-udf \
+    --profile "$profile" \
+    --timings
 
 
 artifacts=(risingwave sqlsmith compaction-test risingwave_regress_test risingwave_e2e_extended_mode_test risedev-dev delete-range-test)
@@ -66,6 +67,7 @@ ldd target/"$profile"/risingwave
 
 echo "--- Upload artifacts"
 echo -n "${artifacts[*]}" | parallel -d ' ' "mv target/$profile/{} ./{}-$profile && compress-and-upload-artifact ./{}-$profile"
+buildkite-agent artifact upload target/cargo-timings/cargo-timing.html
 
 # This magically makes it faster to exit the docker
 rm -rf target
diff --git a/ci/scripts/check-dylint.sh b/ci/scripts/check-dylint.sh
index 77dc5dbff71f0..8dbd9e397757c 100755
--- a/ci/scripts/check-dylint.sh
+++ b/ci/scripts/check-dylint.sh
@@ -6,6 +6,15 @@ set -euo pipefail
 source ci/scripts/common.sh
 unset RUSTC_WRAPPER # disable sccache, see https://github.com/mozilla/sccache/issues/861
 
+echo "--- List all available lints"
+output=$(cargo dylint list)
+if [ -z "$output" ]; then
+  echo "ERROR: No lints detected. There might be an issue with the configuration of the lints crate."
+  exit 1
+else
+  echo "$output"
+fi
+
 echo "--- Run dylint check (dev, all features)"
 # Instead of `-D warnings`, we only deny warnings from our own lints. This is because...
 # - Warnings from `check` or `clippy` are already checked in `check.sh`.
diff --git a/ci/scripts/check.sh b/ci/scripts/check.sh
index f52a1b4fd3e8a..b0d0b3e678ae8 100755
--- a/ci/scripts/check.sh
+++ b/ci/scripts/check.sh
@@ -28,11 +28,11 @@ sccache --show-stats
 sccache --zero-stats
 
 echo "--- Run clippy check (release)"
-cargo clippy --release --all-targets --features "rw-static-link" --locked -- -D warnings
+OPENSSL_STATIC=1 cargo clippy --release --all-targets --features "rw-static-link" --locked -- -D warnings
 
 echo "--- Run cargo check on building the release binary (release)"
-cargo check -p risingwave_cmd_all --features "rw-static-link" --profile release
-cargo check -p risingwave_cmd --bin risectl --features "rw-static-link" --profile release
+OPENSSL_STATIC=1 cargo check -p risingwave_cmd_all --features "rw-static-link" --profile release
+OPENSSL_STATIC=1 cargo check -p risingwave_cmd --bin risectl --features "rw-static-link" --profile release
 
 echo "--- Show sccache stats"
 sccache --show-stats
diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
index d99066cb3d5e2..3b31afeef253d 100755
--- a/ci/scripts/common.sh
+++ b/ci/scripts/common.sh
@@ -12,8 +12,9 @@ export CARGO_INCREMENTAL=0
 export CARGO_MAKE_PRINT_TIME_SUMMARY=true
 export MINIO_DOWNLOAD_BIN=https://rw-ci-deps-dist.s3.amazonaws.com/minio
 export MCLI_DOWNLOAD_BIN=https://rw-ci-deps-dist.s3.amazonaws.com/mc
-export GCLOUD_DOWNLOAD_TGZ=https://rw-ci-deps-dist.s3.amazonaws.com/google-cloud-cli-406.0.0-linux-x86_64.tar.gz
+export GCLOUD_DOWNLOAD_TGZ=https://rw-ci-deps-dist.s3.amazonaws.com/google-cloud-cli-475.0.0-linux-x86_64.tar.gz
 export NEXTEST_HIDE_PROGRESS_BAR=true
+export RW_TELEMETRY_TYPE=test
 unset LANG
 if [ -n "${BUILDKITE_COMMIT:-}" ]; then
   export GIT_SHA=$BUILDKITE_COMMIT
diff --git a/ci/scripts/connector-node-integration-test.sh b/ci/scripts/connector-node-integration-test.sh
index 519215584abb9..8853243b66805 100755
--- a/ci/scripts/connector-node-integration-test.sh
+++ b/ci/scripts/connector-node-integration-test.sh
@@ -30,8 +30,6 @@ shift $((OPTIND -1))
 RISINGWAVE_ROOT=${PWD}
 
 echo "--- install java"
-apt install sudo -y && apt-get update
-
 if [ "$VERSION" = "11" ]; then
   echo "The test imgae default java version is 11, no need to install"
 else
@@ -92,15 +90,15 @@ export PYTHONPATH=proto
 
 echo "--- running streamchunk data format integration tests"
 cd "${RISINGWAVE_ROOT}"/java/connector-node/python-client
-if python3 integration_tests.py --stream_chunk_format_test --input_binary_file="./data/stream_chunk_data" --data_format_use_json=False; then
+if python3 integration_tests.py --stream_chunk_format_test --input_binary_file="./data/stream_chunk_data"; then
   echo "StreamChunk data format test passed"
 else
   echo "StreamChunk data format test failed"
   exit 1
 fi
 
-sink_input_feature=("--input_binary_file=./data/sink_input --data_format_use_json=False")
-upsert_sink_input_feature=("--input_binary_file=./data/upsert_sink_input --data_format_use_json=False")
+sink_input_feature=("--input_binary_file=./data/sink_input")
+upsert_sink_input_feature=("--input_binary_file=./data/upsert_sink_input")
 type=("StreamChunk format")
 
 ${MC_PATH} mb minio/bucket
diff --git a/ci/scripts/docker.sh b/ci/scripts/docker.sh
index f7936f9987d70..a909b7c6a37c5 100755
--- a/ci/scripts/docker.sh
+++ b/ci/scripts/docker.sh
@@ -13,6 +13,14 @@ echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
 echo "--- dockerhub login"
 echo "$DOCKER_TOKEN" | docker login -u "risingwavelabs" --password-stdin
 
+if [[ -n "${ORIGINAL_IMAGE_TAG+x}" ]] && [[ -n "${NEW_IMAGE_TAG+x}" ]]; then
+  echo "--- retag docker image"
+  docker pull ${ghcraddr}:${ORIGINAL_IMAGE_TAG}
+  docker tag ${ghcraddr}:${ORIGINAL_IMAGE_TAG} ${ghcraddr}:${NEW_IMAGE_TAG}-${arch}
+  docker push ${ghcraddr}:${NEW_IMAGE_TAG}-${arch}
+  exit 0
+fi
+
 # Build RisingWave docker image ${BUILDKITE_COMMIT}-${arch}
 echo "--- docker build and tag"
 docker buildx create \
diff --git a/ci/scripts/e2e-kafka-sink-test.sh b/ci/scripts/e2e-kafka-sink-test.sh
index 1dd7a27831d49..7cab1ae1f76f7 100755
--- a/ci/scripts/e2e-kafka-sink-test.sh
+++ b/ci/scripts/e2e-kafka-sink-test.sh
@@ -3,19 +3,22 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only --create > /dev/null 2>&1
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert --create > /dev/null 2>&1
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert-schema --create > /dev/null 2>&1
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-debezium --create > /dev/null 2>&1
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-without-snapshot --create > /dev/null 2>&1
+export RPK_BROKERS="message_queue:29092"
+
+rpk topic create test-rw-sink-append-only
+rpk topic create test-rw-sink-upsert
+rpk topic create test-rw-sink-upsert-schema
+rpk topic create test-rw-sink-debezium
+rpk topic create test-rw-sink-without-snapshot
+rpk topic create test-rw-sink-text-key-id
 
 sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/create_sink.slt'
 sleep 2
 
 # test append-only kafka sink
 echo "testing append-only kafka sink"
-diff ./e2e_test/sink/kafka/append_only1.result \
-<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only --from-beginning --max-messages 10 | sort) 2> /dev/null)
+diff -b ./e2e_test/sink/kafka/append_only1.result \
+<((rpk topic consume test-rw-sink-append-only --offset start --format '%v\n' --num 10 | sort) 2> /dev/null)
 if [ $? -ne 0 ]; then
   echo "The output for append-only sink is not as expected."
   exit 1
@@ -23,8 +26,8 @@ fi
 
 # test upsert kafka sink
 echo "testing upsert kafka sink"
-diff ./e2e_test/sink/kafka/upsert1.result \
-<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert --from-beginning --property print.key=true --max-messages 10 | sort) 2> /dev/null)
+diff -b ./e2e_test/sink/kafka/upsert1.result \
+<((rpk topic consume test-rw-sink-upsert --offset start --format '%k\t%v\n' --num 10 | sort) 2> /dev/null)
 if [ $? -ne 0 ]; then
   echo "The output for upsert sink is not as expected."
   exit 1
@@ -32,8 +35,8 @@ fi
 
 # test upsert kafka sink with schema
 echo "testing upsert kafka sink with schema"
-diff ./e2e_test/sink/kafka/upsert_schema1.result \
-<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 10 | sort) 2> /dev/null)
+diff -b ./e2e_test/sink/kafka/upsert_schema1.result \
+<((rpk topic consume test-rw-sink-upsert-schema --offset start --format '%k\t%v\n' --num 10 | sort) 2> /dev/null)
 if [ $? -ne 0 ]; then
   echo "The output for upsert sink with schema is not as expected."
   exit 1
@@ -41,7 +44,7 @@ fi
 
 # test debezium kafka sink
 echo "testing debezium kafka sink"
-(./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 10 | sort) > ./e2e_test/sink/kafka/debezium1.tmp.result 2> /dev/null
+(rpk topic consume test-rw-sink-debezium --offset start --format '%k\t%v\n'  --num 10 | sort) > ./e2e_test/sink/kafka/debezium1.tmp.result 2> /dev/null
 python3 e2e_test/sink/kafka/debezium.py e2e_test/sink/kafka/debezium1.result e2e_test/sink/kafka/debezium1.tmp.result
 if [ $? -ne 0 ]; then
   echo "The output for debezium sink is not as expected."
@@ -57,8 +60,8 @@ psql -h localhost -p 4566 -d dev -U root -c "update t_kafka set v_varchar = '',
 
 # test append-only kafka sink after update
 echo "testing append-only kafka sink after updating data"
-diff ./e2e_test/sink/kafka/append_only2.result \
-<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only --from-beginning --max-messages 11 | sort) 2> /dev/null)
+diff -b ./e2e_test/sink/kafka/append_only2.result \
+<((rpk topic consume test-rw-sink-append-only --offset start --format '%v\n' --num 11 | sort) 2> /dev/null)
 if [ $? -ne 0 ]; then
   echo "The output for append-only sink after update is not as expected."
   exit 1
@@ -66,8 +69,8 @@ fi
 
 # test upsert kafka sink after update
 echo "testing upsert kafka sink after updating data"
-diff ./e2e_test/sink/kafka/upsert2.result \
-<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert --from-beginning --property print.key=true --max-messages 11 | sort) 2> /dev/null)
+diff -b ./e2e_test/sink/kafka/upsert2.result \
+<((rpk topic consume test-rw-sink-upsert --offset start --format '%k\t%v\n' --num 11 | sort) 2> /dev/null)
 if [ $? -ne 0 ]; then
   echo "The output for upsert sink after update is not as expected."
   exit 1
@@ -75,8 +78,8 @@ fi
 
 # test upsert kafka sink with schema after update
 echo "testing upsert kafka sink with schema after updating data"
-diff ./e2e_test/sink/kafka/upsert_schema2.result \
-<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 11 | sort) 2> /dev/null)
+diff -b ./e2e_test/sink/kafka/upsert_schema2.result \
+<((rpk topic consume test-rw-sink-upsert-schema --offset start --format '%k\t%v\n' --num 11 | sort) 2> /dev/null)
 if [ $? -ne 0 ]; then
   echo "The output for upsert sink with schema is not as expected."
   exit 1
@@ -84,7 +87,7 @@ fi
 
 # test debezium kafka sink after update
 echo "testing debezium kafka sink after updating data"
-(./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 11  | sort) > ./e2e_test/sink/kafka/debezium2.tmp.result 2> /dev/null
+(rpk topic consume test-rw-sink-debezium --offset start --format '%k\t%v\n' --num 11 | sort) > ./e2e_test/sink/kafka/debezium2.tmp.result 2> /dev/null
 python3 e2e_test/sink/kafka/debezium.py e2e_test/sink/kafka/debezium2.result e2e_test/sink/kafka/debezium2.tmp.result
 if [ $? -ne 0 ]; then
   echo "The output for debezium sink after update is not as expected."
@@ -96,8 +99,8 @@ fi
 
 # test without-snapshot kafka sink
 echo "testing without-snapshot kafka sink"
-diff ./e2e_test/sink/kafka/without_snapshot.result \
-<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-without-snapshot --from-beginning --max-messages 3 | sort) 2> /dev/null)
+diff -b ./e2e_test/sink/kafka/without_snapshot.result \
+<((rpk topic consume test-rw-sink-without-snapshot --offset start --format '%v\n' --num 3 | sort) 2> /dev/null)
 if [ $? -ne 0 ]; then
   echo "The output for append-only sink is not as expected."
   exit 1
@@ -109,8 +112,8 @@ psql -h localhost -p 4566 -d dev -U root -c "delete from t_kafka where id = 1;"
 
 # test upsert kafka sink after delete
 echo "testing upsert kafka sink after deleting data"
-diff ./e2e_test/sink/kafka/upsert3.result \
-<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert --from-beginning --property print.key=true --max-messages 12 | sort) 2> /dev/null)
+diff -b ./e2e_test/sink/kafka/upsert3.result \
+<((rpk topic consume test-rw-sink-upsert --offset start --format '%k\t%v\n' --num 12 | sort) 2> /dev/null)
 if [ $? -ne 0 ]; then
   echo "The output for upsert sink after update is not as expected."
   exit 1
@@ -118,8 +121,8 @@ fi
 
 # test upsert kafka sink with schema after delete
 echo "testing upsert kafka sink with schema after deleting data"
-diff ./e2e_test/sink/kafka/upsert_schema3.result \
-<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 12 | sort) 2> /dev/null)
+diff -b ./e2e_test/sink/kafka/upsert_schema3.result \
+<((rpk topic consume test-rw-sink-upsert-schema --offset start --format '%k\t%v\n' --num 12 | sort) 2> /dev/null)
 if [ $? -ne 0 ]; then
   echo "The output for upsert sink with schema is not as expected."
   exit 1
@@ -127,7 +130,7 @@ fi
 
 # test debezium kafka sink after delete
 echo "testing debezium kafka sink after deleting data"
-(./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 13 | sort) > ./e2e_test/sink/kafka/debezium3.tmp.result 2> /dev/null
+(rpk topic consume test-rw-sink-debezium --offset start --format '%k\t%v\n' --num 13 | sort) > ./e2e_test/sink/kafka/debezium3.tmp.result 2> /dev/null
 python3 e2e_test/sink/kafka/debezium.py e2e_test/sink/kafka/debezium3.result e2e_test/sink/kafka/debezium3.tmp.result
 if [ $? -ne 0 ]; then
   echo "The output for debezium sink after delete is not as expected."
@@ -138,9 +141,9 @@ else
 fi
 
 sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/drop_sink.slt'
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only --delete > /dev/null 2>&1
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert --delete > /dev/null 2>&1
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-debezium --delete > /dev/null 2>&1
+rpk topic delete test-rw-sink-append-only
+rpk topic delete test-rw-sink-upsert
+rpk topic delete test-rw-sink-debezium
 
 # test different encoding
 echo "preparing confluent schema registry"
@@ -148,19 +151,19 @@ python3 -m pip install --break-system-packages requests confluent-kafka
 
 echo "testing protobuf"
 cp src/connector/src/test_data/proto_recursive/recursive.pb ./proto-recursive
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only-protobuf --create > /dev/null 2>&1
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only-protobuf-csr-a --create > /dev/null 2>&1
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only-protobuf-csr-hi --create > /dev/null 2>&1
-python3 e2e_test/sink/kafka/register_schema.py 'http://message_queue:8081' 'test-rw-sink-append-only-protobuf-csr-a-value' src/connector/src/test_data/test-index-array.proto
-python3 e2e_test/sink/kafka/register_schema.py 'http://message_queue:8081' 'test-rw-sink-append-only-protobuf-csr-hi-value' src/connector/src/test_data/test-index-array.proto
+rpk topic create test-rw-sink-append-only-protobuf
+rpk topic create test-rw-sink-append-only-protobuf-csr-a
+rpk topic create test-rw-sink-append-only-protobuf-csr-hi
+python3 e2e_test/sink/kafka/register_schema.py 'http://schemaregistry:8082' 'test-rw-sink-append-only-protobuf-csr-a-value' src/connector/src/test_data/test-index-array.proto
+python3 e2e_test/sink/kafka/register_schema.py 'http://schemaregistry:8082' 'test-rw-sink-append-only-protobuf-csr-hi-value' src/connector/src/test_data/test-index-array.proto
 sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/protobuf.slt'
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only-protobuf-csr-hi --delete > /dev/null 2>&1
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only-protobuf-csr-a --delete > /dev/null 2>&1
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-append-only-protobuf --delete > /dev/null 2>&1
+rpk topic delete test-rw-sink-append-only-protobuf
+rpk topic delete test-rw-sink-append-only-protobuf-csr-a
+rpk topic delete test-rw-sink-append-only-protobuf-csr-hi
 
 echo "testing avro"
-python3 e2e_test/sink/kafka/register_schema.py 'http://message_queue:8081' 'test-rw-sink-upsert-avro-value' src/connector/src/test_data/all-types.avsc
-python3 e2e_test/sink/kafka/register_schema.py 'http://message_queue:8081' 'test-rw-sink-upsert-avro-key' src/connector/src/test_data/all-types.avsc 'string_field,int32_field'
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert-avro --create > /dev/null 2>&1
+python3 e2e_test/sink/kafka/register_schema.py 'http://schemaregistry:8082' 'test-rw-sink-upsert-avro-value' src/connector/src/test_data/all-types.avsc
+python3 e2e_test/sink/kafka/register_schema.py 'http://schemaregistry:8082' 'test-rw-sink-upsert-avro-key' src/connector/src/test_data/all-types.avsc 'string_field,int32_field'
+rpk topic create test-rw-sink-upsert-avro
 sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/avro.slt'
-./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server message_queue:29092 --topic test-rw-sink-upsert-avro --delete > /dev/null 2>&1
+rpk topic delete test-rw-sink-upsert-avro
diff --git a/ci/scripts/e2e-redis-sink-test.sh b/ci/scripts/e2e-redis-sink-test.sh
index c7567dd51ed04..45e578b9268af 100755
--- a/ci/scripts/e2e-redis-sink-test.sh
+++ b/ci/scripts/e2e-redis-sink-test.sh
@@ -56,7 +56,7 @@ sqllogictest -p 4566 -d dev './e2e_test/sink/redis_cluster_sink.slt'
 redis-cli -c --cluster call 127.0.0.1:7000 keys \* >> ./query_result_1.txt
 
 line_count=$(wc -l < query_result_1.txt)
-if [ "$line_count" -eq 4 ]; then
+if [ "$line_count" -eq 16 ]; then
     echo "Redis sink check passed"
 else
     cat ./query_result_1.txt
diff --git a/ci/scripts/e2e-sink-test.sh b/ci/scripts/e2e-sink-test.sh
index 5125ddedfdf8e..eaee563c7a992 100755
--- a/ci/scripts/e2e-sink-test.sh
+++ b/ci/scripts/e2e-sink-test.sh
@@ -45,7 +45,7 @@ mysql --host=mysql --port=3306 -u root -p123456 test < ./e2e_test/sink/remote/my
 echo "--- preparing postgresql"
 
 # set up PG sink destination
-apt-get -y install postgresql-client
+apt-get -y install postgresql-client jq
 export PGPASSWORD=postgres
 psql -h db -U postgres -c "CREATE ROLE test LOGIN SUPERUSER PASSWORD 'connector';"
 createdb -h db -U postgres test
diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh
index 8a683f56b8550..84fff651b547c 100755
--- a/ci/scripts/e2e-source-test.sh
+++ b/ci/scripts/e2e-source-test.sh
@@ -31,6 +31,16 @@ buildkite-agent artifact download risingwave-connector.tar.gz ./
 mkdir ./connector-node
 tar xf ./risingwave-connector.tar.gz -C ./connector-node
 
+echo "--- Install dependencies"
+python3 -m pip install --break-system-packages requests protobuf fastavro confluent_kafka jsonschema
+
+echo "--- e2e, inline test"
+RUST_LOG="debug,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
+risedev ci-start ci-inline-source-test
+risedev slt './e2e_test/source_inline/**/*.slt'
+echo "--- Kill cluster"
+risedev ci-kill
+
 echo "--- Prepare data"
 cp src/connector/src/test_data/simple-schema.avsc ./avro-simple-schema.avsc
 cp src/connector/src/test_data/complex-schema.avsc ./avro-complex-schema.avsc
@@ -49,7 +59,7 @@ createdb
 psql < ./e2e_test/source/cdc/postgres_cdc.sql
 
 echo "--- starting risingwave cluster"
-RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
+RUST_LOG="debug,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
 risedev ci-start ci-1cn-1fe-with-recovery
 
 echo "--- mongodb cdc test"
@@ -82,15 +92,6 @@ echo "--- cdc share source test"
 export MYSQL_HOST=mysql MYSQL_TCP_PORT=3306 MYSQL_PWD=123456
 risedev slt './e2e_test/source/cdc/cdc.share_stream.slt'
 
-# create a share source and check whether heartbeat message is received
-risedev slt './e2e_test/source/cdc/cdc.create_source_job.slt'
-table_id=$(psql -U root -h localhost -p 4566 -d dev -t -c "select id from rw_internal_tables where name like '%mysql_source%';" | xargs);
-table_count=$(psql -U root -h localhost -p 4566 -d dev -t -c "select count(*) from rw_table(${table_id}, public);" | xargs);
-if [ "$table_count" -eq 0 ]; then
-    echo "ERROR: internal table of cdc share source is empty!"
-    exit 1
-fi
-
 echo "--- mysql & postgres load and check"
 risedev slt './e2e_test/source/cdc/cdc.load.slt'
 # wait for cdc loading
@@ -138,12 +139,11 @@ echo "--- e2e, ci-1cn-1fe, protobuf schema registry"
 export RISINGWAVE_CI=true
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
 risedev ci-start ci-1cn-1fe
-python3 -m pip install --break-system-packages requests protobuf confluent-kafka
-python3 e2e_test/schema_registry/pb.py "message_queue:29092" "http://message_queue:8081" "sr_pb_test" 20 user
+python3 e2e_test/schema_registry/pb.py "message_queue:29092" "http://schemaregistry:8082" "sr_pb_test" 20 user
 echo "make sure google/protobuf/source_context.proto is NOT in schema registry"
-curl --silent 'http://message_queue:8081/subjects'; echo
-# curl --silent --head -X GET 'http://message_queue:8081/subjects/google%2Fprotobuf%2Fsource_context.proto/versions' | grep 404
-curl --silent 'http://message_queue:8081/subjects' | grep -v 'google/protobuf/source_context.proto'
+curl --silent 'http://schemaregistry:8082/subjects'; echo
+# curl --silent --head -X GET 'http://schemaregistry:8082/subjects/google%2Fprotobuf%2Fsource_context.proto/versions' | grep 404
+curl --silent 'http://schemaregistry:8082/subjects' | grep -v 'google/protobuf/source_context.proto'
 risedev slt './e2e_test/schema_registry/pb.slt'
 risedev slt './e2e_test/schema_registry/alter_sr.slt'
 
@@ -152,13 +152,16 @@ risedev ci-kill
 
 echo "--- e2e, ci-kafka-plus-pubsub, kafka and pubsub source"
 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
-risedev ci-start ci-pubsub-kafka
+risedev ci-start ci-kafka
 ./scripts/source/prepare_ci_kafka.sh
-cargo run --bin prepare_ci_pubsub
 risedev slt './e2e_test/source/basic/*.slt'
 risedev slt './e2e_test/source/basic/old_row_format_syntax/*.slt'
 risedev slt './e2e_test/source/basic/alter/kafka.slt'
 
+echo "--- e2e, kafka alter source rate limit"
+risedev slt './e2e_test/source/basic/alter/rate_limit_source_kafka.slt'
+risedev slt './e2e_test/source/basic/alter/rate_limit_table_kafka.slt'
+
 echo "--- e2e, kafka alter source"
 chmod +x ./scripts/source/prepare_data_after_alter.sh
 ./scripts/source/prepare_data_after_alter.sh 2
@@ -168,9 +171,6 @@ echo "--- e2e, kafka alter source again"
 ./scripts/source/prepare_data_after_alter.sh 3
 risedev slt './e2e_test/source/basic/alter/kafka_after_new_data_2.slt'
 
-echo "--- e2e, inline test"
-risedev slt './e2e_test/source_inline/**/*.slt'
-
 echo "--- Run CH-benCHmark"
 risedev slt './e2e_test/ch_benchmark/batch/ch_benchmark.slt'
 risedev slt './e2e_test/ch_benchmark/streaming/*.slt'
diff --git a/ci/scripts/e2e-sqlserver-sink-test.sh b/ci/scripts/e2e-sqlserver-sink-test.sh
new file mode 100755
index 0000000000000..f1f62941375ce
--- /dev/null
+++ b/ci/scripts/e2e-sqlserver-sink-test.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+source ci/scripts/common.sh
+
+while getopts 'p:' opt; do
+    case ${opt} in
+        p )
+            profile=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+download_and_prepare_rw "$profile" source
+
+echo "--- starting risingwave cluster"
+risedev ci-start ci-sink-test
+sleep 1
+
+echo "--- create SQL Server table"
+curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
+curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
+apt-get update -y
+ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y mssql-tools unixodbc-dev
+export PATH="/opt/mssql-tools/bin/:$PATH"
+sleep 2
+
+sqlcmd -S sqlserver-server -U SA -P SomeTestOnly@SA -Q "
+CREATE DATABASE SinkTest;
+GO
+USE SinkTest;
+CREATE TABLE t_many_data_type (
+  k1 int, k2 int,
+  c_boolean bit,
+  c_int16 smallint,
+  c_int32 int,
+  c_int64 bigint,
+  c_float32 float,
+  c_float64 float,
+  c_decimal decimal,
+  c_date date,
+  c_time time,
+  c_timestamp datetime2,
+  c_timestampz datetime2,
+  c_nvarchar nvarchar(1024),
+  c_varbinary varbinary(1024),
+PRIMARY KEY (k1,k2));
+GO"
+sleep 2
+
+echo "--- testing sinks"
+sqllogictest -p 4566 -d dev './e2e_test/sink/sqlserver_sink.slt'
+sleep 1
+sqlcmd -S sqlserver-server -U SA -P SomeTestOnly@SA -h -1 -Q "
+SELECT * FROM SinkTest.dbo.t_many_data_type;
+GO" > ./query_result.txt
+
+mapfile -t actual < <(tr -s '[:space:]' '\n' < query_result.txt)
+actual=("${actual[@]:1}")
+expected=(0 0 0 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 1 1 0 55 55 1 1.0 1.0 1 2022-04-08 18:20:49.0000000 2022-03-13 01:00:00.0000000 2022-03-13 01:00:00.0000000 Hello World! 0xDE00BEEF 1 2 0 66 66 1 1.0 1.0 1 2022-04-08 18:20:49.0000000 2022-03-13 01:00:00.0000000 2022-03-13 01:00:00.0000000 Hello World! 0xDE00BEEF 1 4 0 2 2 1 1.0 1.0 1 2022-04-08 18:20:49.0000000 2022-03-13 01:00:00.0000000 2022-03-13 01:00:00.0000000 Hello World! 0xDE00BEEF "(4" rows "affected)")
+
+if [[ ${#actual[@]} -eq ${#expected[@]} && ${actual[@]} == ${expected[@]} ]]; then
+  echo "SQL Server sink check passed"
+else
+  cat ./query_result.txt
+  echo "The output is not as expected."
+fi
+
+echo "--- Kill cluster"
+risedev ci-kill
diff --git a/ci/scripts/gen-integration-test-yaml.py b/ci/scripts/gen-integration-test-yaml.py
index 61fd7721acc3f..c778205cfbb3e 100644
--- a/ci/scripts/gen-integration-test-yaml.py
+++ b/ci/scripts/gen-integration-test-yaml.py
@@ -44,6 +44,8 @@
     'presto-trino': ['json'],
     'client-library': ['none'],
     'kafka-cdc': ['json'],
+    'pubsub': ['json'],
+    'dynamodb': ['json'],
 }
 
 def gen_pipeline_steps():
diff --git a/ci/scripts/integration-tests.sh b/ci/scripts/integration-tests.sh
index 9502c297f87ca..c686e83cd9cbd 100755
--- a/ci/scripts/integration-tests.sh
+++ b/ci/scripts/integration-tests.sh
@@ -25,11 +25,12 @@ shift $((OPTIND -1))
 echo "export INTEGRATION_TEST_CASE=${case}" > env_vars.sh
 
 echo "~~~ clean up docker"
+# shellcheck disable=SC2046
 if [ $(docker ps -aq |wc -l) -gt 0 ]; then
   docker rm -f $(docker ps -aq)
 fi
 docker network prune -f
-docker volume prune -f
+docker volume prune -f -a
 
 echo "~~~ ghcr login"
 echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
@@ -54,8 +55,7 @@ fi
 echo "--- case: ${case}, format: ${format}"
 
 if [ "${case}" == "client-library" ]; then
-  cd integration_tests/client-library
-  python3 client_test.py
+  python3 integration_tests/client-library/client_test.py
   exit 0
 fi
 
@@ -70,11 +70,9 @@ export PATH=$PATH:$HOME/.local/bin
 echo "--- download rwctest-key"
 aws secretsmanager get-secret-value --secret-id "gcp-buildkite-rwctest-key" --region us-east-2 --query "SecretString" --output text >gcp-rwctest.json
 
-cd integration_tests/scripts
-
 echo "--- rewrite docker compose for protobuf"
 if [ "${format}" == "protobuf" ]; then
-  python3 gen_pb_compose.py "${case}" "${format}"
+  python3 integration_tests/scripts/gen_pb_compose.py "${case}" "${format}"
 fi
 
 echo "--- set vm.max_map_count=2000000 for doris"
@@ -82,7 +80,7 @@ max_map_count_original_value=$(sysctl -n vm.max_map_count)
 sudo sysctl -w vm.max_map_count=2000000
 
 echo "--- run Demos"
-python3 run_demos.py --case "${case}" --format "${format}"
+python3 integration_tests/scripts/run_demos.py --case "${case}" --format "${format}"
 
 echo "--- run docker ps"
 docker ps
@@ -90,10 +88,7 @@ docker ps
 echo "--- check if the ingestion is successful"
 # extract the type of upstream source,e.g. mysql,postgres,etc
 upstream=$(echo "${case}" | cut -d'-' -f 1)
-python3 check_data.py "${case}" "${upstream}"
-
-echo "--- clean Demos"
-python3 clean_demos.py --case "${case}"
+python3 integration_tests/scripts/check_data.py "${case}" "${upstream}"
 
 echo "--- reset vm.max_map_count={$max_map_count_original_value}"
 sudo sysctl -w vm.max_map_count="$max_map_count_original_value"
diff --git a/ci/scripts/multi-arch-docker.sh b/ci/scripts/multi-arch-docker.sh
index b97b606201d6a..012378347ba28 100755
--- a/ci/scripts/multi-arch-docker.sh
+++ b/ci/scripts/multi-arch-docker.sh
@@ -57,8 +57,20 @@ echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
 echo "--- dockerhub login"
 echo "$DOCKER_TOKEN" | docker login -u "risingwavelabs" --password-stdin
 
+if [[ -n "${ORIGINAL_IMAGE_TAG+x}" ]] && [[ -n "${NEW_IMAGE_TAG+x}" ]]; then
+  echo "--- retag docker image"
+  echo "push to gchr, image tag: ${ghcraddr}:${NEW_IMAGE_TAG}"
+  args=()
+  for arch in "${arches[@]}"
+  do
+    args+=( --amend "${ghcraddr}:${NEW_IMAGE_TAG}-${arch}" )
+  done
+  docker manifest create --insecure "${ghcraddr}:${NEW_IMAGE_TAG}" "${args[@]}"
+  docker manifest push --insecure "${ghcraddr}:${NEW_IMAGE_TAG}"
+  exit 0
+fi
 
-echo "--- multi arch image create "
+echo "--- multi arch image create"
 if [[ "${#BUILDKITE_COMMIT}" = 40 ]]; then
   # If the commit is 40 characters long, it's probably a SHA.
   TAG="git-${BUILDKITE_COMMIT}"
diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh
index 1c68b8acf9fa6..94fd38e2c9c75 100755
--- a/ci/scripts/release.sh
+++ b/ci/scripts/release.sh
@@ -19,11 +19,11 @@ echo "--- Install lld"
 dnf install -y lld
 ld.lld --version
 
-echo "--- Install dependencies for openssl"
-dnf install -y perl-core
+echo "--- Install dependencies"
+dnf install -y perl-core wget python3 python3-devel cyrus-sasl-devel rsync openssl-devel
 
 echo "--- Install java and maven"
-dnf install -y java-11-openjdk java-11-openjdk-devel wget python3 python3-devel cyrus-sasl-devel
+dnf install -y java-17-openjdk java-17-openjdk-devel
 pip3 install toml-cli
 wget https://rw-ci-deps-dist.s3.amazonaws.com/apache-maven-3.9.3-bin.tar.gz && tar -zxvf apache-maven-3.9.3-bin.tar.gz
 export PATH="${REPO_ROOT}/apache-maven-3.9.3/bin:$PATH"
@@ -71,8 +71,8 @@ if [ "${ARCH}" == "aarch64" ]; then
   # see https://github.com/tikv/jemallocator/blob/802969384ae0c581255f3375ee2ba774c8d2a754/jemalloc-sys/build.rs#L218
   export JEMALLOC_SYS_WITH_LG_PAGE=16
 fi
-cargo build -p risingwave_cmd_all --features "rw-static-link" --profile release
-cargo build -p risingwave_cmd --bin risectl --features "rw-static-link" --profile release
+OPENSSL_STATIC=1 cargo build -p risingwave_cmd_all --features "rw-static-link" --features external-udf --features wasm-udf --features js-udf --profile release
+OPENSSL_STATIC=1 cargo build -p risingwave_cmd --bin risectl --features "rw-static-link" --profile release
 cd target/release && chmod +x risingwave risectl
 
 echo "--- Upload nightly binary to s3"
diff --git a/ci/scripts/run-backfill-tests.sh b/ci/scripts/run-backfill-tests.sh
index f01e2c5b3d771..c03762a7150a7 100755
--- a/ci/scripts/run-backfill-tests.sh
+++ b/ci/scripts/run-backfill-tests.sh
@@ -13,6 +13,7 @@
 #   1002 | CREATE MATERIALIZED VIEW m1 AS SELECT * FROM t | 56.12%   | 2023-09-27 06:37:06.636+00:00
 #(1 row)
 
+# TODO: refactor with inline style.
 
 set -euo pipefail
 
@@ -22,7 +23,7 @@ TEST_DIR=$PWD/e2e_test
 BACKGROUND_DDL_DIR=$TEST_DIR/background_ddl
 COMMON_DIR=$BACKGROUND_DDL_DIR/common
 
-CLUSTER_PROFILE='ci-1cn-1fe-kafka-with-recovery'
+CLUSTER_PROFILE='ci-1cn-1fe-user-kafka-with-recovery'
 echo "--- Configuring cluster profiles"
 if [[ -n "${BUILDKITE:-}" ]]; then
   echo "Running in buildkite"
@@ -184,16 +185,19 @@ test_sink_backfill_recovery() {
   # Check progress
   sqllogictest -p 4566 -d dev 'e2e_test/backfill/sink/create_sink.slt'
 
+  # Sleep before restart cluster, to ensure the downstream sink actually gets created.
+  sleep 5
+
   # Restart
   restart_cluster
-  sleep 3
+  sleep 5
 
   # Sink back into rw
   run_sql "CREATE TABLE table_kafka (v1 int primary key)
     WITH (
       connector = 'kafka',
       topic = 's_kafka',
-      properties.bootstrap.server = 'localhost:29092',
+      properties.bootstrap.server = 'message_queue:29092',
   ) FORMAT DEBEZIUM ENCODE JSON;"
 
   sleep 10
diff --git a/ci/scripts/run-e2e-test.sh b/ci/scripts/run-e2e-test.sh
index 5ce0b55f27e9e..4736f4aa53a8a 100755
--- a/ci/scripts/run-e2e-test.sh
+++ b/ci/scripts/run-e2e-test.sh
@@ -70,7 +70,7 @@ download-and-decompress-artifact e2e_test_generated ./
 download-and-decompress-artifact risingwave_e2e_extended_mode_test-"$profile" target/debug/
 mkdir -p e2e_test/udf/wasm/target/wasm32-wasi/release/
 buildkite-agent artifact download udf.wasm e2e_test/udf/wasm/target/wasm32-wasi/release/
-buildkite-agent artifact download risingwave-udf-example.jar ./
+buildkite-agent artifact download udf.jar ./
 mv target/debug/risingwave_e2e_extended_mode_test-"$profile" target/debug/risingwave_e2e_extended_mode_test
 
 chmod +x ./target/debug/risingwave_e2e_extended_mode_test
@@ -80,6 +80,7 @@ RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=i
 cluster_start
 # Please make sure the regression is expected before increasing the timeout.
 sqllogictest -p 4566 -d dev './e2e_test/streaming/**/*.slt' --junit "streaming-${profile}"
+sqllogictest -p 4566 -d dev './e2e_test/backfill/sink/different_pk_and_dist_key.slt'
 
 echo "--- Kill cluster"
 cluster_stop
@@ -91,7 +92,11 @@ sqllogictest -p 4566 -d dev './e2e_test/ddl/**/*.slt' --junit "batch-ddl-${profi
 if [[ "$mode" != "single-node" ]]; then
   sqllogictest -p 4566 -d dev './e2e_test/background_ddl/basic.slt' --junit "batch-ddl-${profile}"
 fi
-sqllogictest -p 4566 -d dev './e2e_test/visibility_mode/*.slt' --junit "batch-${profile}"
+
+if [[ $mode != "single-node" ]]; then
+  sqllogictest -p 4566 -d dev './e2e_test/visibility_mode/*.slt' --junit "batch-${profile}"
+fi
+
 sqllogictest -p 4566 -d dev './e2e_test/ttl/ttl.slt'
 sqllogictest -p 4566 -d dev './e2e_test/database/prepare.slt'
 sqllogictest -p 4566 -d test './e2e_test/database/test.slt'
@@ -105,6 +110,7 @@ echo "--- e2e, $mode, Apache Superset"
 sqllogictest -p 4566 -d dev './e2e_test/superset/*.slt' --junit "batch-${profile}"
 
 echo "--- e2e, $mode, external python udf"
+python3 -m pip install --break-system-packages arrow-udf==0.2.1
 python3 e2e_test/udf/test.py &
 sleep 1
 sqllogictest -p 4566 -d dev './e2e_test/udf/external_udf.slt'
@@ -112,12 +118,11 @@ pkill python3
 
 sqllogictest -p 4566 -d dev './e2e_test/udf/alter_function.slt'
 sqllogictest -p 4566 -d dev './e2e_test/udf/graceful_shutdown_python.slt'
-sqllogictest -p 4566 -d dev './e2e_test/udf/always_retry_python.slt'
 # FIXME: flaky test
 # sqllogictest -p 4566 -d dev './e2e_test/udf/retry_python.slt'
 
 echo "--- e2e, $mode, external java udf"
-java -jar risingwave-udf-example.jar &
+java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -jar udf.jar &
 sleep 1
 sqllogictest -p 4566 -d dev './e2e_test/udf/external_udf.slt'
 pkill java
@@ -284,3 +289,6 @@ if [[ "$mode" == "standalone" ]]; then
   # Make sure any remaining background task exits.
   wait
 fi
+
+echo "--- Upload JUnit test results"
+buildkite-agent artifact upload "*-junit.xml"
diff --git a/ci/scripts/run-meta-backup-test.sh b/ci/scripts/run-meta-backup-test.sh
index 120d247cedd47..098e4dfcd286c 100755
--- a/ci/scripts/run-meta-backup-test.sh
+++ b/ci/scripts/run-meta-backup-test.sh
@@ -67,14 +67,19 @@ cluster_stop() {
 
 download_and_prepare_rw "$profile" common
 
-echo "--- e2e, ci-meta-backup-test"
-test_root="src/storage/backup/integration_tests"
-BACKUP_TEST_MCLI=".risingwave/bin/mcli" \
-BACKUP_TEST_MCLI_CONFIG=".risingwave/config/mcli" \
-BACKUP_TEST_RW_ALL_IN_ONE="target/debug/risingwave" \
-RW_HUMMOCK_URL="hummock+minio://hummockadmin:hummockadmin@127.0.0.1:9301/hummock001" \
-RW_META_ADDR="http://127.0.0.1:5690" \
-RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
-bash "${test_root}/run_all.sh"
-echo "--- Kill cluster"
-risedev kill
\ No newline at end of file
+sudo apt install sqlite3 -y
+for meta_store_type in "etcd" "sql"; do
+  echo "--- e2e, ci-meta-backup-test-${meta_store_type}"
+  test_root="src/storage/backup/integration_tests"
+  BACKUP_TEST_MCLI=".risingwave/bin/mcli" \
+  BACKUP_TEST_MCLI_CONFIG=".risingwave/config/mcli" \
+  BACKUP_TEST_RW_ALL_IN_ONE="target/debug/risingwave" \
+  RW_HUMMOCK_URL="hummock+minio://hummockadmin:hummockadmin@127.0.0.1:9301/hummock001" \
+  RW_META_ADDR="http://127.0.0.1:5690" \
+  RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
+  META_STORE_TYPE="${meta_store_type}" \
+  RW_SQLITE_DB=".risingwave/data/sqlite/metadata.db" \
+  bash "${test_root}/run_all.sh"
+  echo "--- Kill cluster"
+  risedev kill
+done
diff --git a/ci/scripts/run-unit-test.sh b/ci/scripts/run-unit-test.sh
index d9a723a34fa19..394cdb1a78261 100755
--- a/ci/scripts/run-unit-test.sh
+++ b/ci/scripts/run-unit-test.sh
@@ -5,11 +5,6 @@ set -euo pipefail
 
 REPO_ROOT=${PWD}
 
-echo "+++ Run python UDF SDK unit tests"
-cd "${REPO_ROOT}"/src/expr/udf/python
-python3 -m pytest
-cd "${REPO_ROOT}"
-
 echo "+++ Run unit tests"
 # use tee to disable progress bar
 NEXTEST_PROFILE=ci cargo nextest run --features failpoints,sync_point --workspace --exclude risingwave_simulation
diff --git a/ci/scripts/slow-e2e-test.sh b/ci/scripts/slow-e2e-test.sh
new file mode 100755
index 0000000000000..dc33c2719053b
--- /dev/null
+++ b/ci/scripts/slow-e2e-test.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+while getopts 'p:m:' opt; do
+    case ${opt} in
+        p )
+            profile=$OPTARG
+            ;;
+        m )
+            mode=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+source ci/scripts/common.sh
+
+download_and_prepare_rw "$profile" common
+
+
+echo "--- Download artifacts"
+mkdir -p e2e_test/udf/wasm/target/wasm32-wasi/release/
+buildkite-agent artifact download udf.wasm e2e_test/udf/wasm/target/wasm32-wasi/release/
+buildkite-agent artifact download udf.jar ./
+
+echo "--- e2e, $mode, slow-udf-tests"
+python3 -m pip install --break-system-packages arrow-udf==0.2.1
+RUST_LOG="info" \
+risedev ci-start "$mode"
+sqllogictest -p 4566 -d dev './e2e_test/slow_tests/udf/always_retry_python.slt'
+sqllogictest -p 4566 -d dev './e2e_test/slow_tests/backfill/rate_limit/slow-udf.slt'
diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml
index bf09d5fe66344..c9eaf5cf0c38d 100644
--- a/ci/workflows/main-cron.yml
+++ b/ci/workflows/main-cron.yml
@@ -7,10 +7,6 @@ auto-retry: &auto-retry
 steps:
   - label: "build"
     command: "ci/scripts/build.sh -p ci-release"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-build"
-      || build.env("CI_STEPS") =~ /(^|,)build(,|$$)/
     key: "build"
     plugins:
       - docker-compose#v5.1.0:
@@ -22,10 +18,6 @@ steps:
 
   - label: "build other components"
     command: "ci/scripts/build-other.sh"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-build-other"
-      || build.env("CI_STEPS") =~ /(^|,)build-other(,|$$)/
     key: "build-other"
     plugins:
       - seek-oss/aws-sm#v2.3.1:
@@ -42,10 +34,6 @@ steps:
 
   - label: "build simulation test"
     command: "ci/scripts/build-simulation.sh"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-build-simulation"
-      || build.env("CI_STEPS") =~ /(^|,)build-simulation(,|$$)/
     key: "build-simulation"
     plugins:
       - docker-compose#v5.1.0:
@@ -57,10 +45,6 @@ steps:
 
   - label: "docslt"
     command: "ci/scripts/docslt.sh"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-docslt"
-      || build.env("CI_STEPS") =~ /(^|,)docslt(,|$$)/
     key: "docslt"
     plugins:
       - docker-compose#v5.1.0:
@@ -74,7 +58,7 @@ steps:
     key: "e2e-test-release"
     command: "ci/scripts/cron-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-test"
       || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/
     depends_on:
@@ -90,11 +74,30 @@ steps:
     timeout_in_minutes: 15
     retry: *auto-retry
 
+  - label: "slow end-to-end test (release)"
+    key: "slow-e2e-test-release"
+    command: "ci/scripts/slow-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
+    if: |
+        !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-slow-e2e-tests"
+        || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 8
+    retry: *auto-retry
+
   - label: "meta backup test (release)"
     key: "e2e-meta-backup-test-release"
     command: "ci/scripts/run-meta-backup-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-meta-backup-test"
       || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/
     depends_on:
@@ -114,7 +117,7 @@ steps:
     key: "e2e-test-release-parallel"
     command: "ci/scripts/e2e-test-parallel.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-parallel-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-tests?(,|$$)/
     depends_on:
@@ -132,14 +135,14 @@ steps:
           files: "*-junit.xml"
           format: "junit"
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 10
+    timeout_in_minutes: 11
     retry: *auto-retry
 
   - label: "end-to-end test (parallel, in-memory) (release)"
     key: "e2e-test-release-parallel-memory"
     command: "ci/scripts/e2e-test-parallel-in-memory.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-parallel-in-memory-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-in-memory-tests?(,|$$)/
     depends_on:
@@ -158,7 +161,7 @@ steps:
     key: "e2e-test-release-source"
     command: "ci/scripts/e2e-source-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-source-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-source-tests?(,|$$)/
     depends_on:
@@ -177,7 +180,7 @@ steps:
     key: "e2e-test-release-sink"
     command: "ci/scripts/e2e-sink-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-sink-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-sink-tests?(,|$$)/
     depends_on:
@@ -196,7 +199,7 @@ steps:
     key: "fuzz-test"
     command: "ci/scripts/cron-fuzz-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-sqlsmith-fuzzing-tests"
       || build.env("CI_STEPS") =~ /(^|,)sqlsmith-fuzzing-tests?(,|$$)/
     depends_on:
@@ -218,7 +221,7 @@ steps:
     key: "unit-test"
     command: "ci/scripts/unit-test.sh"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-unit-test"
       || build.env("CI_STEPS") =~ /(^|,)unit-tests?(,|$$)/
     plugins:
@@ -236,9 +239,9 @@ steps:
 
   - label: "unit test (madsim)"
     key: "unit-test-deterministic"
-    command: "MADSIM_TEST_NUM=100 timeout 15m ci/scripts/deterministic-unit-test.sh"
+    command: "MADSIM_TEST_NUM=100 timeout 30m ci/scripts/deterministic-unit-test.sh"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-unit-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)unit-tests?-deterministic-simulation(,|$$)/
     plugins:
@@ -246,14 +249,14 @@ steps:
           run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
-    timeout_in_minutes: 15
+    timeout_in_minutes: 30
     retry: *auto-retry
 
   - label: "integration test (madsim) - scale"
     key: "integration-test-deterministic-scale"
     command: "TEST_NUM=60 ci/scripts/deterministic-it-test.sh scale::"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/
     depends_on: "build-simulation"
@@ -270,7 +273,7 @@ steps:
     key: "integration-test-deterministic-recovery"
     command: "TEST_NUM=60 ci/scripts/deterministic-it-test.sh recovery::"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/
     depends_on: "build-simulation"
@@ -287,7 +290,7 @@ steps:
     key: "integration-test-deterministic-backfill"
     command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh backfill_tests::"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/
     depends_on: "build-simulation"
@@ -304,7 +307,7 @@ steps:
     key: "integration-test-deterministic-storage"
     command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh storage::"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/
     depends_on: "build-simulation"
@@ -321,7 +324,7 @@ steps:
     key: "integration-test-deterministic-sink"
     command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh sink::"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/
     depends_on: "build-simulation"
@@ -338,7 +341,7 @@ steps:
     key: "e2e-test-deterministic"
     command: "TEST_NUM=64 timeout 75m ci/scripts/deterministic-e2e-test.sh"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)e2e-tests?-deterministic-simulation(,|$$)/
     depends_on: "build-simulation"
@@ -360,7 +363,7 @@ steps:
     key: "recovery-test-deterministic"
     command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.0 timeout 65m ci/scripts/deterministic-recovery-test.sh"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)recovery-tests?-deterministic-simulation(,|$$)/
     depends_on: "build-simulation"
@@ -379,26 +382,7 @@ steps:
     key: "background-ddl-arrangement-backfill-recovery-test-deterministic"
     command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.8 USE_ARRANGEMENT_BACKFILL=--use-arrangement-backfill timeout 65m ci/scripts/deterministic-recovery-test.sh"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation"
-      || build.env("CI_STEPS") =~ /(^|,)recovery-tests?-deterministic-simulation(,|$$)/
-    depends_on: "build-simulation"
-    plugins:
-      - docker-compose#v5.1.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      # Only upload zipped files, otherwise the logs is too much.
-      - ./ci/plugins/upload-failure-logs-zipped
-    timeout_in_minutes: 70
-    retry: *auto-retry
-
-  # Ddl statements will randomly run with background_ddl.
-  - label: "background_ddl recovery test (madsim)"
-    key: "background-ddl-recovery-test-deterministic"
-    command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.8 timeout 65m ci/scripts/deterministic-recovery-test.sh"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)recovery-tests?-deterministic-simulation(,|$$)/
     depends_on: "build-simulation"
@@ -416,7 +400,7 @@ steps:
     key: "e2e-iceberg-sink-test"
     command: "ci/scripts/e2e-iceberg-sink-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-iceberg-sink-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-iceberg-sink-tests?(,|$$)/
     depends_on:
@@ -436,7 +420,7 @@ steps:
     key: "e2e-iceberg-sink-v2-test"
     command: "ci/scripts/e2e-iceberg-sink-v2-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-iceberg-sink-v2-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-iceberg-sink-v2-tests?(,|$$)/
     depends_on:
@@ -455,7 +439,7 @@ steps:
     key: "e2e-java-binding-tests"
     command: "ci/scripts/java-binding-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-java-binding-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-java-binding-tests?(,|$$)/
     depends_on:
@@ -476,7 +460,7 @@ steps:
     key: "s3-v2-source-check-aws-json-parser"
     command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_source_v2.py -t json"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-s3-source-tests"
       || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
     depends_on: build
@@ -498,7 +482,7 @@ steps:
     key: "s3-v2-source-batch-read-check-aws-json-parser"
     command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_source_batch.py -t json"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-s3-source-tests"
       || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
     depends_on: build
@@ -520,7 +504,7 @@ steps:
     key: "s3-v2-source-check-aws-csv-parser"
     command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_source_v2.py -t csv_without_header"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-s3-source-tests"
       || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
     depends_on: build
@@ -542,7 +526,7 @@ steps:
     key: "s3-source-test-for-opendal-fs-engine-csv-parser"
     command: "ci/scripts/s3-source-test.sh -p ci-release -s posix_fs_source.py -t csv_without_header"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-s3-source-tests"
       || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
     depends_on: build
@@ -560,7 +544,7 @@ steps:
   #   key: "s3-source-test-for-opendal-fs-engine"
   #   command: "ci/scripts/s3-source-test-for-opendal-fs-engine.sh -p ci-release -s gcs_source.py"
   #   if: |
-  #     !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+  #     !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
   #     || build.pull_request.labels includes "ci/run-s3-source-tests"
   #     || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
   #   depends_on: build
@@ -582,7 +566,7 @@ steps:
     key: "pulsar-source-tests"
     command: "ci/scripts/pulsar-source-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-pulsar-source-tests"
       || build.env("CI_STEPS") =~ /(^|,)pulsar-source-tests?(,|$$)/
     depends_on:
@@ -607,7 +591,7 @@ steps:
     key: "run-micro-benchmarks"
     command: "ci/scripts/run-micro-benchmarks.sh"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-micro-benchmarks"
       || build.env("CI_STEPS") =~ /(^|,)micro-benchmarks?(,|$$)/
     plugins:
@@ -622,7 +606,7 @@ steps:
     key: "upload-micro-benchmarks"
     if: |
       build.branch == "main"
-      || !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      || !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-micro-benchmarks"
       || build.env("CI_STEPS") =~ /(^|,)micro-benchmarks?(,|$$)/
     command:
@@ -647,14 +631,14 @@ steps:
     key: "backwards-compat-tests"
     command: "VERSION_OFFSET={{matrix.version_offset}} RW_COMMIT=$BUILDKITE_COMMIT ci/scripts/backwards-compat-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-backwards-compat-tests"
       || build.env("CI_STEPS") =~ /(^|,)backwards?-compat-tests?(,|$$)/
     depends_on:
       - "build"
     plugins:
       - docker-compose#v5.1.0:
-          run: rw-build-env
+          run: source-test-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
@@ -681,7 +665,7 @@ steps:
     key: "sqlsmith-differential-tests"
     command: "ci/scripts/sqlsmith-differential-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-sqlsmith-differential-tests"
       || build.env("CI_STEPS") =~ /(^|,)sqlsmith-differential-tests?(,|$$)/
     depends_on:
@@ -697,14 +681,14 @@ steps:
     key: "backfill-tests"
     command: "BUILDKITE=${BUILDKITE:-} ci/scripts/backfill-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-backfill-tests"
       || build.env("CI_STEPS") =~ /(^|,)backfill-tests?(,|$$)/
     depends_on:
       - "build"
     plugins:
       - docker-compose#v5.1.0:
-          run: rw-build-env
+          run: source-test-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
@@ -715,7 +699,7 @@ steps:
     key: "e2e-standalone-binary-tests"
     command: "ci/scripts/e2e-test.sh -p ci-release -m standalone"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-standalone-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-standalone-tests?(,|$$)/
     depends_on:
@@ -735,7 +719,7 @@ steps:
     key: "e2e-single-node-binary-tests"
     command: "ci/scripts/e2e-test.sh -p ci-release -m single-node"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-single-node-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-single-node-tests?(,|$$)/
     depends_on:
@@ -755,7 +739,7 @@ steps:
     key: "e2e-test-opendal-parallel"
     command: "ci/scripts/e2e-test-parallel-for-opendal.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-parallel-tests-for-opendal"
       || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-tests?-for-opendal(,|$$)/
     depends_on:
@@ -774,7 +758,7 @@ steps:
     key: "e2e-deltalake-sink-rust-tests"
     command: "ci/scripts/e2e-deltalake-sink-rust-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-deltalake-sink-rust-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-deltalake-sink-rust-tests?(,|$$)/
     depends_on:
@@ -793,7 +777,7 @@ steps:
     key: "e2e-redis-sink-tests"
     command: "ci/scripts/e2e-redis-sink-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-redis-sink-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-redis-sink-tests?(,|$$)/
     depends_on:
@@ -812,7 +796,7 @@ steps:
     key: "e2e-doris-sink-tests"
     command: "ci/scripts/e2e-doris-sink-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
     depends_on:
@@ -831,7 +815,7 @@ steps:
     key: "e2e-starrocks-sink-tests"
     command: "ci/scripts/e2e-starrocks-sink-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-starrocks-sink-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-starrocks-sink-tests?(,|$$)/
     depends_on:
@@ -850,7 +834,7 @@ steps:
     key: "e2e-cassandra-sink-tests"
     command: "ci/scripts/e2e-cassandra-sink-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-cassandra-sink-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-cassandra-sink-tests?(,|$$)/
     depends_on:
@@ -869,7 +853,7 @@ steps:
     key: "e2e-clickhouse-sink-tests"
     command: "ci/scripts/e2e-clickhouse-sink-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-clickhouse-sink-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-clickhouse-sink-tests?(,|$$)/
     depends_on:
@@ -884,11 +868,30 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
+  - label: "end-to-end sqlserver sink test"
+    key: "e2e-sqlserver-sink-tests"
+    command: "ci/scripts/e2e-sqlserver-sink-test.sh -p ci-release"
+    if: |
+      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-e2e-sqlserver-sink-tests"
+      || build.env("CI_STEPS") =~ /(^|,)e2e-sqlserver-sink-tests?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: sink-test-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 10
+    retry: *auto-retry
+
   - label: "end-to-end pulsar sink test"
     key: "e2e-pulsar-sink-tests"
     command: "ci/scripts/e2e-pulsar-sink-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-pulsar-sink-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-pulsar-sink-tests?(,|$$)/
     depends_on:
@@ -907,7 +910,7 @@ steps:
     key: "e2e-mqtt-sink-tests"
     command: "ci/scripts/e2e-mqtt-sink-test.sh -p ci-release"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-mqtt-sink-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-mqtt-sink-tests?(,|$$)/
     depends_on:
@@ -926,7 +929,7 @@ steps:
     key: "connector-node-integration-test"
     command: "ci/scripts/connector-node-integration-test.sh -p ci-release -v {{matrix.java_version}}"
     if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-connector-node-integration-tests"
       || build.env("CI_STEPS") =~ /(^|,)connector-node-integration-tests?(,|$$)/
     depends_on:
diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml
index c19b9a774fc12..cc5e670fe078a 100644
--- a/ci/workflows/pull-request.yml
+++ b/ci/workflows/pull-request.yml
@@ -1,10 +1,24 @@
 auto-retry: &auto-retry
-
   automatic:
     # Agent terminated because the AWS EC2 spot instance killed by AWS.
     - signal_reason: agent_stop
       limit: 3
 
+cargo-cache: &cargo-cache
+  id: cargo
+  key: "v1-cache-{{ id }}-{{ runner.os }}-{{ checksum 'Cargo.lock' }}"
+  restore-keys:
+    - "v1-cache-{{ id }}-{{ runner.os }}-"
+    - "v1-cache-{{ id }}-"
+  backend: s3
+  s3:
+    bucket: rw-ci-cache-bucket
+    args: '--no-progress'
+  paths:
+    - ".cargo/registry/index"
+    - ".cargo/registry/cache"
+    - ".cargo/git"
+
 steps:
   - label: "check ci image rebuild"
     plugins:
@@ -20,11 +34,8 @@ steps:
   - label: "build"
     command: "ci/scripts/build.sh -p ci-dev"
     key: "build"
-    if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-build"
-      || build.env("CI_STEPS") =~ /(^|,)build(,|$$)/
     plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v5.1.0:
           run: rw-build-env
           config: ci/docker-compose.yml
@@ -35,11 +46,8 @@ steps:
   - label: "build other components"
     command: "ci/scripts/build-other.sh"
     key: "build-other"
-    if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-build-other"
-      || build.env("CI_STEPS") =~ /(^|,)build-other(,|$$)/
     plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
       - seek-oss/aws-sm#v2.3.1:
           env:
             GITHUB_TOKEN: github-token
@@ -55,11 +63,8 @@ steps:
   - label: "build (deterministic simulation)"
     command: "ci/scripts/build-simulation.sh"
     key: "build-simulation"
-    if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-build-simulation"
-      || build.env("CI_STEPS") =~ /(^|,)build-simulation(,|$$)/
     plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v5.1.0:
           run: rw-build-env
           config: ci/docker-compose.yml
@@ -69,11 +74,8 @@ steps:
   - label: "docslt"
     command: "ci/scripts/docslt.sh"
     key: "docslt"
-    if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-docslt"
-      || build.env("CI_STEPS") =~ /(^|,)docslt(,|$$)/
     plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v5.1.0:
           run: rw-build-env
           config: ci/docker-compose.yml
@@ -84,7 +86,7 @@ steps:
   - label: "end-to-end test"
     command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-test"
       || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/
     depends_on:
@@ -97,13 +99,50 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 21
+    timeout_in_minutes: 23
+    retry: *auto-retry
+
+  - label: "slow end-to-end test"
+    key: "slow-e2e-test"
+    command: "ci/scripts/slow-e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe"
+    if: |
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-slow-e2e-tests"
+      || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 8
+    retry: *auto-retry
+
+  - label: "meta backup test"
+    key: "e2e-meta-backup-test"
+    command: "ci/scripts/run-meta-backup-test.sh -p ci-dev -m ci-3streaming-2serving-3fe"
+    if: |
+      build.pull_request.labels includes "ci/run-e2e-meta-backup-test"
+    depends_on:
+      - "build"
+      - "build-other"
+      - "docslt"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 45
     retry: *auto-retry
 
   - label: "end-to-end test (parallel)"
     command: "ci/scripts/e2e-test-parallel.sh -p ci-dev"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-parallel-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-tests?(,|$$)/
     depends_on:
@@ -115,7 +154,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 15
+    timeout_in_minutes: 17
     retry: *auto-retry
 
   - label: "end-to-end test for opendal (parallel)"
@@ -149,7 +188,7 @@ steps:
   - label: "end-to-end source test"
     command: "ci/scripts/e2e-source-test.sh -p ci-dev"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-source-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-source-tests?(,|$$)/
     depends_on:
@@ -160,6 +199,7 @@ steps:
           run: source-test-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
+          upload-container-logs: always
       - ./ci/plugins/upload-failure-logs
     timeout_in_minutes: 18
     retry: *auto-retry
@@ -167,7 +207,7 @@ steps:
   - label: "end-to-end sink test"
     command: "ci/scripts/e2e-sink-test.sh -p ci-dev"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-sink-tests"
       || build.env("CI_STEPS") =~ /(^|,)e2e-sink-tests?(,|$$)/
     depends_on:
@@ -293,6 +333,21 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
+  - label: "end-to-end sqlserver sink test"
+    if: build.pull_request.labels includes "ci/run-e2e-sqlserver-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-sqlserver-sink-tests?(,|$$)/
+    command: "ci/scripts/e2e-sqlserver-sink-test.sh -p ci-dev"
+    depends_on:
+      - "build"
+      - "build-other"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: sink-test-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 10
+    retry: *auto-retry
+
   - label: "end-to-end deltalake sink test"
     if: build.pull_request.labels includes "ci/run- e2e-deltalake-sink-rust-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-deltalake-sink-rust-tests?(,|$$)/
     command: "ci/scripts/e2e-deltalake-sink-rust-test.sh -p ci-dev"
@@ -386,7 +441,7 @@ steps:
   - label: "regress test"
     command: "ci/scripts/regress-test.sh -p ci-dev"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-regress-test"
       || build.env("CI_STEPS") =~ /(^|,)regress-tests?(,|$$)/
     depends_on: "build"
@@ -405,10 +460,11 @@ steps:
   - label: "unit test"
     command: "ci/scripts/pr-unit-test.sh"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-unit-test"
       || build.env("CI_STEPS") =~ /(^|,)unit-tests?(,|$$)/
     plugins:
+      - gencer/cache#v2.4.10: *cargo-cache
       - ./ci/plugins/swapfile
       - seek-oss/aws-sm#v2.3.1:
           env:
@@ -424,22 +480,11 @@ steps:
   - label: "check"
     command: "ci/scripts/check.sh"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-check"
       || build.env("CI_STEPS") =~ /(^|,)check(,|$$)/
     plugins:
-      - gencer/cache#v2.4.10:
-          id: cache
-          key: "v1-cache-{{ id }}-{{ runner.os }}-{{ checksum 'Cargo.lock' }}"
-          restore-keys:
-            - "v1-cache-{{ id }}-{{ runner.os }}-"
-            - "v1-cache-{{ id }}-"
-          backend: s3
-          s3:
-            bucket: rw-ci-cache-bucket
-            args: "--no-progress"
-          paths:
-            - ".cargo/advisory-db"
+      - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v5.1.0:
           run: rw-build-env
           config: ci/docker-compose.yml
@@ -449,22 +494,11 @@ steps:
   - label: "check dylint"
     command: "ci/scripts/check-dylint.sh"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-check"
       || build.env("CI_STEPS") =~ /(^|,)check(,|$$)/
     plugins:
-      - gencer/cache#v2.4.10:
-          id: cache
-          key: "v1-cache-{{ id }}-{{ runner.os }}-{{ checksum 'Cargo.lock' }}"
-          restore-keys:
-            - "v1-cache-{{ id }}-{{ runner.os }}-"
-            - "v1-cache-{{ id }}-"
-          backend: s3
-          s3:
-            bucket: rw-ci-cache-bucket
-            args: "--no-progress"
-          paths:
-            - ".cargo/advisory-db"
+      - gencer/cache#v2.4.10: *cargo-cache
       - docker-compose#v5.1.0:
           run: rw-build-env
           config: ci/docker-compose.yml
@@ -474,7 +508,7 @@ steps:
   - label: "unit test (deterministic simulation)"
     command: "ci/scripts/deterministic-unit-test.sh"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-unit-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)unit-tests?-deterministic-simulation(,|$$)/
     plugins:
@@ -489,7 +523,7 @@ steps:
   - label: "integration test (deterministic simulation)"
     command: "TEST_NUM=5 ci/scripts/deterministic-it-test.sh"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/
     depends_on: "build-simulation"
@@ -505,7 +539,7 @@ steps:
   - label: "end-to-end test (deterministic simulation)"
     command: "TEST_NUM=16 ci/scripts/deterministic-e2e-test.sh"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)e2e-tests?-deterministic-simulation(,|$$)/
     depends_on: "build-simulation"
@@ -527,7 +561,7 @@ steps:
   - label: "recovery test (deterministic simulation)"
     command: "TEST_NUM=8 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.0 ci/scripts/deterministic-recovery-test.sh"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation"
       || build.env("CI_STEPS") =~ /(^|,)recovery-tests?-deterministic-simulation(,|$$)/
     depends_on: "build-simulation"
@@ -551,7 +585,7 @@ steps:
   - label: "misc check"
     command: "ci/scripts/misc-check.sh"
     if: |
-      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-misc-check"
       || build.env("CI_STEPS") =~ /(^|,)misc-check(,|$$)/
     plugins:
@@ -642,7 +676,7 @@ steps:
       - "build"
     plugins:
       - docker-compose#v5.1.0:
-          run: rw-build-env
+          run: source-test-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
@@ -683,7 +717,7 @@ steps:
       - "build"
     plugins:
       - docker-compose#v5.1.0:
-          run: rw-build-env
+          run: source-test-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
@@ -769,10 +803,10 @@ steps:
     timeout_in_minutes: 15
     retry: *auto-retry
 
-  - label: "enable ci/skip-ci only in draft PRs"
-    if: build.pull_request.labels includes "ci/skip-ci" && !build.pull_request.draft
+  - label: "enable ci/pr/run-selected only in draft PRs"
+    if: build.pull_request.labels includes "ci/pr/run-selected" && !build.pull_request.draft
     commands:
-      - echo "ci/skip-ci is only usable for draft Pull Requests"
+      - echo "ci/pr/run-selected is only usable for draft Pull Requests"
       - exit 1
 
   - label: "micro benchmark"
diff --git a/dashboard/components/Layout.tsx b/dashboard/components/Layout.tsx
index 7be9f727bddec..6daa7e821ce3c 100644
--- a/dashboard/components/Layout.tsx
+++ b/dashboard/components/Layout.tsx
@@ -140,6 +140,7 @@ function Layout({ children }: { children: React.ReactNode }) {
             <NavButton href="/internal_tables/">Internal Tables</NavButton>
             <NavButton href="/sinks/">Sinks</NavButton>
             <NavButton href="/views/">Views</NavButton>
+            <NavButton href="/subscriptions/">Subscriptions</NavButton>
           </Section>
           <Section>
             <NavTitle>Streaming</NavTitle>
diff --git a/dashboard/components/Relations.tsx b/dashboard/components/Relations.tsx
index 6d04d041b368f..49161feea908b 100644
--- a/dashboard/components/Relations.tsx
+++ b/dashboard/components/Relations.tsx
@@ -33,7 +33,13 @@ import Link from "next/link"
 import { Fragment } from "react"
 import Title from "../components/Title"
 import useFetch from "../lib/api/fetch"
-import { Relation, StreamingJob } from "../lib/api/streaming"
+import {
+  Relation,
+  StreamingJob,
+  getDatabases,
+  getSchemas,
+  getUsers,
+} from "../lib/api/streaming"
 import extractColumnInfo from "../lib/extractInfo"
 import {
   Sink as RwSink,
@@ -114,7 +120,22 @@ export function Relations<R extends Relation>(
   getRelations: () => Promise<R[]>,
   extraColumns: Column<R>[]
 ) {
-  const { response: relationList } = useFetch(getRelations)
+  const { response: relationList } = useFetch(async () => {
+    const relations = await getRelations()
+    const users = await getUsers()
+    const databases = await getDatabases()
+    const schemas = await getSchemas()
+    return relations.map((r) => {
+      // Add owner, schema, and database names. It's linear search but the list is small.
+      const owner = users.find((u) => u.id === r.owner)
+      const ownerName = owner?.name
+      const schema = schemas.find((s) => s.id === r.schemaId)
+      const schemaName = schema?.name
+      const database = databases.find((d) => d.id === r.databaseId)
+      const databaseName = database?.name
+      return { ...r, ownerName, schemaName, databaseName }
+    })
+  })
   const [modalData, setModalId] = useCatalogModal(relationList)
 
   const modal = (
@@ -129,6 +150,8 @@ export function Relations<R extends Relation>(
           <Thead>
             <Tr>
               <Th width={3}>Id</Th>
+              <Th width={5}>Database</Th>
+              <Th width={5}>Schema</Th>
               <Th width={5}>Name</Th>
               <Th width={3}>Owner</Th>
               {extraColumns.map((c) => (
@@ -153,17 +176,23 @@ export function Relations<R extends Relation>(
                     {r.id}
                   </Button>
                 </Td>
+                <Td>{r.databaseName}</Td>
+                <Td>{r.schemaName}</Td>
                 <Td>{r.name}</Td>
-                <Td>{r.owner}</Td>
+                <Td>{r.ownerName}</Td>
                 {extraColumns.map((c) => (
                   <Td key={c.name}>{c.content(r)}</Td>
                 ))}
-                <Td overflowWrap="normal">
-                  {r.columns
-                    .filter((col) => ("isHidden" in col ? !col.isHidden : true))
-                    .map((col) => extractColumnInfo(col))
-                    .join(", ")}
-                </Td>
+                {r.columns && r.columns.length > 0 && (
+                  <Td overflowWrap="normal">
+                    {r.columns
+                      .filter((col) =>
+                        "isHidden" in col ? !col.isHidden : true
+                      )
+                      .map((col) => extractColumnInfo(col))
+                      .join(", ")}
+                  </Td>
+                )}
               </Tr>
             ))}
           </Tbody>
diff --git a/dashboard/lib/api/api.ts b/dashboard/lib/api/api.ts
index 0df0454295482..af0116e31f3bb 100644
--- a/dashboard/lib/api/api.ts
+++ b/dashboard/lib/api/api.ts
@@ -26,7 +26,7 @@ export const PREDEFINED_API_ENDPOINTS = [
 ]
 
 export const DEFAULT_API_ENDPOINT: string =
-  process.env.NODE_ENV === "production" ? PROD_API_ENDPOINT : MOCK_API_ENDPOINT
+  process.env.NODE_ENV === "production" ? PROD_API_ENDPOINT : MOCK_API_ENDPOINT // EXTERNAL_META_NODE_API_ENDPOINT to debug with RisingWave servers
 
 export const API_ENDPOINT_KEY = "risingwave.dashboard.api.endpoint"
 
diff --git a/dashboard/lib/api/fetch.ts b/dashboard/lib/api/fetch.ts
index 7aa34826e6a5b..6cf980202b548 100644
--- a/dashboard/lib/api/fetch.ts
+++ b/dashboard/lib/api/fetch.ts
@@ -33,6 +33,7 @@ export default function useFetch<T>(
   const [response, setResponse] = useState<T>()
   const toast = useErrorToast()
 
+  // NOTE(eric): Don't put `fetchFn` in the dependency array. It might be a lambda function
   useEffect(() => {
     const fetchData = async () => {
       if (when) {
@@ -52,7 +53,7 @@ export default function useFetch<T>(
 
     const timer = setInterval(fetchData, intervalMs)
     return () => clearInterval(timer)
-  }, [toast, fetchFn, intervalMs, when])
+  }, [toast, intervalMs, when])
 
   return { response }
 }
diff --git a/dashboard/lib/api/streaming.ts b/dashboard/lib/api/streaming.ts
index 4295e639fb218..948cd567d3f2b 100644
--- a/dashboard/lib/api/streaming.ts
+++ b/dashboard/lib/api/streaming.ts
@@ -17,12 +17,21 @@
 
 import _ from "lodash"
 import sortBy from "lodash/sortBy"
-import { Sink, Source, Table, View } from "../../proto/gen/catalog"
+import {
+  Database,
+  Schema,
+  Sink,
+  Source,
+  Subscription,
+  Table,
+  View,
+} from "../../proto/gen/catalog"
 import {
   ListObjectDependenciesResponse_ObjectDependencies as ObjectDependencies,
   TableFragments,
 } from "../../proto/gen/meta"
 import { ColumnCatalog, Field } from "../../proto/gen/plan_common"
+import { UserInfo } from "../../proto/gen/user"
 import api from "./api"
 
 export async function getFragments(): Promise<TableFragments[]> {
@@ -37,7 +46,14 @@ export interface Relation {
   id: number
   name: string
   owner: number
-  columns: (ColumnCatalog | Field)[]
+  schemaId: number
+  databaseId: number
+
+  // For display
+  columns?: (ColumnCatalog | Field)[]
+  ownerName?: string
+  schemaName?: string
+  databaseName?: string
 }
 
 export interface StreamingJob extends Relation {
@@ -51,6 +67,8 @@ export function relationType(x: Relation) {
     return "SINK"
   } else if ((x as Source).info !== undefined) {
     return "SOURCE"
+  } else if ((x as Subscription).dependentTableId !== undefined) {
+    return "SUBSCRIPTION"
   } else {
     return "UNKNOWN"
   }
@@ -83,7 +101,8 @@ export async function getRelations() {
     await getTables(),
     await getIndexes(),
     await getSinks(),
-    await getSources()
+    await getSources(),
+    await getSubscriptions()
   )
   relations = sortBy(relations, (x) => x.id)
   return relations
@@ -135,6 +154,34 @@ export async function getViews() {
   return views
 }
 
+export async function getSubscriptions() {
+  let subscriptions: Subscription[] = (await api.get("/subscriptions")).map(
+    Subscription.fromJSON
+  )
+  subscriptions = sortBy(subscriptions, (x) => x.id)
+  return subscriptions
+}
+
+export async function getUsers() {
+  let users: UserInfo[] = (await api.get("/users")).map(UserInfo.fromJSON)
+  users = sortBy(users, (x) => x.id)
+  return users
+}
+
+export async function getDatabases() {
+  let databases: Database[] = (await api.get("/databases")).map(
+    Database.fromJSON
+  )
+  databases = sortBy(databases, (x) => x.id)
+  return databases
+}
+
+export async function getSchemas() {
+  let schemas: Schema[] = (await api.get("/schemas")).map(Schema.fromJSON)
+  schemas = sortBy(schemas, (x) => x.id)
+  return schemas
+}
+
 export async function getObjectDependencies() {
   let objDependencies: ObjectDependencies[] = (
     await api.get("/object_dependencies")
diff --git a/dashboard/mock-server.js b/dashboard/mock-server.js
index 2db52df788e22..50c55e12686b8 100644
--- a/dashboard/mock-server.js
+++ b/dashboard/mock-server.js
@@ -45,6 +45,10 @@ app.get("/indexes", (req, res, next) => {
   res.json(require("./mock/indexes.json"))
 })
 
+app.get("/indexes", (req, res, next) => {
+  res.json(require("./mock/indexes.json"))
+})
+
 app.get("/internal_tables", (req, res, next) => {
   res.json(require("./mock/internal_tables.json"))
 })
diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json
index 1c462e1675207..857888e4d4cfe 100644
--- a/dashboard/package-lock.json
+++ b/dashboard/package-lock.json
@@ -6,6 +6,7 @@
     "": {
       "hasInstallScript": true,
       "dependencies": {
+        "16": "^0.0.2",
         "@chakra-ui/react": "^2.3.1",
         "@emotion/react": "^11.10.4",
         "@emotion/styled": "^11.10.4",
@@ -26,7 +27,7 @@
         "fabric": "^5.2.1",
         "framer-motion": "^6.5.1",
         "lodash": "^4.17.21",
-        "next": "^14.1.0",
+        "next": "^14.1.1",
         "nuqs": "^1.14.1",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
@@ -2092,9 +2093,9 @@
       }
     },
     "node_modules/@next/env": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.1.0.tgz",
-      "integrity": "sha512-Py8zIo+02ht82brwwhTg36iogzFqGLPXlRGKQw5s+qP/kMNc4MAyDeEwBKDijk6zTIbegEgu8Qy7C1LboslQAw=="
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.1.1.tgz",
+      "integrity": "sha512-7CnQyD5G8shHxQIIg3c7/pSeYFeMhsNbpU/bmvH7ZnDql7mNRgg8O2JZrhrc/soFnfBnKP4/xXNiiSIPn2w8gA=="
     },
     "node_modules/@next/eslint-plugin-next": {
       "version": "14.1.0",
@@ -2161,9 +2162,9 @@
       }
     },
     "node_modules/@next/swc-darwin-arm64": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.0.tgz",
-      "integrity": "sha512-nUDn7TOGcIeyQni6lZHfzNoo9S0euXnu0jhsbMOmMJUBfgsnESdjN97kM7cBqQxZa8L/bM9om/S5/1dzCrW6wQ==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1.tgz",
+      "integrity": "sha512-yDjSFKQKTIjyT7cFv+DqQfW5jsD+tVxXTckSe1KIouKk75t1qZmj/mV3wzdmFb0XHVGtyRjDMulfVG8uCKemOQ==",
       "cpu": [
         "arm64"
       ],
@@ -2176,9 +2177,9 @@
       }
     },
     "node_modules/@next/swc-darwin-x64": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.0.tgz",
-      "integrity": "sha512-1jgudN5haWxiAl3O1ljUS2GfupPmcftu2RYJqZiMJmmbBT5M1XDffjUtRUzP4W3cBHsrvkfOFdQ71hAreNQP6g==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1.tgz",
+      "integrity": "sha512-KCQmBL0CmFmN8D64FHIZVD9I4ugQsDBBEJKiblXGgwn7wBCSe8N4Dx47sdzl4JAg39IkSN5NNrr8AniXLMb3aw==",
       "cpu": [
         "x64"
       ],
@@ -2191,9 +2192,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-gnu": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.0.tgz",
-      "integrity": "sha512-RHo7Tcj+jllXUbK7xk2NyIDod3YcCPDZxj1WLIYxd709BQ7WuRYl3OWUNG+WUfqeQBds6kvZYlc42NJJTNi4tQ==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1.tgz",
+      "integrity": "sha512-YDQfbWyW0JMKhJf/T4eyFr4b3tceTorQ5w2n7I0mNVTFOvu6CGEzfwT3RSAQGTi/FFMTFcuspPec/7dFHuP7Eg==",
       "cpu": [
         "arm64"
       ],
@@ -2206,9 +2207,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-musl": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.0.tgz",
-      "integrity": "sha512-v6kP8sHYxjO8RwHmWMJSq7VZP2nYCkRVQ0qolh2l6xroe9QjbgV8siTbduED4u0hlk0+tjS6/Tuy4n5XCp+l6g==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1.tgz",
+      "integrity": "sha512-fiuN/OG6sNGRN/bRFxRvV5LyzLB8gaL8cbDH5o3mEiVwfcMzyE5T//ilMmaTrnA8HLMS6hoz4cHOu6Qcp9vxgQ==",
       "cpu": [
         "arm64"
       ],
@@ -2221,9 +2222,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-gnu": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.0.tgz",
-      "integrity": "sha512-zJ2pnoFYB1F4vmEVlb/eSe+VH679zT1VdXlZKX+pE66grOgjmKJHKacf82g/sWE4MQ4Rk2FMBCRnX+l6/TVYzQ==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1.tgz",
+      "integrity": "sha512-rv6AAdEXoezjbdfp3ouMuVqeLjE1Bin0AuE6qxE6V9g3Giz5/R3xpocHoAi7CufRR+lnkuUjRBn05SYJ83oKNQ==",
       "cpu": [
         "x64"
       ],
@@ -2236,9 +2237,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-musl": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.0.tgz",
-      "integrity": "sha512-rbaIYFt2X9YZBSbH/CwGAjbBG2/MrACCVu2X0+kSykHzHnYH5FjHxwXLkcoJ10cX0aWCEynpu+rP76x0914atg==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1.tgz",
+      "integrity": "sha512-YAZLGsaNeChSrpz/G7MxO3TIBLaMN8QWMr3X8bt6rCvKovwU7GqQlDu99WdvF33kI8ZahvcdbFsy4jAFzFX7og==",
       "cpu": [
         "x64"
       ],
@@ -2251,9 +2252,9 @@
       }
     },
     "node_modules/@next/swc-win32-arm64-msvc": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.0.tgz",
-      "integrity": "sha512-o1N5TsYc8f/HpGt39OUQpQ9AKIGApd3QLueu7hXk//2xq5Z9OxmV6sQfNp8C7qYmiOlHYODOGqNNa0e9jvchGQ==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1.tgz",
+      "integrity": "sha512-1L4mUYPBMvVDMZg1inUYyPvFSduot0g73hgfD9CODgbr4xiTYe0VOMTZzaRqYJYBA9mana0x4eaAaypmWo1r5A==",
       "cpu": [
         "arm64"
       ],
@@ -2266,9 +2267,9 @@
       }
     },
     "node_modules/@next/swc-win32-ia32-msvc": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.0.tgz",
-      "integrity": "sha512-XXIuB1DBRCFwNO6EEzCTMHT5pauwaSj4SWs7CYnME57eaReAKBXCnkUE80p/pAZcewm7hs+vGvNqDPacEXHVkw==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1.tgz",
+      "integrity": "sha512-jvIE9tsuj9vpbbXlR5YxrghRfMuG0Qm/nZ/1KDHc+y6FpnZ/apsgh+G6t15vefU0zp3WSpTMIdXRUsNl/7RSuw==",
       "cpu": [
         "ia32"
       ],
@@ -2281,9 +2282,9 @@
       }
     },
     "node_modules/@next/swc-win32-x64-msvc": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.0.tgz",
-      "integrity": "sha512-9WEbVRRAqJ3YFVqEZIxUqkiO8l1nool1LmNxygr5HWF8AcSYsEpneUDhmjUVJEzO2A04+oPtZdombzzPPkTtgg==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1.tgz",
+      "integrity": "sha512-S6K6EHDU5+1KrBDLko7/c1MNy/Ya73pIAmvKeFwsF4RmBFJSO7/7YeD4FnZ4iBdzE69PpQ4sOMU9ORKeNuxe8A==",
       "cpu": [
         "x64"
       ],
@@ -3350,6 +3351,14 @@
       "resolved": "https://registry.npmjs.org/@zag-js/focus-visible/-/focus-visible-0.1.0.tgz",
       "integrity": "sha512-PeaBcTmdZWcFf7n1aM+oiOdZc+sy14qi0emPIeUuGMTjbP0xLGrZu43kdpHnWSXy7/r4Ubp/vlg50MCV8+9Isg=="
     },
+    "node_modules/16": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/16/-/16-0.0.2.tgz",
+      "integrity": "sha512-AhG4lpdn+/it+U5Xl1bm5SbaHYTH5NfU/vXZkP7E7CHjtVtITuFVZKa3AZP3gN38RDJHYYtEqWmqzCutlXaR7w==",
+      "dependencies": {
+        "numeric": "^1.2.6"
+      }
+    },
     "node_modules/abab": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
@@ -3861,12 +3870,12 @@
       }
     },
     "node_modules/braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
       "dev": true,
       "dependencies": {
-        "fill-range": "^7.0.1"
+        "fill-range": "^7.1.1"
       },
       "engines": {
         "node": ">=8"
@@ -6316,9 +6325,9 @@
       }
     },
     "node_modules/fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
       "dev": true,
       "dependencies": {
         "to-regex-range": "^5.0.1"
@@ -8538,11 +8547,11 @@
       }
     },
     "node_modules/next": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/next/-/next-14.1.0.tgz",
-      "integrity": "sha512-wlzrsbfeSU48YQBjZhDzOwhWhGsy+uQycR8bHAOt1LY1bn3zZEcDyHQOEoN3aWzQ8LHCAJ1nqrWCc9XF2+O45Q==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/next/-/next-14.1.1.tgz",
+      "integrity": "sha512-McrGJqlGSHeaz2yTRPkEucxQKe5Zq7uPwyeHNmJaZNY4wx9E9QdxmTp310agFRoMuIYgQrCrT3petg13fSVOww==",
       "dependencies": {
-        "@next/env": "14.1.0",
+        "@next/env": "14.1.1",
         "@swc/helpers": "0.5.2",
         "busboy": "1.6.0",
         "caniuse-lite": "^1.0.30001579",
@@ -8557,15 +8566,15 @@
         "node": ">=18.17.0"
       },
       "optionalDependencies": {
-        "@next/swc-darwin-arm64": "14.1.0",
-        "@next/swc-darwin-x64": "14.1.0",
-        "@next/swc-linux-arm64-gnu": "14.1.0",
-        "@next/swc-linux-arm64-musl": "14.1.0",
-        "@next/swc-linux-x64-gnu": "14.1.0",
-        "@next/swc-linux-x64-musl": "14.1.0",
-        "@next/swc-win32-arm64-msvc": "14.1.0",
-        "@next/swc-win32-ia32-msvc": "14.1.0",
-        "@next/swc-win32-x64-msvc": "14.1.0"
+        "@next/swc-darwin-arm64": "14.1.1",
+        "@next/swc-darwin-x64": "14.1.1",
+        "@next/swc-linux-arm64-gnu": "14.1.1",
+        "@next/swc-linux-arm64-musl": "14.1.1",
+        "@next/swc-linux-x64-gnu": "14.1.1",
+        "@next/swc-linux-x64-musl": "14.1.1",
+        "@next/swc-win32-arm64-msvc": "14.1.1",
+        "@next/swc-win32-ia32-msvc": "14.1.1",
+        "@next/swc-win32-x64-msvc": "14.1.1"
       },
       "peerDependencies": {
         "@opentelemetry/api": "^1.1.0",
@@ -8680,6 +8689,11 @@
         "set-blocking": "^2.0.0"
       }
     },
+    "node_modules/numeric": {
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/numeric/-/numeric-1.2.6.tgz",
+      "integrity": "sha512-avBiDAP8siMa7AfJgYyuxw1oyII4z2sswS23+O+ZfV28KrtNzy0wxUFwi4f3RyM4eeeXNs1CThxR7pb5QQcMiw=="
+    },
     "node_modules/nuqs": {
       "version": "1.14.1",
       "resolved": "https://registry.npmjs.org/nuqs/-/nuqs-1.14.1.tgz",
@@ -11586,6 +11600,14 @@
     }
   },
   "dependencies": {
+    "16": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/16/-/16-0.0.2.tgz",
+      "integrity": "sha512-AhG4lpdn+/it+U5Xl1bm5SbaHYTH5NfU/vXZkP7E7CHjtVtITuFVZKa3AZP3gN38RDJHYYtEqWmqzCutlXaR7w==",
+      "requires": {
+        "numeric": "^1.2.6"
+      }
+    },
     "@aashutoshrathi/word-wrap": {
       "version": "1.2.6",
       "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz",
@@ -13123,9 +13145,9 @@
       }
     },
     "@next/env": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.1.0.tgz",
-      "integrity": "sha512-Py8zIo+02ht82brwwhTg36iogzFqGLPXlRGKQw5s+qP/kMNc4MAyDeEwBKDijk6zTIbegEgu8Qy7C1LboslQAw=="
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.1.1.tgz",
+      "integrity": "sha512-7CnQyD5G8shHxQIIg3c7/pSeYFeMhsNbpU/bmvH7ZnDql7mNRgg8O2JZrhrc/soFnfBnKP4/xXNiiSIPn2w8gA=="
     },
     "@next/eslint-plugin-next": {
       "version": "14.1.0",
@@ -13176,57 +13198,57 @@
       }
     },
     "@next/swc-darwin-arm64": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.0.tgz",
-      "integrity": "sha512-nUDn7TOGcIeyQni6lZHfzNoo9S0euXnu0jhsbMOmMJUBfgsnESdjN97kM7cBqQxZa8L/bM9om/S5/1dzCrW6wQ==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1.tgz",
+      "integrity": "sha512-yDjSFKQKTIjyT7cFv+DqQfW5jsD+tVxXTckSe1KIouKk75t1qZmj/mV3wzdmFb0XHVGtyRjDMulfVG8uCKemOQ==",
       "optional": true
     },
     "@next/swc-darwin-x64": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.0.tgz",
-      "integrity": "sha512-1jgudN5haWxiAl3O1ljUS2GfupPmcftu2RYJqZiMJmmbBT5M1XDffjUtRUzP4W3cBHsrvkfOFdQ71hAreNQP6g==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1.tgz",
+      "integrity": "sha512-KCQmBL0CmFmN8D64FHIZVD9I4ugQsDBBEJKiblXGgwn7wBCSe8N4Dx47sdzl4JAg39IkSN5NNrr8AniXLMb3aw==",
       "optional": true
     },
     "@next/swc-linux-arm64-gnu": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.0.tgz",
-      "integrity": "sha512-RHo7Tcj+jllXUbK7xk2NyIDod3YcCPDZxj1WLIYxd709BQ7WuRYl3OWUNG+WUfqeQBds6kvZYlc42NJJTNi4tQ==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1.tgz",
+      "integrity": "sha512-YDQfbWyW0JMKhJf/T4eyFr4b3tceTorQ5w2n7I0mNVTFOvu6CGEzfwT3RSAQGTi/FFMTFcuspPec/7dFHuP7Eg==",
       "optional": true
     },
     "@next/swc-linux-arm64-musl": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.0.tgz",
-      "integrity": "sha512-v6kP8sHYxjO8RwHmWMJSq7VZP2nYCkRVQ0qolh2l6xroe9QjbgV8siTbduED4u0hlk0+tjS6/Tuy4n5XCp+l6g==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1.tgz",
+      "integrity": "sha512-fiuN/OG6sNGRN/bRFxRvV5LyzLB8gaL8cbDH5o3mEiVwfcMzyE5T//ilMmaTrnA8HLMS6hoz4cHOu6Qcp9vxgQ==",
       "optional": true
     },
     "@next/swc-linux-x64-gnu": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.0.tgz",
-      "integrity": "sha512-zJ2pnoFYB1F4vmEVlb/eSe+VH679zT1VdXlZKX+pE66grOgjmKJHKacf82g/sWE4MQ4Rk2FMBCRnX+l6/TVYzQ==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1.tgz",
+      "integrity": "sha512-rv6AAdEXoezjbdfp3ouMuVqeLjE1Bin0AuE6qxE6V9g3Giz5/R3xpocHoAi7CufRR+lnkuUjRBn05SYJ83oKNQ==",
       "optional": true
     },
     "@next/swc-linux-x64-musl": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.0.tgz",
-      "integrity": "sha512-rbaIYFt2X9YZBSbH/CwGAjbBG2/MrACCVu2X0+kSykHzHnYH5FjHxwXLkcoJ10cX0aWCEynpu+rP76x0914atg==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1.tgz",
+      "integrity": "sha512-YAZLGsaNeChSrpz/G7MxO3TIBLaMN8QWMr3X8bt6rCvKovwU7GqQlDu99WdvF33kI8ZahvcdbFsy4jAFzFX7og==",
       "optional": true
     },
     "@next/swc-win32-arm64-msvc": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.0.tgz",
-      "integrity": "sha512-o1N5TsYc8f/HpGt39OUQpQ9AKIGApd3QLueu7hXk//2xq5Z9OxmV6sQfNp8C7qYmiOlHYODOGqNNa0e9jvchGQ==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1.tgz",
+      "integrity": "sha512-1L4mUYPBMvVDMZg1inUYyPvFSduot0g73hgfD9CODgbr4xiTYe0VOMTZzaRqYJYBA9mana0x4eaAaypmWo1r5A==",
       "optional": true
     },
     "@next/swc-win32-ia32-msvc": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.0.tgz",
-      "integrity": "sha512-XXIuB1DBRCFwNO6EEzCTMHT5pauwaSj4SWs7CYnME57eaReAKBXCnkUE80p/pAZcewm7hs+vGvNqDPacEXHVkw==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1.tgz",
+      "integrity": "sha512-jvIE9tsuj9vpbbXlR5YxrghRfMuG0Qm/nZ/1KDHc+y6FpnZ/apsgh+G6t15vefU0zp3WSpTMIdXRUsNl/7RSuw==",
       "optional": true
     },
     "@next/swc-win32-x64-msvc": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.0.tgz",
-      "integrity": "sha512-9WEbVRRAqJ3YFVqEZIxUqkiO8l1nool1LmNxygr5HWF8AcSYsEpneUDhmjUVJEzO2A04+oPtZdombzzPPkTtgg==",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1.tgz",
+      "integrity": "sha512-S6K6EHDU5+1KrBDLko7/c1MNy/Ya73pIAmvKeFwsF4RmBFJSO7/7YeD4FnZ4iBdzE69PpQ4sOMU9ORKeNuxe8A==",
       "optional": true
     },
     "@nodelib/fs.scandir": {
@@ -14494,12 +14516,12 @@
       }
     },
     "braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
       "dev": true,
       "requires": {
-        "fill-range": "^7.0.1"
+        "fill-range": "^7.1.1"
       }
     },
     "browser-process-hrtime": {
@@ -16328,9 +16350,9 @@
       }
     },
     "fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
       "dev": true,
       "requires": {
         "to-regex-range": "^5.0.1"
@@ -17939,20 +17961,20 @@
       "dev": true
     },
     "next": {
-      "version": "14.1.0",
-      "resolved": "https://registry.npmjs.org/next/-/next-14.1.0.tgz",
-      "integrity": "sha512-wlzrsbfeSU48YQBjZhDzOwhWhGsy+uQycR8bHAOt1LY1bn3zZEcDyHQOEoN3aWzQ8LHCAJ1nqrWCc9XF2+O45Q==",
-      "requires": {
-        "@next/env": "14.1.0",
-        "@next/swc-darwin-arm64": "14.1.0",
-        "@next/swc-darwin-x64": "14.1.0",
-        "@next/swc-linux-arm64-gnu": "14.1.0",
-        "@next/swc-linux-arm64-musl": "14.1.0",
-        "@next/swc-linux-x64-gnu": "14.1.0",
-        "@next/swc-linux-x64-musl": "14.1.0",
-        "@next/swc-win32-arm64-msvc": "14.1.0",
-        "@next/swc-win32-ia32-msvc": "14.1.0",
-        "@next/swc-win32-x64-msvc": "14.1.0",
+      "version": "14.1.1",
+      "resolved": "https://registry.npmjs.org/next/-/next-14.1.1.tgz",
+      "integrity": "sha512-McrGJqlGSHeaz2yTRPkEucxQKe5Zq7uPwyeHNmJaZNY4wx9E9QdxmTp310agFRoMuIYgQrCrT3petg13fSVOww==",
+      "requires": {
+        "@next/env": "14.1.1",
+        "@next/swc-darwin-arm64": "14.1.1",
+        "@next/swc-darwin-x64": "14.1.1",
+        "@next/swc-linux-arm64-gnu": "14.1.1",
+        "@next/swc-linux-arm64-musl": "14.1.1",
+        "@next/swc-linux-x64-gnu": "14.1.1",
+        "@next/swc-linux-x64-musl": "14.1.1",
+        "@next/swc-win32-arm64-msvc": "14.1.1",
+        "@next/swc-win32-ia32-msvc": "14.1.1",
+        "@next/swc-win32-x64-msvc": "14.1.1",
         "@swc/helpers": "0.5.2",
         "busboy": "1.6.0",
         "caniuse-lite": "^1.0.30001579",
@@ -18034,6 +18056,11 @@
         "set-blocking": "^2.0.0"
       }
     },
+    "numeric": {
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/numeric/-/numeric-1.2.6.tgz",
+      "integrity": "sha512-avBiDAP8siMa7AfJgYyuxw1oyII4z2sswS23+O+ZfV28KrtNzy0wxUFwi4f3RyM4eeeXNs1CThxR7pb5QQcMiw=="
+    },
     "nuqs": {
       "version": "1.14.1",
       "resolved": "https://registry.npmjs.org/nuqs/-/nuqs-1.14.1.tgz",
diff --git a/dashboard/package.json b/dashboard/package.json
index 1e84bcb8abb2d..a3716f7802ccf 100644
--- a/dashboard/package.json
+++ b/dashboard/package.json
@@ -13,6 +13,7 @@
     "clean": "rm -rf .next/ && rm -rf out/"
   },
   "dependencies": {
+    "16": "^0.0.2",
     "@chakra-ui/react": "^2.3.1",
     "@emotion/react": "^11.10.4",
     "@emotion/styled": "^11.10.4",
@@ -33,7 +34,7 @@
     "fabric": "^5.2.1",
     "framer-motion": "^6.5.1",
     "lodash": "^4.17.21",
-    "next": "^14.1.0",
+    "next": "^14.1.1",
     "nuqs": "^1.14.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
diff --git a/dashboard/pages/await_tree.tsx b/dashboard/pages/await_tree.tsx
index 2fb03afa3fc46..6babcccf7e91a 100644
--- a/dashboard/pages/await_tree.tsx
+++ b/dashboard/pages/await_tree.tsx
@@ -86,8 +86,16 @@ export default function AwaitTreeDump() {
         .entries()
         .map(([k, v]) => `[Barrier ${k}]\n${v}`)
         .join("\n")
+      const barrierWorkerState = _(response.barrierWorkerState)
+        .entries()
+        .map(([k, v]) => `[BarrierWorkerState (Worker ${k})]\n${v}`)
+        .join("\n")
+      const jvmStackTraces = _(response.jvmStackTraces)
+        .entries()
+        .map(([k, v]) => `[JVM (Worker ${k})]\n${v}`)
+        .join("\n")
 
-      result = `${title}\n\n${actorTraces}\n${rpcTraces}\n${compactionTraces}\n${barrierTraces}`
+      result = `${title}\n\n${actorTraces}\n${rpcTraces}\n${compactionTraces}\n${barrierTraces}\n${barrierWorkerState}\n\n${jvmStackTraces}`
     } catch (e: any) {
       result = `${title}\n\nERROR: ${e.message}\n${e.cause}`
     }
diff --git a/dashboard/pages/subscriptions.tsx b/dashboard/pages/subscriptions.tsx
new file mode 100644
index 0000000000000..b2daa38c3f95c
--- /dev/null
+++ b/dashboard/pages/subscriptions.tsx
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2024 RisingWave Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Column, Relations } from "../components/Relations"
+import { getSubscriptions } from "../lib/api/streaming"
+import { Subscription as RwSubscription } from "../proto/gen/catalog"
+
+export default function Subscriptions() {
+  const subscriptionRetentionSeconds: Column<RwSubscription> = {
+    name: "Retention Seconds",
+    width: 3,
+    content: (r) => r.retentionSeconds ?? "unknown",
+  }
+
+  const subscriptionDependentTableId: Column<RwSubscription> = {
+    name: "Dependent Table Id",
+    width: 3,
+    content: (r) => r.dependentTableId ?? "unknown",
+  }
+  return Relations("Subscriptions", getSubscriptions, [
+    subscriptionRetentionSeconds,
+    subscriptionDependentTableId,
+  ])
+}
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 3d902bc2ffc6c..4cd5c5157dc42 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,19 +1,13 @@
-FROM ubuntu:22.04 AS base
+FROM ubuntu:24.04 AS base
 
 ENV LANG en_US.utf8
 
 RUN apt-get update \
-  && apt-get -y install ca-certificates build-essential libsasl2-dev openjdk-11-jdk software-properties-common
-
-# Install Python 3.12
-RUN add-apt-repository ppa:deadsnakes/ppa -y && \
-    apt-get update -yy && \
-    DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy
-ENV PYO3_PYTHON=python3.12
+  && apt-get -y install ca-certificates build-essential libsasl2-dev openjdk-17-jdk software-properties-common python3.12 python3.12-dev openssl pkg-config
 
 FROM base AS rust-base
 
-RUN apt-get update && apt-get -y install make cmake protobuf-compiler curl bash lld unzip
+RUN apt-get update && apt-get -y install make cmake protobuf-compiler curl bash lld unzip rsync
 
 # Install Node.js as dependency for building the dashboard.
 # Bump version together with `dashboard/.node-version`.
@@ -73,9 +67,10 @@ COPY ./ /risingwave
 WORKDIR /risingwave
 
 ENV ENABLE_BUILD_DASHBOARD=1
+ENV OPENSSL_STATIC=1
 
 RUN cargo fetch && \
-  cargo build -p risingwave_cmd_all --release --features "rw-static-link" --features embedded-deno-udf --features embedded-python-udf && \
+  cargo build -p risingwave_cmd_all --release --features "rw-static-link" --features all-udf && \
   mkdir -p /risingwave/bin && \
     mv /risingwave/target/release/risingwave /risingwave/bin/ && \
     mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \
@@ -124,9 +119,14 @@ LABEL org.opencontainers.image.source https://github.com/risingwavelabs/risingwa
 RUN apt-get update && apt-get -y install linux-tools-generic \
     && ln -s "$(find /usr/lib/linux-tools/*/perf | head -1)" /usr/local/bin/perf
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install gdb libpam-krb5 krb5-user telnet kafkacat \
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install gdb libpam-krb5 krb5-user telnet kafkacat rustup \
   && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
+# Set default Rust toolchain but don't install it to keep the image size small
+# The toolchain will be installed when it is first used
+# Do not use `rustup default stable` because it will install immediately
+RUN rustup show && sed -i '1s/^/default_toolchain = "stable"\n/' ~/.rustup/settings.toml
+
 RUN mkdir -p /risingwave/bin/connector-node && mkdir -p /risingwave/lib
 
 COPY --from=rust-builder /risingwave/bin/risingwave /risingwave/bin/risingwave
@@ -142,4 +142,4 @@ ENV CONNECTOR_LIBS_PATH /risingwave/bin/connector-node/libs
 ENV IN_CONTAINER=1
 
 ENTRYPOINT [ "/risingwave/bin/risingwave" ]
-CMD [ "playground" ]
+CMD [ "single_node" ]
diff --git a/docker/Dockerfile.hdfs b/docker/Dockerfile.hdfs
index 53a6da30fe6e0..b6eba07c421c0 100644
--- a/docker/Dockerfile.hdfs
+++ b/docker/Dockerfile.hdfs
@@ -1,16 +1,9 @@
-FROM ubuntu:22.04 AS base
+FROM ubuntu:24.04 AS base
 
 ENV LANG en_US.utf8
 
 RUN apt-get update \
-  && apt-get -y install ca-certificates build-essential libsasl2-dev openjdk-11-jdk software-properties-common
-
-# Install Python 3.12
-RUN add-apt-repository ppa:deadsnakes/ppa -y && \
-    apt-get update -yy && \
-    DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy
-ENV PYO3_PYTHON=python3.12
-
+  && apt-get -y install ca-certificates build-essential libsasl2-dev openjdk-17-jdk software-properties-common python3.12 python3.12-dev openssl pkg-config
 
 FROM base AS dashboard-builder
 
@@ -104,8 +97,10 @@ ARG JAVA_HOME_PATH
 ENV JAVA_HOME ${JAVA_HOME_PATH}
 ENV LD_LIBRARY_PATH ${JAVA_HOME_PATH}/lib/server:${LD_LIBRARY_PATH}
 
+ENV OPENSSL_STATIC=1
+
 RUN cargo fetch && \
-  cargo build -p risingwave_cmd_all --release -p risingwave_object_store --features hdfs-backend --features "rw-static-link" --features embedded-deno-udf --features embedded-python-udf && \
+  cargo build -p risingwave_cmd_all --release -p risingwave_object_store --features hdfs-backend --features "rw-static-link" --features all-udf && \
   mkdir -p /risingwave/bin && \
     mv /risingwave/target/release/risingwave /risingwave/bin/ && \
     mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \
@@ -117,8 +112,8 @@ RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true -Dno-build-rust
     mkdir -p /risingwave/bin/connector-node && \
     tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node
 
-FROM ubuntu:22.04 as image-base
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install ca-certificates openjdk-11-jdk wget libsasl2-dev && rm -rf /var/lib/{apt,dpkg,cache,log}/
+FROM ubuntu:24.04 as image-base
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install ca-certificates openjdk-17-jdk wget libsasl2-dev && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
 FROM image-base as risingwave
 LABEL org.opencontainers.image.source https://github.com/risingwavelabs/risingwave
diff --git a/docker/aws/Dockerfile b/docker/aws/Dockerfile
index b1ada55db4816..1de5f1c7cce08 100644
--- a/docker/aws/Dockerfile
+++ b/docker/aws/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install ca-certificates && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
 RUN mkdir -p /risingwave/bin
diff --git a/docker/dashboards/risingwave-dev-dashboard.json b/docker/dashboards/risingwave-dev-dashboard.json
index 0ca7142657562..223e94e051841 100644
--- a/docker/dashboards/risingwave-dev-dashboard.json
+++ b/docker/dashboards/risingwave-dev-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration from the last committed barrier's epoch time to the current time. This metric reflects the data freshness of the system. During this time, no new data has been committed.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_pending_time","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier pending time (secs)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragmend_id={{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragmend_id={{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, table_id) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - fragment_id {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":36,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":41,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, wait_side, job)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The actor-level memory usage statistics reported by TaskLocalAlloc. (Disabled by default)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":67,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":84,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":87,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during computation. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during source data ingestion. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during data sink out. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":91,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":97,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":118,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 upload task duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 upload task duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax upload task duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"syncing epoch count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handler pending event number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 {{event_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{event_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{event_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax finished_task_wait_poll {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handle latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":133,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":159,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":168,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} size @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_slow_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":187,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":207,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":210,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":214,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":217,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":221,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":226,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":234,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":243,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":254,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":257,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":272,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka high watermark by source and partition and source latest message by partition, source and actor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"high watermark: source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest msg: source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka high watermark and source latest message","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":285,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":291,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":299,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":304,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":29},"height":null,"hideTimeOverride":false,"id":305,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":309,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":310,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":10,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":14,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration from the last committed barrier's epoch time to the current time. This metric reflects the data freshness of the system. During this time, no new data has been committed.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_pending_time","metric":"","query":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier pending time (secs)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}} fragment_id {{fragment_id}}","metric":"","query":"rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":37,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","query":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":42,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} desc: {{desc}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, executor, fragment_id, wait_side, job)(rate(stream_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,executor,fragment_id,wait_side,job) (rate(stream_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"sum by(le, executor, fragment_id, wait_side, job)(rate(stream_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,executor,fragment_id,wait_side,job) (rate(stream_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} - {{job}}","metric":"","query":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Merger Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"sum by(le, job, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The actor-level memory usage statistics reported by TaskLocalAlloc. (Disabled by default)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":69,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":86,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":89,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during computation. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during source data ingestion. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during data sink out. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":93,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":99,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"prefetch cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the estimated hit ratio of a block while in the block cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p10 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p25 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p50 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p75 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p90 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p100 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Cache Efficiency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{iter_type}} - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Existing {{iter_type}} count @ {{table_id}}","metric":"","query":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter_log op count @ {{table_id}} {{op_type}}","metric":"","query":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":121,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploader imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unflushed imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"orphan imm size - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"old value size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg Sync duration - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"syncing epoch count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handler pending event number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handle latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":136,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","query":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":162,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|streaming_read_init',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|streaming_read_init',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":171,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - size @ {{instance}}","metric":"","query":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - {{type}} region - size @ {{instance}}","metric":"","query":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Region Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":197,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":217,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":220,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":224,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":227,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":231,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":236,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":244,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":253,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager eviction policy","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager sequence","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":264,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":267,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","query":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Kv log store uncomsuned min epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, executor_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Backpressure @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, executor_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Backpressure Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed item count @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed row count @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed epoch count @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Buffer State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total size of chunks buffered in a barrier","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Chunk Buffer Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":285,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka high watermark by source and partition and source latest message by partition, source and actor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"high watermark: source={{source_id}} partition={{partition}}","metric":"","query":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest msg: source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka high watermark and source latest message","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":291,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":299,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":304,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":305,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":309,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":310,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":311,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":312,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":313,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":314,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}}","metric":"","query":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":315,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":316,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":317,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":29},"height":null,"hideTimeOverride":false,"id":318,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":319,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":320,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":321,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":322,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":323,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Currently only embedded JS UDF supports this. Others will always show 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":324,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{instance}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{name}} {{fragment_id}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Memory Usage (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
diff --git a/docker/dashboards/risingwave-user-dashboard.json b/docker/dashboards/risingwave-user-dashboard.json
index 5fbc9681be606..037679704e38f 100644
--- a/docker/dashboards/risingwave-user-dashboard.json
+++ b/docker/dashboards/risingwave-user-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n            - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n            - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n            - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n            - Write Stall: Compaction cannot keep up. Stall foreground write.\n            ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":15,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":18,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":27,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":31,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":38,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":45,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":51,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n            - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n            - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n            - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n            - Write Stall: Compaction cannot keep up. Stall foreground write.\n            ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","query":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","query":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","query":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","query":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","query":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","query":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":19,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","query":"rate(actor_memory_usage[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","query":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","query":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":28,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":32,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","query":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":46,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":52,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0}
diff --git a/docker/docker-compose-distributed-etcd.yml b/docker/docker-compose-distributed-etcd.yml
new file mode 100644
index 0000000000000..aed8ccab0329a
--- /dev/null
+++ b/docker/docker-compose-distributed-etcd.yml
@@ -0,0 +1,379 @@
+---
+version: "3"
+x-image: &image
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1}
+services:
+  compactor-0:
+    <<: *image
+    command:
+      - compactor-node
+      - "--listen-addr"
+      - "0.0.0.0:6660"
+      - "--advertise-addr"
+      - "compactor-0:6660"
+      - "--prometheus-listener-addr"
+      - "0.0.0.0:1260"
+      - "--meta-address"
+      - "http://meta-node-0:5690"
+      - "--config-path"
+      - /risingwave.toml
+    expose:
+      - "6660"
+      - "1260"
+    ports: []
+    depends_on:
+      - meta-node-0
+      # - minio-0
+    volumes:
+      - "./risingwave.toml:/risingwave.toml"
+    environment:
+      RUST_BACKTRACE: "1"
+      # If ENABLE_TELEMETRY is not set, telemetry will start by default
+      ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
+    container_name: compactor-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/6660; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+        reservations:
+          memory: 1G
+  compute-node-0:
+    <<: *image
+    command:
+      - compute-node
+      - "--listen-addr"
+      - "0.0.0.0:5688"
+      - "--advertise-addr"
+      - "compute-node-0:5688"
+      - "--prometheus-listener-addr"
+      - "0.0.0.0:1222"
+      - "--meta-address"
+      - "http://meta-node-0:5690"
+      - "--config-path"
+      - /risingwave.toml
+    expose:
+      - "5688"
+      - "1222"
+    ports: []
+    depends_on:
+      - meta-node-0
+      # - minio-0
+    volumes:
+      - "./risingwave.toml:/risingwave.toml"
+    environment:
+      RUST_BACKTRACE: "1"
+      # If ENABLE_TELEMETRY is not set, telemetry will start by default
+      ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
+    container_name: compute-node-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/5688; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+    deploy:
+      resources:
+        limits:
+          memory: 26G
+        reservations:
+          memory: 26G
+  etcd-0:
+    image: "quay.io/coreos/etcd:v3.5.10"
+    command:
+      - /usr/local/bin/etcd
+      - "--listen-client-urls"
+      - "http://0.0.0.0:2388"
+      - "--advertise-client-urls"
+      - "http://etcd-0:2388"
+      - "--listen-peer-urls"
+      - "http://0.0.0.0:2389"
+      - "--initial-advertise-peer-urls"
+      - "http://etcd-0:2389"
+      - "--listen-metrics-urls"
+      - "http://0.0.0.0:2379"
+      - "--name"
+      - risedev-meta
+      - "--max-txn-ops"
+      - "999999"
+      - "--max-request-bytes"
+      - "10485760"
+      - "--auto-compaction-mode"
+      - periodic
+      - "--auto-compaction-retention"
+      - 1m
+      - "--snapshot-count"
+      - "10000"
+      - "--data-dir"
+      - /etcd-data
+    expose:
+      - "2388"
+    ports:
+      - "2388:2388"
+      - "2389:2389"
+    depends_on: []
+    volumes:
+      - "etcd-0:/etcd-data"
+    environment: {}
+    container_name: etcd-0
+    healthcheck:
+      test:
+        - CMD
+        - etcdctl
+        - --endpoints=http://localhost:2388
+        - endpoint
+        - health
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+  frontend-node-0:
+    <<: *image
+    command:
+      - frontend-node
+      - "--listen-addr"
+      - "0.0.0.0:4566"
+      - "--meta-addr"
+      - "http://meta-node-0:5690"
+      - "--advertise-addr"
+      - "frontend-node-0:4566"
+      - "--config-path"
+      - /risingwave.toml
+      - "--prometheus-listener-addr"
+      - "0.0.0.0:2222"
+    expose:
+      - "4566"
+    ports:
+      - "4566:4566"
+    depends_on:
+      - meta-node-0
+    volumes:
+      - "./risingwave.toml:/risingwave.toml"
+    environment:
+      RUST_BACKTRACE: "1"
+      # If ENABLE_TELEMETRY is not set, telemetry will start by default
+      ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
+    container_name: frontend-node-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/4566; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+        reservations:
+          memory: 1G
+  grafana-0:
+    image: "grafana/grafana-oss:latest"
+    command: []
+    expose:
+      - "3001"
+    ports:
+      - "3001:3001"
+    depends_on: []
+    volumes:
+      - "grafana-0:/var/lib/grafana"
+      - "./grafana.ini:/etc/grafana/grafana.ini"
+      - "./grafana-risedev-datasource.yml:/etc/grafana/provisioning/datasources/grafana-risedev-datasource.yml"
+      - "./grafana-risedev-dashboard.yml:/etc/grafana/provisioning/dashboards/grafana-risedev-dashboard.yml"
+      - "./dashboards:/dashboards"
+    environment: {}
+    container_name: grafana-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/3001; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+  meta-node-0:
+    <<: *image
+    command:
+      - meta-node
+      - "--listen-addr"
+      - "0.0.0.0:5690"
+      - "--advertise-addr"
+      - "meta-node-0:5690"
+      - "--dashboard-host"
+      - "0.0.0.0:5691"
+      - "--prometheus-host"
+      - "0.0.0.0:1250"
+      - "--prometheus-endpoint"
+      - "http://prometheus-0:9500"
+      - "--backend"
+      - etcd
+      - "--etcd-endpoints"
+      - "etcd-0:2388"
+      - "--state-store"
+      - "hummock+minio://hummockadmin:hummockadmin@minio-0:9301/hummock001"
+      - "--data-directory"
+      - "hummock_001"
+      - "--config-path"
+      - /risingwave.toml
+    expose:
+      - "5690"
+      - "1250"
+      - "5691"
+    ports:
+      - "5690:5690"
+      - "5691:5691"
+    depends_on:
+      - "etcd-0"
+    volumes:
+      - "./risingwave.toml:/risingwave.toml"
+    environment:
+      RUST_BACKTRACE: "1"
+      # If ENABLE_TELEMETRY is not set, telemetry will start by default
+      ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
+    container_name: meta-node-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/5690; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+        reservations:
+          memory: 1G
+  minio-0:
+    image: "quay.io/minio/minio:latest"
+    command:
+      - server
+      - "--address"
+      - "0.0.0.0:9301"
+      - "--console-address"
+      - "0.0.0.0:9400"
+      - /data
+    expose:
+      - "9301"
+      - "9400"
+    ports:
+      - "9301:9301"
+      - "9400:9400"
+    depends_on: []
+    volumes:
+      - "minio-0:/data"
+    entrypoint: "
+
+      /bin/sh -c '
+
+      set -e
+
+      mkdir -p \"/data/hummock001\"
+
+      /usr/bin/docker-entrypoint.sh \"$$0\" \"$$@\"
+
+      '"
+    environment:
+      MINIO_CI_CD: "1"
+      MINIO_PROMETHEUS_AUTH_TYPE: public
+      MINIO_PROMETHEUS_URL: "http://prometheus-0:9500"
+      MINIO_ROOT_PASSWORD: hummockadmin
+      MINIO_ROOT_USER: hummockadmin
+      MINIO_DOMAIN: "minio-0"
+    container_name: minio-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/9301; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+  prometheus-0:
+    image: "prom/prometheus:latest"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+      - "--web.listen-address=0.0.0.0:9500"
+      - "--storage.tsdb.retention.time=30d"
+    expose:
+      - "9500"
+    ports:
+      - "9500:9500"
+    depends_on: []
+    volumes:
+      - "prometheus-0:/prometheus"
+      - "./prometheus.yaml:/etc/prometheus/prometheus.yml"
+    environment: {}
+    container_name: prometheus-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - sh -c 'printf "GET /-/healthy HTTP/1.0\n\n" | nc localhost 9500; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+  message_queue:
+    image: "docker.vectorized.io/vectorized/redpanda:latest"
+    command:
+      - redpanda
+      - start
+      - "--smp"
+      - "1"
+      - "--reserve-memory"
+      - 0M
+      - "--memory"
+      - 4G
+      - "--overprovisioned"
+      - "--node-id"
+      - "0"
+      - "--check=false"
+      - "--kafka-addr"
+      - "PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092"
+      - "--advertise-kafka-addr"
+      - "PLAINTEXT://message_queue:29092,OUTSIDE://localhost:9092"
+    expose:
+      - "29092"
+      - "9092"
+      - "9644"
+    ports:
+      - "29092:29092"
+      - "9092:9092"
+      - "9644:9644"
+      - "8081:8081"
+    depends_on: []
+    volumes:
+      - "message_queue:/var/lib/redpanda/data"
+    environment: {}
+    container_name: message_queue
+    healthcheck:
+      test: curl -f localhost:9644/v1/status/ready
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+volumes:
+  etcd-0:
+    external: false
+  grafana-0:
+    external: false
+  minio-0:
+    external: false
+  prometheus-0:
+    external: false
+  message_queue:
+    external: false
diff --git a/docker/docker-compose-distributed.yml b/docker/docker-compose-distributed.yml
index 55cb1cbcffe3c..b43f5f1f1921f 100644
--- a/docker/docker-compose-distributed.yml
+++ b/docker/docker-compose-distributed.yml
@@ -1,7 +1,6 @@
 ---
-version: "3"
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.8.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1}
 services:
   compactor-0:
     <<: *image
@@ -87,52 +86,22 @@ services:
           memory: 26G
         reservations:
           memory: 26G
-  etcd-0:
-    image: "quay.io/coreos/etcd:v3.5.10"
-    command:
-      - /usr/local/bin/etcd
-      - "--listen-client-urls"
-      - "http://0.0.0.0:2388"
-      - "--advertise-client-urls"
-      - "http://etcd-0:2388"
-      - "--listen-peer-urls"
-      - "http://0.0.0.0:2389"
-      - "--initial-advertise-peer-urls"
-      - "http://etcd-0:2389"
-      - "--listen-metrics-urls"
-      - "http://0.0.0.0:2379"
-      - "--name"
-      - risedev-meta
-      - "--max-txn-ops"
-      - "999999"
-      - "--max-request-bytes"
-      - "10485760"
-      - "--auto-compaction-mode"
-      - periodic
-      - "--auto-compaction-retention"
-      - 1m
-      - "--snapshot-count"
-      - "10000"
-      - "--data-dir"
-      - /etcd-data
+  postgres-0:
+    image: "postgres:15-alpine"
+    environment:
+      - POSTGRES_HOST_AUTH_METHOD=trust
+      - POSTGRES_USER=postgres
+      - POSTGRES_DB=metadata
+      - POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C
     expose:
-      - "2388"
+      - "5432"
     ports:
-      - "2388:2388"
-      - "2389:2389"
-    depends_on: []
+      - "8432:5432"
     volumes:
-      - "etcd-0:/etcd-data"
-    environment: {}
-    container_name: etcd-0
+      - "postgres-0:/var/lib/postgresql/data"
     healthcheck:
-      test:
-        - CMD
-        - etcdctl
-        - --endpoints=http://localhost:2388
-        - endpoint
-        - health
-      interval: 1s
+      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      interval: 2s
       timeout: 5s
       retries: 5
     restart: always
@@ -216,9 +185,9 @@ services:
       - "--prometheus-endpoint"
       - "http://prometheus-0:9500"
       - "--backend"
-      - etcd
-      - "--etcd-endpoints"
-      - "etcd-0:2388"
+      - sql
+      - "--sql-endpoint"
+      - "postgres://postgres:@postgres-0:5432/metadata"
       - "--state-store"
       - "hummock+minio://hummockadmin:hummockadmin@minio-0:9301/hummock001"
       - "--data-directory"
@@ -233,7 +202,8 @@ services:
       - "5690:5690"
       - "5691:5691"
     depends_on:
-      - "etcd-0"
+      - "postgres-0"
+      - "minio-0"
     volumes:
       - "./risingwave.toml:/risingwave.toml"
     environment:
@@ -367,7 +337,7 @@ services:
       retries: 5
     restart: always
 volumes:
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/docker/docker-compose-etcd.yml b/docker/docker-compose-etcd.yml
new file mode 100644
index 0000000000000..fb1787a029f01
--- /dev/null
+++ b/docker/docker-compose-etcd.yml
@@ -0,0 +1,278 @@
+---
+version: "3"
+x-image: &image
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1}
+services:
+  risingwave-standalone:
+    <<: *image
+    command: "standalone --meta-opts=\" \
+                    --listen-addr 0.0.0.0:5690 \
+                    --advertise-addr 0.0.0.0:5690 \
+                    --dashboard-host 0.0.0.0:5691 \
+                    --prometheus-host 0.0.0.0:1250 \
+                    --prometheus-endpoint http://prometheus-0:9500 \
+                    --backend etcd \
+                    --etcd-endpoints etcd-0:2388 \
+                    --state-store hummock+minio://hummockadmin:hummockadmin@minio-0:9301/hummock001 \
+                    --data-directory hummock_001 \
+                    --config-path /risingwave.toml\" \
+                 --compute-opts=\" \
+                    --config-path /risingwave.toml \
+                    --listen-addr 0.0.0.0:5688 \
+                    --prometheus-listener-addr 0.0.0.0:1250 \
+                    --advertise-addr 0.0.0.0:5688 \
+                    --async-stack-trace verbose \
+                    #--parallelism 4 \
+                    #--total-memory-bytes 8589934592 \
+                    --role both \
+                    --meta-address http://0.0.0.0:5690\" \
+                 --frontend-opts=\" \
+                   --config-path /risingwave.toml \
+                   --listen-addr 0.0.0.0:4566 \
+                   --advertise-addr 0.0.0.0:4566 \
+                   --prometheus-listener-addr 0.0.0.0:1250 \
+                   --health-check-listener-addr 0.0.0.0:6786 \
+                   --meta-addr http://0.0.0.0:5690\" \
+                 --compactor-opts=\" \
+                   --listen-addr 0.0.0.0:6660 \
+                   --prometheus-listener-addr 0.0.0.0:1250 \
+                   --advertise-addr 0.0.0.0:6660 \
+                   --meta-address http://0.0.0.0:5690\""
+    expose:
+      - "6660"
+      - "4566"
+      - "5688"
+      - "5690"
+      - "1250"
+      - "5691"
+    ports:
+      - "4566:4566"
+      - "5690:5690"
+      - "5691:5691"
+      - "1250:1250"
+    depends_on:
+      - etcd-0
+      - minio-0
+    volumes:
+      - "./risingwave.toml:/risingwave.toml"
+    environment:
+      RUST_BACKTRACE: "1"
+      # If ENABLE_TELEMETRY is not set, telemetry will start by default
+      ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
+    container_name: risingwave-standalone
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/6660; exit $$?;'
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/5688; exit $$?;'
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/4566; exit $$?;'
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/5690; exit $$?;'
+      interval: 1s
+      timeout: 5s
+    restart: always
+    deploy:
+      resources:
+        limits:
+          memory: 28G
+        reservations:
+          memory: 28G
+
+  etcd-0:
+    image: "quay.io/coreos/etcd:v3.5.10"
+    command:
+      - /usr/local/bin/etcd
+      - "--listen-client-urls"
+      - "http://0.0.0.0:2388"
+      - "--advertise-client-urls"
+      - "http://etcd-0:2388"
+      - "--listen-peer-urls"
+      - "http://0.0.0.0:2389"
+      - "--initial-advertise-peer-urls"
+      - "http://etcd-0:2389"
+      - "--listen-metrics-urls"
+      - "http://0.0.0.0:2379"
+      - "--name"
+      - risedev-meta
+      - "--max-txn-ops"
+      - "999999"
+      - "--max-request-bytes"
+      - "10485760"
+      - "--auto-compaction-mode"
+      - periodic
+      - "--auto-compaction-retention"
+      - 1m
+      - "--snapshot-count"
+      - "10000"
+      - "--data-dir"
+      - /etcd-data
+    expose:
+      - "2388"
+    ports:
+      - "2388:2388"
+      - "2389:2389"
+    depends_on: []
+    volumes:
+      - "etcd-0:/etcd-data"
+    environment: {}
+    container_name: etcd-0
+    healthcheck:
+      test:
+        - CMD
+        - etcdctl
+        - --endpoints=http://localhost:2388
+        - endpoint
+        - health
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+
+  grafana-0:
+    image: "grafana/grafana-oss:latest"
+    command: []
+    expose:
+      - "3001"
+    ports:
+      - "3001:3001"
+    depends_on: []
+    volumes:
+      - "grafana-0:/var/lib/grafana"
+      - "./grafana.ini:/etc/grafana/grafana.ini"
+      - "./grafana-risedev-datasource.yml:/etc/grafana/provisioning/datasources/grafana-risedev-datasource.yml"
+      - "./grafana-risedev-dashboard.yml:/etc/grafana/provisioning/dashboards/grafana-risedev-dashboard.yml"
+      - "./dashboards:/dashboards"
+    environment: {}
+    container_name: grafana-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/3001; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+
+  minio-0:
+    image: "quay.io/minio/minio:latest"
+    command:
+      - server
+      - "--address"
+      - "0.0.0.0:9301"
+      - "--console-address"
+      - "0.0.0.0:9400"
+      - /data
+    expose:
+      - "9301"
+      - "9400"
+    ports:
+      - "9301:9301"
+      - "9400:9400"
+    depends_on: []
+    volumes:
+      - "minio-0:/data"
+    entrypoint: "
+
+      /bin/sh -c '
+
+      set -e
+
+      mkdir -p \"/data/hummock001\"
+
+      /usr/bin/docker-entrypoint.sh \"$$0\" \"$$@\"
+
+      '"
+    environment:
+      MINIO_CI_CD: "1"
+      MINIO_PROMETHEUS_AUTH_TYPE: public
+      MINIO_PROMETHEUS_URL: "http://prometheus-0:9500"
+      MINIO_ROOT_PASSWORD: hummockadmin
+      MINIO_ROOT_USER: hummockadmin
+      MINIO_DOMAIN: "minio-0"
+    container_name: minio-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/9301; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+
+  prometheus-0:
+    image: "prom/prometheus:latest"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+      - "--web.listen-address=0.0.0.0:9500"
+      - "--storage.tsdb.retention.time=30d"
+    expose:
+      - "9500"
+    ports:
+      - "9500:9500"
+    depends_on: []
+    volumes:
+      - "prometheus-0:/prometheus"
+      - "./prometheus.yaml:/etc/prometheus/prometheus.yml"
+    environment: {}
+    container_name: prometheus-0
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - sh -c 'printf "GET /-/healthy HTTP/1.0\n\n" | nc localhost 9500; exit $$?;'
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+
+  message_queue:
+    image: "docker.vectorized.io/vectorized/redpanda:latest"
+    command:
+      - redpanda
+      - start
+      - "--smp"
+      - "1"
+      - "--reserve-memory"
+      - 0M
+      - "--memory"
+      - 4G
+      - "--overprovisioned"
+      - "--node-id"
+      - "0"
+      - "--check=false"
+      - "--kafka-addr"
+      - "PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092"
+      - "--advertise-kafka-addr"
+      - "PLAINTEXT://message_queue:29092,OUTSIDE://localhost:9092"
+    expose:
+      - "29092"
+      - "9092"
+      - "9644"
+    ports:
+      - "29092:29092"
+      - "9092:9092"
+      - "9644:9644"
+      - "8081:8081"
+    depends_on: []
+    volumes:
+      - "message_queue:/var/lib/redpanda/data"
+    environment: {}
+    container_name: message_queue
+    healthcheck:
+      test: curl -f localhost:9644/v1/status/ready
+      interval: 1s
+      timeout: 5s
+      retries: 5
+    restart: always
+volumes:
+  etcd-0:
+    external: false
+  grafana-0:
+    external: false
+  minio-0:
+    external: false
+  prometheus-0:
+    external: false
+  message_queue:
+    external: false
diff --git a/docker/docker-compose-with-azblob.yml b/docker/docker-compose-with-azblob.yml
index e0b44c5768011..3269b2c815a82 100644
--- a/docker/docker-compose-with-azblob.yml
+++ b/docker/docker-compose-with-azblob.yml
@@ -1,7 +1,6 @@
 ---
-version: "3"
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.8.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1}
 services:
   risingwave-standalone:
     <<: *image
@@ -11,8 +10,8 @@ services:
                     --dashboard-host 0.0.0.0:5691 \
                     --prometheus-host 0.0.0.0:1250 \
                     --prometheus-endpoint http://prometheus-0:9500 \
-                    --backend etcd \
-                    --etcd-endpoints etcd-0:2388 \
+                    --backend sql \
+                    --sql-endpoint postgres://postgres:@postgres-0:5432/metadata \
                     --state-store hummock+azblob://<container_name> \
                     --data-directory hummock_001 \
                     --config-path /risingwave.toml\" \
@@ -51,7 +50,7 @@ services:
       - "5691:5691"
       - "1250:1250"
     depends_on:
-      - etcd-0
+      - postgres-0
     env_file: multiple_object_storage.env
     volumes:
       - "./risingwave.toml:/risingwave.toml"
@@ -76,10 +75,10 @@ services:
           memory: 28G
         reservations:
           memory: 28G
-  etcd-0:
+  postgres-0:
     extends:
       file: docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: docker-compose.yml
@@ -93,7 +92,7 @@ services:
       file: docker-compose.yml
       service: message_queue
 volumes:
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/docker/docker-compose-with-gcs.yml b/docker/docker-compose-with-gcs.yml
index 847172c2d09c1..1de6b4ee6253d 100644
--- a/docker/docker-compose-with-gcs.yml
+++ b/docker/docker-compose-with-gcs.yml
@@ -1,7 +1,6 @@
 ---
-version: "3"
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.8.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1}
 services:
   risingwave-standalone:
     <<: *image
@@ -11,8 +10,8 @@ services:
                     --dashboard-host 0.0.0.0:5691 \
                     --prometheus-host 0.0.0.0:1250 \
                     --prometheus-endpoint http://prometheus-0:9500 \
-                    --backend etcd \
-                    --etcd-endpoints etcd-0:2388 \
+                    --backend sql \
+                    --sql-endpoint postgres://postgres:@postgres-0:5432/metadata \
                     --state-store hummock+gcs://<bucket-name> \
                     --data-directory hummock_001 \
                     --config-path /risingwave.toml\" \
@@ -51,7 +50,7 @@ services:
       - "5691:5691"
       - "1250:1250"
     depends_on:
-      - etcd-0
+      - postgres-0
     env_file: multiple_object_storage.env
     volumes:
       - "./risingwave.toml:/risingwave.toml"
@@ -76,10 +75,10 @@ services:
           memory: 28G
         reservations:
           memory: 28G
-  etcd-0:
+  postgres-0:
     extends:
       file: docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: docker-compose.yml
@@ -93,7 +92,7 @@ services:
       file: docker-compose.yml
       service: message_queue
 volumes:
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/docker/docker-compose-with-hdfs.yml b/docker/docker-compose-with-hdfs.yml
index cf2b45078bac5..974cf922e77b4 100644
--- a/docker/docker-compose-with-hdfs.yml
+++ b/docker/docker-compose-with-hdfs.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 services:
   compactor-0:
     image: ghcr.io/risingwavelabs/risingwave:RisingWave_1.6.1_HDFS_2.7-x86_64
diff --git a/docker/docker-compose-with-local-fs.yml b/docker/docker-compose-with-local-fs.yml
index b45e624c619b3..aa02531db1d3e 100644
--- a/docker/docker-compose-with-local-fs.yml
+++ b/docker/docker-compose-with-local-fs.yml
@@ -1,7 +1,6 @@
 ---
-version: "3"
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:nightly-20231211}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1}
 services:
   risingwave-standalone:
     <<: *image
@@ -11,8 +10,8 @@ services:
                     --dashboard-host 0.0.0.0:5691 \
                     --prometheus-host 0.0.0.0:1250 \
                     --prometheus-endpoint http://prometheus-0:9500 \
-                    --backend etcd \
-                    --etcd-endpoints etcd-0:2388 \
+                    --backend sql \
+                    --sql-endpoint postgres://postgres:@postgres-0:5432/metadata \
                     --state-store hummock+fs://<local-path> \
                     --data-directory hummock_001 \
                     --config-path /risingwave.toml\" \
@@ -50,7 +49,7 @@ services:
       - "5691:5691"
       - "1250:1250"
     depends_on:
-      - etcd-0
+      - postgres-0
     volumes:
       - "./risingwave.toml:/risingwave.toml"
     environment:
@@ -74,10 +73,10 @@ services:
           memory: <config-the-allocated-memory>
         reservations:
           memory: <config-the-allocated-memory>
-  etcd-0:
+  postgres-0:
     extends:
       file: docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: docker-compose.yml
@@ -87,7 +86,7 @@ services:
       file: docker-compose.yml
       service: prometheus-0
 volumes:
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/docker/docker-compose-with-obs.yml b/docker/docker-compose-with-obs.yml
index 5d0df0ca4f72d..39fb135132da5 100644
--- a/docker/docker-compose-with-obs.yml
+++ b/docker/docker-compose-with-obs.yml
@@ -1,7 +1,6 @@
 ---
-version: "3"
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.8.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1}
 services:
   risingwave-standalone:
     <<: *image
@@ -11,8 +10,8 @@ services:
                     --dashboard-host 0.0.0.0:5691 \
                     --prometheus-host 0.0.0.0:1250 \
                     --prometheus-endpoint http://prometheus-0:9500 \
-                    --backend etcd \
-                    --etcd-endpoints etcd-0:2388 \
+                    --backend sql \
+                    --sql-endpoint postgres://postgres:@postgres-0:5432/metadata \
                     --state-store hummock+obs://<bucket-name> \
                     --data-directory hummock_001 \
                     --config-path /risingwave.toml\" \
@@ -51,7 +50,7 @@ services:
       - "5691:5691"
       - "1250:1250"
     depends_on:
-      - etcd-0
+      - postgres-0
     env_file: multiple_object_storage.env
     volumes:
       - "./risingwave.toml:/risingwave.toml"
@@ -76,10 +75,10 @@ services:
           memory: 28G
         reservations:
           memory: 28G
-  etcd-0:
+  postgres-0:
     extends:
       file: docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: docker-compose.yml
@@ -93,7 +92,7 @@ services:
       file: docker-compose.yml
       service: message_queue
 volumes:
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/docker/docker-compose-with-oss.yml b/docker/docker-compose-with-oss.yml
index 7296a7074d5a6..677d0ce82b27c 100644
--- a/docker/docker-compose-with-oss.yml
+++ b/docker/docker-compose-with-oss.yml
@@ -1,7 +1,6 @@
 ---
-version: "3"
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.8.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1}
 services:
   risingwave-standalone:
     <<: *image
@@ -11,8 +10,8 @@ services:
                     --dashboard-host 0.0.0.0:5691 \
                     --prometheus-host 0.0.0.0:1250 \
                     --prometheus-endpoint http://prometheus-0:9500 \
-                    --backend etcd \
-                    --etcd-endpoints etcd-0:2388 \
+                    --backend sql \
+                    --sql-endpoint postgres://postgres:@postgres-0:5432/metadata \
                     --state-store hummock+oss://<bucket-name> \
                     --data-directory hummock_001 \
                     --config-path /risingwave.toml\" \
@@ -51,7 +50,7 @@ services:
       - "5691:5691"
       - "1250:1250"
     depends_on:
-      - etcd-0
+      - postgres-0
     env_file: multiple_object_storage.env
     volumes:
       - "./risingwave.toml:/risingwave.toml"
@@ -76,10 +75,10 @@ services:
           memory: 28G
         reservations:
           memory: 28G
-  etcd-0:
+  postgres-0:
     extends:
       file: docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: docker-compose.yml
@@ -93,7 +92,7 @@ services:
       file: docker-compose.yml
       service: message_queue
 volumes:
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/docker/docker-compose-with-s3.yml b/docker/docker-compose-with-s3.yml
index 815489f82493e..9a3d33b261aff 100644
--- a/docker/docker-compose-with-s3.yml
+++ b/docker/docker-compose-with-s3.yml
@@ -1,7 +1,6 @@
 ---
-version: "3"
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.8.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1}
 services:
   risingwave-standalone:
     <<: *image
@@ -11,8 +10,8 @@ services:
                     --dashboard-host 0.0.0.0:5691 \
                     --prometheus-host 0.0.0.0:1250 \
                     --prometheus-endpoint http://prometheus-0:9500 \
-                    --backend etcd \
-                    --etcd-endpoints etcd-0:2388 \
+                    --backend sql \
+                    --sql-endpoint postgres://postgres:@postgres-0:5432/metadata \
                     --state-store hummock+s3://<bucket-name> \
                     --data-directory hummock_001 \
                     --config-path /risingwave.toml\" \
@@ -51,7 +50,7 @@ services:
       - "5691:5691"
       - "1250:1250"
     depends_on:
-      - etcd-0
+      - postgres-0
     env_file: aws.env
     volumes:
       - "./risingwave.toml:/risingwave.toml"
@@ -76,10 +75,10 @@ services:
           memory: 28G
         reservations:
           memory: 28G
-  etcd-0:
+  postgres-0:
     extends:
       file: docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: docker-compose.yml
@@ -93,7 +92,7 @@ services:
       file: docker-compose.yml
       service: message_queue
 volumes:
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/docker/docker-compose-with-sqlite.yml b/docker/docker-compose-with-sqlite.yml
index c303a09b2b4f9..1d495fd0e7875 100644
--- a/docker/docker-compose-with-sqlite.yml
+++ b/docker/docker-compose-with-sqlite.yml
@@ -1,7 +1,6 @@
 ---
-version: "3"
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.8.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 6259a5757b14f..71280dc6e98b5 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -1,7 +1,6 @@
 ---
-version: "3"
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.8.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.9.1}
 services:
   risingwave-standalone:
     <<: *image
@@ -11,8 +10,8 @@ services:
                     --dashboard-host 0.0.0.0:5691 \
                     --prometheus-host 0.0.0.0:1250 \
                     --prometheus-endpoint http://prometheus-0:9500 \
-                    --backend etcd \
-                    --etcd-endpoints etcd-0:2388 \
+                    --backend sql \
+                    --sql-endpoint postgres://postgres:@postgres-0:5432/metadata \
                     --state-store hummock+minio://hummockadmin:hummockadmin@minio-0:9301/hummock001 \
                     --data-directory hummock_001 \
                     --config-path /risingwave.toml\" \
@@ -51,7 +50,7 @@ services:
       - "5691:5691"
       - "1250:1250"
     depends_on:
-      - etcd-0
+      - postgres-0
       - minio-0
     volumes:
       - "./risingwave.toml:/risingwave.toml"
@@ -77,52 +76,22 @@ services:
         reservations:
           memory: 28G
 
-  etcd-0:
-    image: "quay.io/coreos/etcd:v3.5.10"
-    command:
-      - /usr/local/bin/etcd
-      - "--listen-client-urls"
-      - "http://0.0.0.0:2388"
-      - "--advertise-client-urls"
-      - "http://etcd-0:2388"
-      - "--listen-peer-urls"
-      - "http://0.0.0.0:2389"
-      - "--initial-advertise-peer-urls"
-      - "http://etcd-0:2389"
-      - "--listen-metrics-urls"
-      - "http://0.0.0.0:2379"
-      - "--name"
-      - risedev-meta
-      - "--max-txn-ops"
-      - "999999"
-      - "--max-request-bytes"
-      - "10485760"
-      - "--auto-compaction-mode"
-      - periodic
-      - "--auto-compaction-retention"
-      - 1m
-      - "--snapshot-count"
-      - "10000"
-      - "--data-dir"
-      - /etcd-data
+  postgres-0:
+    image: "postgres:15-alpine"
+    environment:
+      - POSTGRES_HOST_AUTH_METHOD=trust
+      - POSTGRES_USER=postgres
+      - POSTGRES_DB=metadata
+      - POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C
     expose:
-      - "2388"
+      - "5432"
     ports:
-      - "2388:2388"
-      - "2389:2389"
-    depends_on: []
+      - "8432:5432"
     volumes:
-      - "etcd-0:/etcd-data"
-    environment: {}
-    container_name: etcd-0
+      - "postgres-0:/var/lib/postgresql/data"
     healthcheck:
-      test:
-        - CMD
-        - etcdctl
-        - --endpoints=http://localhost:2388
-        - endpoint
-        - health
-      interval: 1s
+      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      interval: 2s
       timeout: 5s
       retries: 5
     restart: always
@@ -266,7 +235,7 @@ services:
       retries: 5
     restart: always
 volumes:
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/docs/backfill.md b/docs/backfill.md
index 6784070853712..aac20615caf10 100644
--- a/docs/backfill.md
+++ b/docs/backfill.md
@@ -358,6 +358,22 @@ and arrangement backfill will consume this historical data snapshot:
 | 1                  | 'Jack' | 29 |
 | 2                  | 'Jill' | 30 |
 
+#### Initialization
+
+Something to note is that for the first snapshot,
+upstream may not have finished committing data in that epoch to s3.
+
+Additionally, we have not replicated any upstream records
+during that epoch, only in the subsequent ones.
+
+As such, we must wait for that first checkpoint to be committed,
+before reading, or we risk missing the uncommitted data in our backfill.
+
+This is supported internally inside `init_epoch` for replicated state table.
+```shell
+        upstream_table.init_epoch(first_epoch).await?;
+```
+
 ### Recovery
 
 TODO
diff --git a/docs/developer-guide.md b/docs/developer-guide.md
index 8cf5dcd4ebbaa..9b03c60f3d9a1 100644
--- a/docs/developer-guide.md
+++ b/docs/developer-guide.md
@@ -72,6 +72,7 @@ RiseDev is the development mode of RisingWave. To develop RisingWave, you need t
 * Rust toolchain
 * CMake
 * protobuf (>= 3.12.0)
+* OpenSSL (>= 3)
 * PostgreSQL (psql) (>= 14.1)
 * Tmux (>= v3.2a)
 * LLVM 16 (For macOS only, to workaround some bugs in macOS toolchain. See https://github.com/risingwavelabs/risingwave/issues/6205)
@@ -80,7 +81,7 @@ RiseDev is the development mode of RisingWave. To develop RisingWave, you need t
 To install the dependencies on macOS, run:
 
 ```shell
-brew install postgresql cmake protobuf tmux cyrus-sasl llvm
+brew install postgresql cmake protobuf tmux cyrus-sasl llvm openssl@3
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 ```
 
@@ -97,7 +98,7 @@ Then you'll be able to compile and start RiseDev!
 >
 > `.cargo/config.toml` contains `rustflags` configurations like `-Clink-arg` and `-Ctarget-feature`. Since it will be [merged](https://doc.rust-lang.org/cargo/reference/config.html#hierarchical-structure) with `$HOME/.cargo/config.toml`, check the config files and make sure they don't conflict if you have global `rustflags` configurations for e.g. linker there.
 
-> [!INFO]
+> [!TIP]
 >
 > If you want to build RisingWave with `embedded-python-udf` feature, you need to install Python 3.12.
 >
@@ -185,16 +186,10 @@ For example, you can modify the default section to:
     - use: frontend
     - use: prometheus
     - use: grafana
-    - use: zookeeper
-      persist-data: true
     - use: kafka
       persist-data: true
 ```
 
-> [!NOTE]
->
-> The Kafka service depends on the ZooKeeper service. If you want to enable the Kafka component, enable the ZooKeeper component first.
-
 Now you can run `./risedev d` to start a new dev cluster. The new dev cluster will contain components as configured in the yaml file. RiseDev will automatically configure the components to use the available storage service and to monitor the target.
 
 You may also add multiple compute nodes in the cluster. The `ci-3cn-1fe` config is an example.
@@ -553,17 +548,18 @@ Instructions about submitting PRs are included in the [contribution guidelines](
 
 ## CI Labels Guide
 
-- `[ci/run-xxx ...]`: Run additional steps indicated by `ci/run-xxx` in your PR.
-- `ci/skip-ci` + `[ci/run-xxx ...]` : Skip steps except for those indicated by `ci/run-xxx` in your **DRAFT PR.**
-- `ci/run-main-cron`: Run full `main-cron`.
-- `ci/run-main-cron` + `ci/main-cron/skip-ci` + `[ci/run-xxx …]` : Run specific steps indicated by `ci/run-xxx`
+- `[ci/run-xxx ...]`: Run additional steps in the PR workflow indicated by `ci/run-xxx` in your PR.
+- `ci/pr/run-selected` + `[ci/run-xxx ...]` : Only run selected steps indicated by `ci/run-xxx` in your **DRAFT PR.**
+- `ci/main-cron/run-all`: Run full `main-cron` workflow for your PR.
+- `ci/main-cron/run-selected` + `[ci/run-xxx …]` : Run specific steps indicated by `ci/run-xxx`
   from the `main-cron` workflow, in your PR. Can use to verify some `main-cron` fix works as expected.
 - To reference `[ci/run-xxx ...]` labels, you may look at steps from `pull-request.yml` and `main-cron.yml`.
-- **Be sure to add all the dependencies.**
-
-  For example to run `e2e-test` for `main-cron` in your pull request:
-  1. Add `ci/run-build`, `ci/run-build-other`, `ci/run-docslt` .
-     These correspond to its `depends` field in `pull-request.yml` and `main-cron.yml` .
-  2. Add `ci/run-e2e-test` to run the step as well.
-  3. Add `ci/run-main-cron` to run `main-cron` workflow in your pull request,
-  4. Add `ci/main-cron/skip-ci` to skip all other steps which were not selected with `ci/run-xxx`.
\ No newline at end of file
+
+### Example
+
+https://github.com/risingwavelabs/risingwave/pull/17197
+
+To run `e2e-test` and `e2e-source-test` for `main-cron` in your pull request:
+1. Add `ci/run-e2e-test`.
+2. Add `ci/run-e2e-source-tests`.
+3. Add `ci/main-cron/run-selected` to skip all other steps which were not selected with `ci/run-xxx`.
\ No newline at end of file
diff --git a/e2e_test/backfill/sink/create_sink.slt b/e2e_test/backfill/sink/create_sink.slt
index bc9fba04da5c8..016e3bcb2049b 100644
--- a/e2e_test/backfill/sink/create_sink.slt
+++ b/e2e_test/backfill/sink/create_sink.slt
@@ -5,9 +5,9 @@ statement ok
 create table t (v1 int);
 
 statement ok
-SET STREAMING_RATE_LIMIT = 1000;
+SET STREAMING_RATE_LIMIT = 500;
 
-# Should finish in 10s
+# Should finish in 20s
 statement ok
 insert into t select * from generate_series(1, 10000);
 
@@ -20,7 +20,7 @@ from t x join t y
     on x.v1 = y.v1
 with (
     connector='kafka',
-    properties.bootstrap.server='localhost:29092',
+    properties.bootstrap.server='message_queue:29092',
     topic='s_kafka',
     primary_key='v1',
     allow.auto.create.topics=true,
diff --git a/e2e_test/backfill/sink/different_pk_and_dist_key.slt b/e2e_test/backfill/sink/different_pk_and_dist_key.slt
new file mode 100644
index 0000000000000..bc8256b28e62a
--- /dev/null
+++ b/e2e_test/backfill/sink/different_pk_and_dist_key.slt
@@ -0,0 +1,49 @@
+statement ok
+create table t(v1 int, v2 int primary key, v3 int);
+
+statement ok
+create table t2(v1 int, v2 int primary key, v3 int);
+
+# Let snapshot side pk >= upstream side
+statement ok
+insert into t select 50000 + generate_series, 60000 + generate_series, 70000 + generate_series from generate_series(1, 10000);
+
+statement ok
+insert into t2 select 50000 + generate_series, 60000 + generate_series, 70000 + generate_series from generate_series(1, 10000);
+
+statement ok
+flush;
+
+statement ok
+create materialized view m1 as select t.v1, t.v2, t.v3 from t join t2 using(v1);
+
+statement ok
+set streaming_rate_limit = 1;
+
+statement ok
+set background_ddl = true;
+
+statement ok
+create sink s1 as select t.v1, t.v2, t.v3 from m1 join t using(v3) with (connector = 'blackhole');
+
+# Let snapshot side pk >= upstream side
+statement ok
+insert into t select 10000 + generate_series, 20000 + generate_series, 30000 + generate_series from generate_series(1, 10000);
+
+statement ok
+insert into t2 select 10000 + generate_series, 20000 + generate_series, 30000 + generate_series from generate_series(1, 10000);
+
+statement ok
+flush;
+
+statement ok
+drop sink s1;
+
+statement ok
+drop materialized view m1;
+
+statement ok
+drop table t;
+
+statement ok
+drop table t2;
\ No newline at end of file
diff --git a/backwards-compat-tests/README.md b/e2e_test/backwards-compat-tests/README.md
similarity index 100%
rename from backwards-compat-tests/README.md
rename to e2e_test/backwards-compat-tests/README.md
diff --git a/backwards-compat-tests/scripts/run_local.sh b/e2e_test/backwards-compat-tests/scripts/run_local.sh
similarity index 59%
rename from backwards-compat-tests/scripts/run_local.sh
rename to e2e_test/backwards-compat-tests/scripts/run_local.sh
index 76b1a4f333458..7ac952e6d1c4a 100755
--- a/backwards-compat-tests/scripts/run_local.sh
+++ b/e2e_test/backwards-compat-tests/scripts/run_local.sh
@@ -10,11 +10,14 @@ on_exit() {
 
 trap on_exit EXIT
 
-source backwards-compat-tests/scripts/utils.sh
+source e2e_test/backwards-compat-tests/scripts/utils.sh
 
 configure_rw() {
-echo "--- Setting up cluster config"
-cat <<EOF > risedev-profiles.user.yml
+  VERSION="$1"
+
+  echo "--- Setting up cluster config"
+  if version_le "$VERSION" "1.9.0"; then
+    cat <<EOF > risedev-profiles.user.yml
 full-without-monitoring:
   steps:
     - use: minio
@@ -23,16 +26,34 @@ full-without-monitoring:
     - use: compute-node
     - use: frontend
     - use: compactor
-    - use: zookeeper
     - use: kafka
+      user-managed: true
+      address: message_queue
+      port: 29092
+EOF
+  else
+    cat <<EOF > risedev-profiles.user.yml
+ full-without-monitoring:
+   steps:
+     - use: minio
+     - use: etcd
+     - use: meta-node
+       meta-backend: etcd
+     - use: compute-node
+     - use: frontend
+     - use: compactor
+     - use: kafka
+       user-managed: true
+       address: message_queue
+       port: 29092
 EOF
+  fi
 
 cat <<EOF > risedev-components.user.env
 RISEDEV_CONFIGURED=false
 
 ENABLE_MINIO=true
 ENABLE_ETCD=true
-ENABLE_KAFKA=true
 
 # Fetch risingwave binary from release.
 ENABLE_BUILD_RUST=true
@@ -57,12 +78,12 @@ main() {
   set -euo pipefail
   get_rw_versions
   setup_old_cluster
-  configure_rw
+  configure_rw "$OLD_VERSION"
   seed_old_cluster "$OLD_VERSION"
 
   setup_new_cluster
-  configure_rw
+  configure_rw "99.99.99"
   validate_new_cluster "$NEW_VERSION"
 }
 
-main
\ No newline at end of file
+main
diff --git a/backwards-compat-tests/scripts/utils.sh b/e2e_test/backwards-compat-tests/scripts/utils.sh
similarity index 86%
rename from backwards-compat-tests/scripts/utils.sh
rename to e2e_test/backwards-compat-tests/scripts/utils.sh
index 0fd3e3908ca6e..8f41dad0860f1 100644
--- a/backwards-compat-tests/scripts/utils.sh
+++ b/e2e_test/backwards-compat-tests/scripts/utils.sh
@@ -18,36 +18,9 @@
 RECOVERY_DURATION=20
 
 # Setup test directory
-TEST_DIR=.risingwave/backwards-compat-tests/
-KAFKA_PATH=.risingwave/bin/kafka
+TEST_DIR=.risingwave/e2e_test/backwards-compat-tests/
 mkdir -p $TEST_DIR
-cp -r backwards-compat-tests/slt/* $TEST_DIR
-
-wait_kafka_exit() {
-  # Follow kafka-server-stop.sh
-  while [[ -n "$(ps ax | grep ' kafka\.Kafka ' | grep java | grep -v grep | awk '{print $1}')" ]]; do
-    echo "Waiting for kafka to exit"
-    sleep 1
-  done
-}
-
-wait_zookeeper_exit() {
-  # Follow zookeeper-server-stop.sh
-  while [[ -n "$(ps ax | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}')" ]]; do
-    echo "Waiting for zookeeper to exit"
-    sleep 1
-  done
-}
-
-kill_kafka() {
-  $KAFKA_PATH/bin/kafka-server-stop.sh
-  wait_kafka_exit
-}
-
-kill_zookeeper() {
-  $KAFKA_PATH/bin/zookeeper-server-stop.sh
-  wait_zookeeper_exit
-}
+cp -r e2e_test/backwards-compat-tests/slt/* $TEST_DIR
 
 wait_for_process() {
   process_name="$1"
@@ -77,24 +50,9 @@ kill_cluster() {
 
   # Kill other components
   $TMUX list-windows -t risedev -F "#{window_name} #{pane_id}" |
-    grep -v 'kafka' |
-    grep -v 'zookeeper' |
     awk '{ print $2 }' |
     xargs -I {} $TMUX send-keys -t {} C-c C-d
 
-  set +e
-  if [[ -n $($TMUX list-windows -t risedev | grep kafka) ]]; then
-    echo "kill kafka"
-    kill_kafka
-
-    echo "kill zookeeper"
-    kill_zookeeper
-
-    # Kill their tmux sessions
-    $TMUX list-windows -t risedev -F "#{pane_id}" | xargs -I {} $TMUX send-keys -t {} C-c C-d
-  fi
-  set -e
-
   $TMUX kill-server
   test $? -eq 0 || {
     echo "Failed to stop all RiseDev components."
@@ -120,18 +78,16 @@ check_version() {
 }
 
 create_kafka_topic() {
-  "$KAFKA_PATH"/bin/kafka-topics.sh \
-    --create \
-    --topic backwards_compat_test_kafka_source --bootstrap-server localhost:29092
+  RPK_BROKERS=message_queue:29092 \
+  rpk topic create backwards_compat_test_kafka_source
 }
 
 insert_json_kafka() {
   local JSON=$1
-  echo "$JSON" | "$KAFKA_PATH"/bin/kafka-console-producer.sh \
-    --topic backwards_compat_test_kafka_source \
-    --bootstrap-server localhost:29092 \
-    --property "parse.key=true" \
-    --property "key.separator=,"
+
+  echo "$JSON" | \
+  RPK_BROKERS=message_queue:29092 \
+  rpk topic produce backwards_compat_test_kafka_source -f "%k,%v"
 }
 
 seed_json_kafka() {
diff --git a/backwards-compat-tests/slt/basic/seed.slt b/e2e_test/backwards-compat-tests/slt/basic/seed.slt
similarity index 100%
rename from backwards-compat-tests/slt/basic/seed.slt
rename to e2e_test/backwards-compat-tests/slt/basic/seed.slt
diff --git a/backwards-compat-tests/slt/basic/validate_original.slt b/e2e_test/backwards-compat-tests/slt/basic/validate_original.slt
similarity index 100%
rename from backwards-compat-tests/slt/basic/validate_original.slt
rename to e2e_test/backwards-compat-tests/slt/basic/validate_original.slt
diff --git a/backwards-compat-tests/slt/basic/validate_restart.slt b/e2e_test/backwards-compat-tests/slt/basic/validate_restart.slt
similarity index 100%
rename from backwards-compat-tests/slt/basic/validate_restart.slt
rename to e2e_test/backwards-compat-tests/slt/basic/validate_restart.slt
diff --git a/backwards-compat-tests/slt/kafka/invalid_options/seed.slt b/e2e_test/backwards-compat-tests/slt/kafka/invalid_options/seed.slt
similarity index 92%
rename from backwards-compat-tests/slt/kafka/invalid_options/seed.slt
rename to e2e_test/backwards-compat-tests/slt/kafka/invalid_options/seed.slt
index 6021cf1253b8e..2fef943c8ab49 100644
--- a/backwards-compat-tests/slt/kafka/invalid_options/seed.slt
+++ b/e2e_test/backwards-compat-tests/slt/kafka/invalid_options/seed.slt
@@ -14,7 +14,7 @@ CREATE SOURCE IF NOT EXISTS kafka_source_with_invalid_option
 WITH (
  connector='kafka',
  topic='backwards_compat_test_kafka_source',
- properties.bootstrap.server='localhost:29092',
+ properties.bootstrap.server='message_queue:29092',
  scan.startup.mode='earliest',
  invalid_option='oops'
 ) FORMAT PLAIN ENCODE JSON;
diff --git a/backwards-compat-tests/slt/kafka/invalid_options/validate_original.slt b/e2e_test/backwards-compat-tests/slt/kafka/invalid_options/validate_original.slt
similarity index 100%
rename from backwards-compat-tests/slt/kafka/invalid_options/validate_original.slt
rename to e2e_test/backwards-compat-tests/slt/kafka/invalid_options/validate_original.slt
diff --git a/backwards-compat-tests/slt/kafka/invalid_options/validate_restart.slt b/e2e_test/backwards-compat-tests/slt/kafka/invalid_options/validate_restart.slt
similarity index 100%
rename from backwards-compat-tests/slt/kafka/invalid_options/validate_restart.slt
rename to e2e_test/backwards-compat-tests/slt/kafka/invalid_options/validate_restart.slt
diff --git a/backwards-compat-tests/slt/kafka/seed.slt b/e2e_test/backwards-compat-tests/slt/kafka/seed.slt
similarity index 87%
rename from backwards-compat-tests/slt/kafka/seed.slt
rename to e2e_test/backwards-compat-tests/slt/kafka/seed.slt
index 3840ce0c96b15..aa2a196694e4d 100644
--- a/backwards-compat-tests/slt/kafka/seed.slt
+++ b/e2e_test/backwards-compat-tests/slt/kafka/seed.slt
@@ -11,7 +11,7 @@ CREATE SOURCE IF NOT EXISTS kafka_source
 WITH (
  connector='kafka',
  topic='backwards_compat_test_kafka_source',
- properties.bootstrap.server='localhost:29092',
+ properties.bootstrap.server='message_queue:29092',
  scan.startup.mode='earliest',
 ) FORMAT PLAIN ENCODE JSON;
 
diff --git a/backwards-compat-tests/slt/kafka/upsert/deprecate_upsert.slt b/e2e_test/backwards-compat-tests/slt/kafka/upsert/deprecate_upsert.slt
similarity index 76%
rename from backwards-compat-tests/slt/kafka/upsert/deprecate_upsert.slt
rename to e2e_test/backwards-compat-tests/slt/kafka/upsert/deprecate_upsert.slt
index 55cfce886455d..57f1e2d27391b 100644
--- a/backwards-compat-tests/slt/kafka/upsert/deprecate_upsert.slt
+++ b/e2e_test/backwards-compat-tests/slt/kafka/upsert/deprecate_upsert.slt
@@ -11,6 +11,6 @@ CREATE TABLE IF NOT EXISTS kafka_table
 WITH (
  connector='kafka',
  topic='backwards_compat_test_kafka_source',
- properties.bootstrap.server='localhost:29092',
+ properties.bootstrap.server='message_queue:29092',
  scan.startup.mode='earliest',
-) FORMAT UPSERT ENCODE JSON;
\ No newline at end of file
+) FORMAT UPSERT ENCODE JSON;
diff --git a/backwards-compat-tests/slt/kafka/upsert/include_key_as.slt b/e2e_test/backwards-compat-tests/slt/kafka/upsert/include_key_as.slt
similarity index 78%
rename from backwards-compat-tests/slt/kafka/upsert/include_key_as.slt
rename to e2e_test/backwards-compat-tests/slt/kafka/upsert/include_key_as.slt
index 36ef426574223..37d41e195ee05 100644
--- a/backwards-compat-tests/slt/kafka/upsert/include_key_as.slt
+++ b/e2e_test/backwards-compat-tests/slt/kafka/upsert/include_key_as.slt
@@ -13,6 +13,6 @@ INCLUDE key as _rw_key
 WITH (
  connector='kafka',
  topic='backwards_compat_test_kafka_source',
- properties.bootstrap.server='localhost:29092',
+ properties.bootstrap.server='message_queue:29092',
  scan.startup.mode='earliest',
-) FORMAT UPSERT ENCODE JSON;
\ No newline at end of file
+) FORMAT UPSERT ENCODE JSON;
diff --git a/backwards-compat-tests/slt/kafka/validate_original.slt b/e2e_test/backwards-compat-tests/slt/kafka/validate_original.slt
similarity index 100%
rename from backwards-compat-tests/slt/kafka/validate_original.slt
rename to e2e_test/backwards-compat-tests/slt/kafka/validate_original.slt
diff --git a/backwards-compat-tests/slt/kafka/validate_restart.slt b/e2e_test/backwards-compat-tests/slt/kafka/validate_restart.slt
similarity index 100%
rename from backwards-compat-tests/slt/kafka/validate_restart.slt
rename to e2e_test/backwards-compat-tests/slt/kafka/validate_restart.slt
diff --git a/backwards-compat-tests/slt/nexmark-backwards-compat/delete.slt.part b/e2e_test/backwards-compat-tests/slt/nexmark-backwards-compat/delete.slt.part
similarity index 100%
rename from backwards-compat-tests/slt/nexmark-backwards-compat/delete.slt.part
rename to e2e_test/backwards-compat-tests/slt/nexmark-backwards-compat/delete.slt.part
diff --git a/backwards-compat-tests/slt/nexmark-backwards-compat/insert.slt b/e2e_test/backwards-compat-tests/slt/nexmark-backwards-compat/insert.slt
similarity index 100%
rename from backwards-compat-tests/slt/nexmark-backwards-compat/insert.slt
rename to e2e_test/backwards-compat-tests/slt/nexmark-backwards-compat/insert.slt
diff --git a/backwards-compat-tests/slt/nexmark-backwards-compat/seed.slt b/e2e_test/backwards-compat-tests/slt/nexmark-backwards-compat/seed.slt
similarity index 100%
rename from backwards-compat-tests/slt/nexmark-backwards-compat/seed.slt
rename to e2e_test/backwards-compat-tests/slt/nexmark-backwards-compat/seed.slt
diff --git a/backwards-compat-tests/slt/nexmark-backwards-compat/validate_original.slt b/e2e_test/backwards-compat-tests/slt/nexmark-backwards-compat/validate_original.slt
similarity index 100%
rename from backwards-compat-tests/slt/nexmark-backwards-compat/validate_original.slt
rename to e2e_test/backwards-compat-tests/slt/nexmark-backwards-compat/validate_original.slt
diff --git a/backwards-compat-tests/slt/nexmark-backwards-compat/validate_restart.slt b/e2e_test/backwards-compat-tests/slt/nexmark-backwards-compat/validate_restart.slt
similarity index 100%
rename from backwards-compat-tests/slt/nexmark-backwards-compat/validate_restart.slt
rename to e2e_test/backwards-compat-tests/slt/nexmark-backwards-compat/validate_restart.slt
diff --git a/backwards-compat-tests/slt/tpch-backwards-compat/delete.slt.part b/e2e_test/backwards-compat-tests/slt/tpch-backwards-compat/delete.slt.part
similarity index 100%
rename from backwards-compat-tests/slt/tpch-backwards-compat/delete.slt.part
rename to e2e_test/backwards-compat-tests/slt/tpch-backwards-compat/delete.slt.part
diff --git a/backwards-compat-tests/slt/tpch-backwards-compat/insert.slt.part b/e2e_test/backwards-compat-tests/slt/tpch-backwards-compat/insert.slt.part
similarity index 100%
rename from backwards-compat-tests/slt/tpch-backwards-compat/insert.slt.part
rename to e2e_test/backwards-compat-tests/slt/tpch-backwards-compat/insert.slt.part
diff --git a/backwards-compat-tests/slt/tpch-backwards-compat/seed.slt b/e2e_test/backwards-compat-tests/slt/tpch-backwards-compat/seed.slt
similarity index 100%
rename from backwards-compat-tests/slt/tpch-backwards-compat/seed.slt
rename to e2e_test/backwards-compat-tests/slt/tpch-backwards-compat/seed.slt
diff --git a/backwards-compat-tests/slt/tpch-backwards-compat/validate_original.slt b/e2e_test/backwards-compat-tests/slt/tpch-backwards-compat/validate_original.slt
similarity index 100%
rename from backwards-compat-tests/slt/tpch-backwards-compat/validate_original.slt
rename to e2e_test/backwards-compat-tests/slt/tpch-backwards-compat/validate_original.slt
diff --git a/backwards-compat-tests/slt/tpch-backwards-compat/validate_restart.slt b/e2e_test/backwards-compat-tests/slt/tpch-backwards-compat/validate_restart.slt
similarity index 100%
rename from backwards-compat-tests/slt/tpch-backwards-compat/validate_restart.slt
rename to e2e_test/backwards-compat-tests/slt/tpch-backwards-compat/validate_restart.slt
diff --git a/e2e_test/batch/catalog/has_privilege.slt.part b/e2e_test/batch/catalog/has_privilege.slt.part
new file mode 100644
index 0000000000000..a742db0c51d6f
--- /dev/null
+++ b/e2e_test/batch/catalog/has_privilege.slt.part
@@ -0,0 +1,264 @@
+statement ok
+CREATE USER test_user;
+
+statement ok
+CREATE SCHEMA test_schema;
+
+statement ok
+CREATE TABLE foo (id INT, name VARCHAR);
+
+statement ok
+CREATE VIEW foo_view AS SELECT * FROM foo;
+
+statement ok
+CREATE INDEX foo_index ON foo(id);
+
+statement ok
+CREATE MATERIALIZED VIEW foo_mv AS SELECT * FROM foo;
+
+statement ok
+CREATE SOURCE foo_source (a int, b int) with (
+    connector = 'datagen',
+    datagen.rows.per.second = '1',
+    datagen.split.num = '1'
+);
+
+statement ok
+CREATE TABLE bar (id INT);
+
+statement ok
+GRANT ALL PRIVILEGES ON foo TO test_user GRANTED BY root;
+
+statement ok
+GRANT INSERT ON bar TO test_user WITH GRANT OPTION GRANTED BY root;
+
+statement ok
+GRANT INSERT ON foo_view TO test_user WITH GRANT OPTION GRANTED BY root;
+
+statement ok
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO test_user WITH GRANT OPTION GRANTED BY root;
+
+statement ok
+GRANT SELECT ON ALL MATERIALIZED VIEWS IN SCHEMA public TO test_user WITH GRANT OPTION GRANTED BY root;
+
+statement ok
+GRANT SELECT ON ALL SOURCES IN SCHEMA public TO test_user WITH GRANT OPTION GRANTED BY root;
+
+statement ok
+GRANT CREATE ON SCHEMA test_schema TO test_user;
+
+query error table not found: bar_err
+GRANT INSERT ON bar_err TO test_user WITH GRANT OPTION GRANTED BY root;
+
+query error Invalid parameter user: User test_user_err not found
+SELECT has_table_privilege('test_user_err', 'foo', 'SELECT');
+
+query error Invalid parameter name: class not found: foo_err
+SELECT has_table_privilege('test_user', 'foo_err', 'SELECT');
+
+query error Invalid parameter privilege: unrecognized privilege type: "SELE CT"
+SELECT has_table_privilege('test_user', 'foo', 'SELE CT');
+
+query error Invalid parameter privilege: unrecognized privilege type: "SELECT INSERT"
+SELECT has_table_privilege('test_user', 'foo', 'SELECT INSERT');
+
+query error Invalid parameter privilege
+SELECT has_table_privilege('test_user', 'foo', 'SELECT, INSERT  WITH GRANT OPTION');
+
+query error Invalid parameter user: User test_user_err not found
+SELECT has_schema_privilege('test_user_err', 'test_schema', 'CREATE');
+
+query error Invalid parameter schema: schema not found: test_schema_err
+SELECT has_schema_privilege('test_user', 'test_schema_err', 'CREATE');
+
+query error Invalid parameter privilege: unrecognized privilege type: "INSERT"
+SELECT has_schema_privilege('test_user', 'test_schema', 'INSERT');
+
+query error Invalid parameter privilege: unrecognized privilege type: "DELETE"
+SELECT has_any_column_privilege('test_user', 'foo_mv'::regclass, 'DELETE');
+
+query I
+SELECT has_table_privilege('test_user', 'foo', 'SELECT');
+----
+t
+
+query I
+SELECT has_table_privilege('test_user', 'foo', 'SELECT WITH GRANT OPTION');
+----
+t
+
+query I
+SELECT has_table_privilege('test_user', 'foo', 'INSERT WITH GRANT OPTION');
+----
+f
+
+query I
+SELECT has_table_privilege('test_user', 'foo', 'INSERT, SELECT WITH GRANT OPTION');
+----
+t
+
+query I
+SELECT has_table_privilege('test_user', 'foo', 'DELETE, INSERT, SELECT WITH GRANT OPTION');
+----
+t
+
+query I
+SELECT has_table_privilege('test_user', 'foo', 'DELETE WITH GRANT OPTION, INSERT, SELECT WITH GRANT OPTION');
+----
+f
+
+query I
+SELECT has_table_privilege('test_user', 'foo_view', 'SELECT');
+----
+t
+
+query I
+SELECT has_table_privilege('test_user', 'foo_view'::regclass, 'INSERT');
+----
+t
+
+query I
+SELECT has_table_privilege('test_user', 'foo_view'::regclass, 'UPDATE');
+----
+f
+
+query I
+SELECT has_any_column_privilege('test_user', 'foo_view'::regclass, 'INSERT');
+----
+t
+
+query I
+SELECT has_table_privilege('test_user', 'foo_mv', 'SELECT');
+----
+t
+
+query I
+SELECT has_table_privilege('test_user', 'foo_mv'::regclass, 'SELECT WITH GRANT OPTION');
+----
+t
+
+query I
+SELECT has_any_column_privilege('test_user', 'foo_mv'::regclass, 'SELECT WITH GRANT OPTION');
+----
+t
+
+query I
+SELECT has_table_privilege('test_user', 'foo_mv', 'INSERT');
+----
+f
+
+query I
+SELECT has_table_privilege('test_user', 'foo_source'::regclass, 'SELECT');
+----
+t
+
+query I
+SELECT has_table_privilege('test_user', 'foo_source', 'INSERT');
+----
+f
+
+# Indexes are granted by `GRANT SELECT ON ALL MATERIALIZED VIEWS`
+query I
+SELECT has_table_privilege('test_user', 'foo_index'::regclass, 'SELECT');
+----
+t
+
+query I
+SELECT has_table_privilege('test_user', 'foo_index', 'INSERT');
+----
+f
+
+query I
+SELECT has_table_privilege('test_user', 'bar', 'INSERT');
+----
+t
+
+query I
+SELECT has_table_privilege('bar', 'INSERT');
+----
+t
+
+query I
+SELECT has_table_privilege('bar'::regclass, 'SELECT');
+----
+t
+
+query I
+SELECT has_table_privilege('bar'::regclass, 'SELECT');
+----
+t
+
+query I
+SELECT has_table_privilege('test_user', 'bar', 'UPDATE');
+----
+f
+
+query I
+SELECT has_table_privilege('test_user', 'bar'::regclass, 'INSERT WITH GRANT OPTION');
+----
+t
+
+query I
+SELECT has_schema_privilege('public', 'USAGE');
+----
+t
+
+query I
+SELECT has_schema_privilege('test_user', 'test_schema', 'USAGE');
+----
+f
+
+query I
+SELECT has_schema_privilege('test_user', 'test_schema', 'CREATE');
+----
+t
+
+statement ok
+REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM test_user GRANTED BY root;
+
+query I
+SELECT has_table_privilege('test_user', 'bar'::regclass, 'SELECT');
+----
+f
+
+query I
+SELECT has_table_privilege('test_user', 'foo_view', 'SELECT');
+----
+f
+
+query I
+SELECT has_table_privilege('test_user', 'foo_view', 'INSERT');
+----
+t
+
+statement ok
+REVOKE INSERT ON foo_view FROM test_user GRANTED BY root;
+
+query I
+SELECT has_table_privilege('test_user', 'foo_view', 'INSERT');
+----
+f
+
+statement ok
+DROP SOURCE foo_source;
+
+statement ok
+DROP MATERIALIZED VIEW foo_mv;
+
+statement ok
+DROP INDEX foo_index;
+
+statement ok
+DROP VIEW foo_view;
+
+statement ok
+DROP TABLE foo;
+
+statement ok
+DROP TABLE bar;
+
+statement ok
+DROP SCHEMA test_schema;
+
+statement ok
+DROP USER test_user;
diff --git a/e2e_test/batch/catalog/pg_class.slt.part b/e2e_test/batch/catalog/pg_class.slt.part
index ffb53e32d66b5..a6fe4a1257122 100644
--- a/e2e_test/batch/catalog/pg_class.slt.part
+++ b/e2e_test/batch/catalog/pg_class.slt.part
@@ -20,4 +20,4 @@ SELECT oid,relname,relowner,relkind FROM pg_catalog.pg_class ORDER BY oid limit
 query ITIT
 SELECT oid,relname,relowner,relkind FROM pg_catalog.pg_class WHERE oid = 'pg_namespace'::regclass;
 ----
-2147478671 pg_namespace 1 v
+2147478670 pg_namespace 1 v
diff --git a/e2e_test/batch/catalog/pg_settings.slt.part b/e2e_test/batch/catalog/pg_settings.slt.part
index d9af757ba4c36..2a130de04c19c 100644
--- a/e2e_test/batch/catalog/pg_settings.slt.part
+++ b/e2e_test/batch/catalog/pg_settings.slt.part
@@ -19,6 +19,7 @@ user background_ddl
 user batch_enable_distributed_dml
 user batch_parallelism
 user bytea_output
+user cdc_source_wait_streaming_start_timeout
 user client_encoding
 user client_min_messages
 user create_compaction_group_for_mv
diff --git a/e2e_test/batch/functions/to_char.slt.part b/e2e_test/batch/functions/to_char.slt.part
index b4d10ec34d49c..4398efa3ff9cf 100644
--- a/e2e_test/batch/functions/to_char.slt.part
+++ b/e2e_test/batch/functions/to_char.slt.part
@@ -106,51 +106,17 @@ select to_char('-23:22:57.124562'::interval, 'HH12 MI SS MS US');
 ----
 -11 -22 -57 -124 -124562
 
-query error
+query error invalid format specification for an interval value
 select to_char('1year 1month 1day 1hours 1minute 1second'::interval, 'IY MM DD AM HH12 MM SS tzhtzm');
-----
-db error: ERROR: Failed to run the query
-
-Caused by these errors (recent errors listed first):
-  1: Expr error
-  2: Invalid parameter pattern: invalid format specification for an interval value, HINT: Intervals are not tied to specific calendar dates.
 
-
-query error
+query error invalid format specification for an interval value
 select to_char('1year 1month 1day 1hours 1minute 1second'::interval, 'IY MM DD AM HH12 MI SS TZH:TZM');
-----
-db error: ERROR: Failed to run the query
 
-Caused by these errors (recent errors listed first):
-  1: Expr error
-  2: Invalid parameter pattern: invalid format specification for an interval value, HINT: Intervals are not tied to specific calendar dates.
-
-
-query error
+query error invalid format specification for an interval value
 select to_char('1year 1month 1day 1hours 1minute 1second'::interval, 'IY MM DD AM HH12 MI SS TZH');
-----
-db error: ERROR: Failed to run the query
 
-Caused by these errors (recent errors listed first):
-  1: Expr error
-  2: Invalid parameter pattern: invalid format specification for an interval value, HINT: Intervals are not tied to specific calendar dates.
-
-
-query error
+query error invalid format specification for an interval value
 select to_char('1year 1month 1day 1hours 1minute 1second'::interval, 'IY MM DD AM HH12 MI SS Month');
-----
-db error: ERROR: Failed to run the query
-
-Caused by these errors (recent errors listed first):
-  1: Expr error
-  2: Invalid parameter pattern: invalid format specification for an interval value, HINT: Intervals are not tied to specific calendar dates.
 
-
-query error
+query error invalid format specification for an interval value
 select to_char('1year 1month 1day 1hours 1minute 1second'::interval, 'IY MM DD AM HH12 MI SS Mon');
-----
-db error: ERROR: Failed to run the query
-
-Caused by these errors (recent errors listed first):
-  1: Expr error
-  2: Invalid parameter pattern: invalid format specification for an interval value, HINT: Intervals are not tied to specific calendar dates.
diff --git a/e2e_test/batch/over_window/main.slt.part b/e2e_test/batch/over_window/main.slt.part
index dca370c0306ee..9ef0a7572380d 100644
--- a/e2e_test/batch/over_window/main.slt.part
+++ b/e2e_test/batch/over_window/main.slt.part
@@ -1 +1,2 @@
 include ./generated/main.slt.part
+include ./session/mod.slt.part
diff --git a/e2e_test/batch/over_window/session/mod.slt.part b/e2e_test/batch/over_window/session/mod.slt.part
new file mode 100644
index 0000000000000..eb0e09675cff7
--- /dev/null
+++ b/e2e_test/batch/over_window/session/mod.slt.part
@@ -0,0 +1,61 @@
+# Because currently general streaming version of session window is not supported yet,
+# we only add e2e for batch mode.
+
+statement ok
+create table t (
+    tm timestamp,
+    foo int,
+    bar int
+);
+
+statement ok
+create view v1 as
+select
+    *,
+    first_value(tm) over (partition by bar order by tm session with gap '10 minutes') as window_start,
+    last_value(tm) over (partition by bar order by tm session with gap '10 minutes') as window_end
+from t;
+
+statement ok
+insert into t values
+  ('2023-05-06 16:51:00', 1, 100)
+, ('2023-05-06 16:56:00', 8, 100)
+, ('2023-05-06 17:30:00', 3, 200)
+, ('2023-05-06 17:35:00', 5, 100)
+, ('2023-05-06 17:59:00', 4, 100)
+, ('2023-05-06 18:01:00', 6, 200)
+;
+
+query TiiTT
+select * from v1 order by tm;
+----
+2023-05-06 16:51:00	1	100	2023-05-06 16:51:00	2023-05-06 16:56:00
+2023-05-06 16:56:00	8	100	2023-05-06 16:51:00	2023-05-06 16:56:00
+2023-05-06 17:30:00	3	200	2023-05-06 17:30:00	2023-05-06 17:30:00
+2023-05-06 17:35:00	5	100	2023-05-06 17:35:00	2023-05-06 17:35:00
+2023-05-06 17:59:00	4	100	2023-05-06 17:59:00	2023-05-06 17:59:00
+2023-05-06 18:01:00	6	200	2023-05-06 18:01:00	2023-05-06 18:01:00
+
+statement ok
+insert into t values
+  ('2023-05-06 18:08:00', 7, 100)
+, ('2023-05-06 18:10:00', 9, 200)
+;
+
+query TiiTT
+select * from v1 order by tm;
+----
+2023-05-06 16:51:00	1	100	2023-05-06 16:51:00	2023-05-06 16:56:00
+2023-05-06 16:56:00	8	100	2023-05-06 16:51:00	2023-05-06 16:56:00
+2023-05-06 17:30:00	3	200	2023-05-06 17:30:00	2023-05-06 17:30:00
+2023-05-06 17:35:00	5	100	2023-05-06 17:35:00	2023-05-06 17:35:00
+2023-05-06 17:59:00	4	100	2023-05-06 17:59:00	2023-05-06 18:08:00
+2023-05-06 18:01:00	6	200	2023-05-06 18:01:00	2023-05-06 18:10:00
+2023-05-06 18:08:00	7	100	2023-05-06 17:59:00	2023-05-06 18:08:00
+2023-05-06 18:10:00	9	200	2023-05-06 18:01:00	2023-05-06 18:10:00
+
+statement ok
+drop view v1;
+
+statement ok
+drop table t;
diff --git a/e2e_test/commands/README.md b/e2e_test/commands/README.md
new file mode 100644
index 0000000000000..56f51159d844e
--- /dev/null
+++ b/e2e_test/commands/README.md
@@ -0,0 +1,8 @@
+# "built-in" Sqllogictest system commands
+
+Scripts (executables) in this directory are expected to be used as `system` commands in sqllogictests. You can use any language like bash, python, zx.
+
+They will be loaded in `PATH` by `risedev slt`, and thus function as kind of "built-in" commands.
+
+Only general commands should be put here.
+If the script is ad-hoc (only used for one test), it's better to put next to the test file.
diff --git a/e2e_test/commands/internal_table.mjs b/e2e_test/commands/internal_table.mjs
new file mode 100755
index 0000000000000..b51cc0313dca2
--- /dev/null
+++ b/e2e_test/commands/internal_table.mjs
@@ -0,0 +1,67 @@
+#!/usr/bin/env zx
+
+// zx: A tool for writing better scripts
+// https://google.github.io/zx/
+
+const {
+  name: job_name,
+  type: table_type,
+  count: count,
+} = minimist(process.argv.slice(3), {
+  string: ["name", "type"],
+  boolean: ["count"],
+});
+
+// Return an array of CSV string
+async function psql(query) {
+  return (
+    await $`
+psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d dev \
+--csv --tuples-only -c ${query}
+`
+  )
+    .toString()
+    .trim()
+    .split("\n")
+    .filter((line) => line.trim() != "");
+}
+
+// If `table_type` is null, return all internal tables for the job.
+// If `job_name` is null, return all jobs' internal tables.
+async function select_internal_table(job_name, table_type) {
+  // Note: if we have `t1`, and `t1_balabala`, the latter one will also be matched 😄.
+  const internal_tables = await psql(
+    `select name from rw_internal_tables where name like '__internal_${job_name}_%_${table_type}_%'`
+  );
+  if (internal_tables.length == 0) {
+    throw new Error(
+      `No internal tables found for the pattern '__internal_${job_name}_%_${table_type}_%'`
+    );
+  }
+
+  const res = new Map(
+    await Promise.all(
+      internal_tables.map(async (t) => {
+        let rows = await psql(`select * from ${t}`);
+        return [t, rows];
+      })
+    )
+  );
+  return res;
+}
+
+const tables = await select_internal_table(job_name, table_type);
+for (const [table_name, rows] of tables) {
+  if (tables.size > 1) {
+    console.log(`Table: ${table_name}`);
+  }
+  if (count) {
+    console.log(`count: ${rows.length}`);
+  } else {
+    if (rows.length == 0) {
+      console.log("(empty)");
+    } else {
+      console.log(`${rows.join("\n")}`);
+    }
+  }
+}
diff --git a/e2e_test/commands/mysql b/e2e_test/commands/mysql
new file mode 100755
index 0000000000000..881ecd93811ac
--- /dev/null
+++ b/e2e_test/commands/mysql
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# A simple wrapper around `mysql` but add an `-u` option to specify the user.
+# The environment variable `RISEDEV_MYSQL_USER` will be set by `risedev-env`.
+
+# Strip the current directory from the PATH to access the system `mysql`.
+REAL_PATH=$(echo "$PATH" | sed "s,$(dirname "$0"):,,g")
+
+PATH=${REAL_PATH} mysql -u "${RISEDEV_MYSQL_USER}" "$@"
diff --git a/e2e_test/ddl/alter_parallelism.slt b/e2e_test/ddl/alter_parallelism.slt
index 4d774321f1d64..025496ca1c571 100644
--- a/e2e_test/ddl/alter_parallelism.slt
+++ b/e2e_test/ddl/alter_parallelism.slt
@@ -13,9 +13,6 @@ create view mview_parallelism as select m.name, tf.parallelism from rw_materiali
 statement ok
 create view sink_parallelism as select s.name, tf.parallelism from rw_sinks s, rw_table_fragments tf where s.id = tf.table_id;
 
-statement ok
-create view subscription_parallelism as select s.name, tf.parallelism from rw_subscriptions s, rw_table_fragments tf where s.id = tf.table_id;
-
 statement ok
 create view fragment_parallelism as select t.name as table_name, f.fragment_id, f.parallelism from rw_fragments f, rw_tables t where f.table_id = t.id;
 
@@ -97,28 +94,9 @@ select parallelism from sink_parallelism where name = 's';
 ----
 FIXED(4)
 
-statement ok
-create subscription subscription1 from t with (retention = '1D');
-
-query T
-select parallelism from subscription_parallelism where name = 'subscription1';
-----
-ADAPTIVE
-
-statement ok
-alter subscription subscription1 set parallelism = 4;
-
-query T
-select parallelism from subscription_parallelism where name = 'subscription1';
-----
-FIXED(4)
-
 statement ok
 drop sink s;
 
-statement ok
-drop subscription subscription1;
-
 statement ok
 drop materialized view m_join;
 
@@ -179,8 +157,5 @@ drop view mview_parallelism;
 statement ok
 drop view sink_parallelism;
 
-statement ok
-drop view subscription_parallelism;
-
 statement ok
 drop view fragment_parallelism;
\ No newline at end of file
diff --git a/e2e_test/ddl/alter_set_schema.slt b/e2e_test/ddl/alter_set_schema.slt
index 9ea95e9cc09c2..74dcc5a77e64a 100644
--- a/e2e_test/ddl/alter_set_schema.slt
+++ b/e2e_test/ddl/alter_set_schema.slt
@@ -53,13 +53,14 @@ CREATE SOURCE test_source (v INT) WITH (
 statement ok
 ALTER SOURCE test_source SET SCHEMA test_schema;
 
-query TT
+query TT rowsort
 SELECT name AS sourcename, nspname AS schemaname
 FROM rw_sources
 JOIN pg_namespace ON pg_namespace.oid = rw_sources.schema_id
 WHERE nspname = 'test_schema';
 ----
 test_source test_schema
+test_table test_schema
 
 statement ok
 CREATE SINK test_sink AS SELECT u FROM test_schema.test_table WITH (
diff --git a/e2e_test/ddl/schema.slt b/e2e_test/ddl/schema.slt
index 218b4bf08bb08..d998233b3b088 100644
--- a/e2e_test/ddl/schema.slt
+++ b/e2e_test/ddl/schema.slt
@@ -67,3 +67,35 @@ drop schema Myschema;
 
 statement ok
 drop schema "Myschema";
+
+# test derived schema name and authorization with user
+statement ok
+create user user_for_authorization;
+
+statement ok
+create schema if not exists authorization user_for_authorization;
+
+statement ok
+create schema myschema authorization user_for_authorization;
+
+query T rowsort
+show schemas;
+----
+information_schema
+myschema
+pg_catalog
+public
+rw_catalog
+user_for_authorization
+
+statement error Permission denied
+drop user user_for_authorization;
+
+statement ok
+drop schema myschema;
+
+statement ok
+drop schema user_for_authorization;
+
+statement ok
+drop user user_for_authorization;
diff --git a/e2e_test/ddl/secret.slt b/e2e_test/ddl/secret.slt
new file mode 100644
index 0000000000000..7c11e2e6245a3
--- /dev/null
+++ b/e2e_test/ddl/secret.slt
@@ -0,0 +1,23 @@
+statement error secret backend "fake-backend" is not supported
+create secret secret_1 with (
+  backend = 'fake-backend'
+) as 'demo_secret';
+
+statement ok
+create secret secret_1 with (
+  backend = 'meta'
+) as 'demo_secret';
+
+# wait for support for hashicorp_vault backend
+# statement ok
+# create secret secret_2 with (
+# backend = 'hashicorp_vault'
+# );
+
+query T
+show secrets;
+----
+secret_1
+
+statement ok
+drop secret secret_1;
diff --git a/e2e_test/ddl/show.slt b/e2e_test/ddl/show.slt
index f2bd63b4e3768..db9894459071f 100644
--- a/e2e_test/ddl/show.slt
+++ b/e2e_test/ddl/show.slt
@@ -215,3 +215,37 @@ tablespace character varying false NULL
 hasindexes boolean false NULL
 ispopulated boolean false NULL
 definition character varying false NULL
+
+# show tables according to search path
+
+statement ok
+create schema show_another_schema;
+
+statement ok
+create table show_another_schema.t_another (v1 int);
+
+statement ok
+create table t4 (v1 int);
+
+query T
+show tables;
+----
+t4
+
+statement ok
+set search_path to show_another_schema, public;
+
+query T
+show tables;
+----
+t_another
+t4
+
+statement ok
+drop table t4;
+
+statement ok
+drop table show_another_schema.t_another;
+
+statement ok
+drop schema show_another_schema;
diff --git a/e2e_test/error_ui/extended/main.slt b/e2e_test/error_ui/extended/main.slt
index 67db6ccf3393f..6b9be9d26b14f 100644
--- a/e2e_test/error_ui/extended/main.slt
+++ b/e2e_test/error_ui/extended/main.slt
@@ -4,8 +4,9 @@ selet 1;
 db error: ERROR: Failed to prepare the statement
 
 Caused by:
-  sql parser error: Expected an SQL statement, found: selet at line:1, column:6
-Near "selet"
+  sql parser error: expected statement, found: selet
+LINE 1: selet 1;
+        ^
 
 
 query error
@@ -15,4 +16,5 @@ db error: ERROR: Failed to execute the statement
 
 Caused by these errors (recent errors listed first):
   1: Expr error
-  2: Division by zero
+  2: error while evaluating expression `general_div('1', '0')`
+  3: Division by zero
diff --git a/e2e_test/error_ui/simple/expr.slt b/e2e_test/error_ui/simple/expr.slt
new file mode 100644
index 0000000000000..b983baae99a01
--- /dev/null
+++ b/e2e_test/error_ui/simple/expr.slt
@@ -0,0 +1,71 @@
+# Regular function
+query error
+select pow(114, 514);
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: Expr error
+  2: error while evaluating expression `pow_f64('114', '514')`
+  3: Numeric out of range: overflow
+
+
+# Nullable arguments
+query error
+select array_position(array[1, null, 2, null], null, null::int);
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: Expr error
+  2: error while evaluating expression `array_position_start('{1,NULL,2,NULL}', NULL, NULL)`
+  3: Invalid parameter start: initial position must not be null
+
+
+# Operator
+query error
+select 11111111444 * 51444444444444444;
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: Expr error
+  2: error while evaluating expression `general_mul('11111111444', '51444444444444444')`
+  3: Numeric out of range
+
+
+# Cast
+query error
+select 'foo'::bigint;
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: Expr error
+  2: error while evaluating expression `str_parse('foo')`
+  3: Parse error: bigint invalid digit found in string
+
+
+# Prebuild context
+# TODO: not included in the error message
+query error
+select jsonb_path_exists('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '"foo"');
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: Expr error
+  2: error while evaluating expression `jsonb_path_exists3('{"a": [1, 2, 3, 4, 5]}', '"foo"')`
+  3: Invalid parameter jsonpath: "vars" argument is not an object
+
+
+# Variadic arguments
+query error
+select format('%L %s', 'Hello', 'World');
+----
+db error: ERROR: Failed to run the query
+
+Caused by these errors (recent errors listed first):
+  1: Expr error
+  2: error while evaluating expression `format('Hello', 'World')`
+  3: Unsupported function: unsupported specifier type 'L'
diff --git a/e2e_test/error_ui/simple/main.slt b/e2e_test/error_ui/simple/main.slt
index 8ef82e1f0d1c7..f09e47302f3c6 100644
--- a/e2e_test/error_ui/simple/main.slt
+++ b/e2e_test/error_ui/simple/main.slt
@@ -4,8 +4,9 @@ selet 1;
 db error: ERROR: Failed to run the query
 
 Caused by:
-  sql parser error: Expected an SQL statement, found: selet at line:1, column:6
-Near "selet"
+  sql parser error: expected statement, found: selet
+LINE 1: selet 1;
+        ^
 
 
 statement error
@@ -13,8 +14,9 @@ create function int_42() returns int as int_42 using link '555.0.0.1:8815';
 ----
 db error: ERROR: Failed to run the query
 
-Caused by:
-  Flight service error: invalid address: 555.0.0.1:8815, err: failed to parse address: http://555.0.0.1:8815: invalid IPv4 address
+Caused by these errors (recent errors listed first):
+  1: failed to parse address: http://555.0.0.1:8815
+  2: invalid IPv4 address
 
 
 statement error
@@ -59,7 +61,8 @@ db error: ERROR: Failed to run the query
 
 Caused by these errors (recent errors listed first):
   1: Expr error
-  2: Division by zero
+  2: error while evaluating expression `general_div('1', '0')`
+  3: Division by zero
 
 
 query error
@@ -69,7 +72,8 @@ db error: ERROR: Failed to run the query
 
 Caused by these errors (recent errors listed first):
   1: Expr error
-  2: Division by zero
+  2: error while evaluating expression `general_div('1', '0')`
+  3: Division by zero
 
 
 statement error
diff --git a/e2e_test/error_ui/simple/recovery.slt b/e2e_test/error_ui/simple/recovery.slt
index b47afc94c47ad..e3830be6d25c8 100644
--- a/e2e_test/error_ui/simple/recovery.slt
+++ b/e2e_test/error_ui/simple/recovery.slt
@@ -1,15 +1,15 @@
-# TODO: the test triggers a recovery caused by a known issue: https://github.com/risingwavelabs/risingwave/issues/12474.
+# TODO: the test triggers a recovery caused by a known issue: https://github.com/risingwavelabs/risingwave/issues/11915.
 # We should consider using a mechanism designed for testing recovery instead of depending on a bug.
 
 statement ok
-create table t (v int);
+create table t (v decimal);
 
 statement ok
-create materialized view mv as select generate_series(1, 10), coalesce(pg_sleep(2), v) / 0 bomb from t;
+create materialized view mv as select sum(coalesce(pg_sleep(1), v)) from t;
 
-# The bomb will be triggered after 2 seconds of sleep, so the insertion should return successfully.
+# The bomb will be triggered after 1 seconds of sleep, so the insertion should return successfully.
 statement ok
-insert into t values (1);
+insert into t values (4e28), (4e28);
 
 # Wait for recovery to complete.
 sleep 15s
@@ -25,7 +25,7 @@ with error as (
     limit 1
 )
 select
-case when error like '%Actor % exited unexpectedly: Executor error: Chunk operation error: Division by zero%' then 'ok'
+case when error like '%Actor % exited unexpectedly: Executor error: %Numeric out of range%' then 'ok'
      else error
 end as result
 from error;
diff --git a/e2e_test/schema_registry/alter_sr.slt b/e2e_test/schema_registry/alter_sr.slt
index 8daf41d87b633..d703c0401a35e 100644
--- a/e2e_test/schema_registry/alter_sr.slt
+++ b/e2e_test/schema_registry/alter_sr.slt
@@ -9,7 +9,7 @@ CREATE SOURCE src_user WITH (
     scan.startup.mode = 'earliest'
 )
 FORMAT PLAIN ENCODE PROTOBUF(
-    schema.registry = 'http://message_queue:8081',
+    schema.registry = 'http://schemaregistry:8082',
     message = 'test.User'
 );
 
@@ -24,7 +24,7 @@ CREATE TABLE t_user WITH (
     scan.startup.mode = 'earliest'
 )
 FORMAT PLAIN ENCODE PROTOBUF(
-    schema.registry = 'http://message_queue:8081',
+    schema.registry = 'http://schemaregistry:8082',
     message = 'test.User'
 );
 
@@ -36,7 +36,7 @@ SELECT age FROM t_user;
 
 # Push more events with extended fields
 system ok
-python3 e2e_test/schema_registry/pb.py "message_queue:29092" "http://message_queue:8081" "sr_pb_test" 5 user_with_more_fields
+python3 e2e_test/schema_registry/pb.py "message_queue:29092" "http://schemaregistry:8082" "sr_pb_test" 5 user_with_more_fields
 
 sleep 5s
 
@@ -58,7 +58,7 @@ SELECT COUNT(*), MAX(age), MIN(age), SUM(age) FROM mv_user_more;
 
 # Push more events with extended fields
 system ok
-python3 e2e_test/schema_registry/pb.py "message_queue:29092" "http://message_queue:8081" "sr_pb_test" 5 user_with_more_fields
+python3 e2e_test/schema_registry/pb.py "message_queue:29092" "http://schemaregistry:8082" "sr_pb_test" 5 user_with_more_fields
 
 sleep 5s
 
diff --git a/e2e_test/schema_registry/pb.slt b/e2e_test/schema_registry/pb.slt
index d9c0edca1b21c..7b60b4fa8d7a4 100644
--- a/e2e_test/schema_registry/pb.slt
+++ b/e2e_test/schema_registry/pb.slt
@@ -9,7 +9,7 @@ create table sr_pb_test with (
     properties.bootstrap.server = 'message_queue:29092',
     scan.startup.mode = 'earliest')
 FORMAT plain ENCODE protobuf(
-        schema.registry = 'http://message_queue:8081',
+        schema.registry = 'http://schemaregistry:8082',
         message = 'test.User'
     );
 
@@ -21,7 +21,7 @@ create table sr_pb_test_bk with (
     properties.bootstrap.server = 'message_queue:29092',
     scan.startup.mode = 'earliest')
 FORMAT plain ENCODE protobuf(
-        schema.registry = 'http://message_queue:8081,http://message_queue:8081',
+        schema.registry = 'http://schemaregistry:8082,http://schemaregistry:8082',
         message = 'test.User'
     );
 
diff --git a/e2e_test/sink/clickhouse_sink.slt b/e2e_test/sink/clickhouse_sink.slt
index 2adc70dcf409e..e5bac0d8d521d 100644
--- a/e2e_test/sink/clickhouse_sink.slt
+++ b/e2e_test/sink/clickhouse_sink.slt
@@ -1,3 +1,6 @@
+statement ok
+set sink_decouple = false;
+
 statement ok
 CREATE TABLE t6 (v1 int primary key, v2 bigint, v3 varchar, v4 smallint, v5 decimal);
 
diff --git a/e2e_test/sink/kafka/avro.slt b/e2e_test/sink/kafka/avro.slt
index d9fa53bc589ac..e0e339eead402 100644
--- a/e2e_test/sink/kafka/avro.slt
+++ b/e2e_test/sink/kafka/avro.slt
@@ -1,3 +1,6 @@
+statement ok
+set sink_decouple = false;
+
 statement ok
 create table from_kafka ( *, gen_i32_field int as int32_field + 2, primary key (some_key) )
 include key as some_key
@@ -6,7 +9,7 @@ with (
   topic = 'test-rw-sink-upsert-avro',
   properties.bootstrap.server = 'message_queue:29092')
 format upsert encode avro (
-  schema.registry = 'http://message_queue:8081');
+  schema.registry = 'http://schemaregistry:8082');
 
 statement ok
 create table into_kafka (
@@ -40,10 +43,128 @@ create sink sink0 from into_kafka with (
   properties.bootstrap.server = 'message_queue:29092',
   primary_key = 'int32_field,string_field')
 format upsert encode avro (
-  schema.registry = 'http://message_queue:8081');
+  schema.registry = 'http://schemaregistry:8082');
+
+system ok
+rpk topic create test-rw-sink-plain-avro
+
+system ok
+jq '{"schema": tojson}' << EOF | curl -X POST -H 'content-type: application/json' -d @- 'http://schemaregistry:8082/subjects/test-rw-sink-plain-avro-value/versions'
+{
+  "type": "record",
+  "name": "Simple",
+  "fields": [
+    {
+      "name": "int32_field",
+      "type": ["null", "int"]
+    },
+    {
+      "name": "string_field",
+      "type": ["null", "string"]
+    }
+  ]
+}
+EOF
+
+statement ok
+create table from_kafka_plain
+include key as raw_key
+with (
+  connector = 'kafka',
+  topic = 'test-rw-sink-plain-avro',
+  properties.bootstrap.server = 'message_queue:29092')
+format plain encode avro (
+  schema.registry = 'http://schemaregistry:8082');
+
+statement ok
+create sink sink_plain_key_none as select int32_field, string_field from into_kafka with (
+  connector = 'kafka',
+  topic = 'test-rw-sink-plain-avro',
+  properties.bootstrap.server = 'message_queue:29092')
+format plain encode avro (
+  force_append_only = true,
+  schema.registry = 'http://schemaregistry:8082');
+
+statement ok
+create sink sink_plain_key_text as select int32_field, string_field from into_kafka with (
+  connector = 'kafka',
+  topic = 'test-rw-sink-plain-avro',
+  properties.bootstrap.server = 'message_queue:29092',
+  primary_key = 'int32_field')
+format plain encode avro (
+  force_append_only = true,
+  schema.registry = 'http://schemaregistry:8082')
+key encode text;
+
+system ok
+jq '{"schema": tojson}' << EOF | curl -X POST -H 'content-type: application/json' -d @- 'http://schemaregistry:8082/subjects/test-rw-sink-plain-avro-key/versions'
+{
+  "type": "record",
+  "name": "Key",
+  "fields": [
+    {
+      "name": "int32_field",
+      "type": ["null", "int"]
+    }
+  ]
+}
+EOF
+
+statement ok
+create sink sink_plain_key_avro as select int32_field, string_field from into_kafka with (
+  connector = 'kafka',
+  topic = 'test-rw-sink-plain-avro',
+  properties.bootstrap.server = 'message_queue:29092',
+  primary_key = 'int32_field')
+format plain encode avro (
+  force_append_only = true,
+  schema.registry = 'http://schemaregistry:8082');
 
 sleep 2s
 
+query ITT
+select
+  int32_field,
+  string_field,
+  case when octet_length(raw_key) > 5 and substring(raw_key from 1 for 1) = '\x00'
+  then substring(raw_key from 6) -- skip indeterministic confluent schema registry header
+  else raw_key end
+from from_kafka_plain order by string_field, raw_key;
+----
+22 Rising \x022c
+22 Rising \x3232
+22 Rising NULL
+11 Wave   \x0216
+11 Wave   \x3131
+11 Wave   NULL
+
+statement ok
+drop sink sink_plain_key_avro;
+
+statement ok
+drop sink sink_plain_key_text;
+
+statement ok
+drop sink sink_plain_key_none;
+
+statement ok
+drop table from_kafka_plain;
+
+system ok
+curl -X DELETE 'http://schemaregistry:8082/subjects/test-rw-sink-plain-avro-key'
+
+system ok
+curl -X DELETE 'http://schemaregistry:8082/subjects/test-rw-sink-plain-avro-key?permanent=true'
+
+system ok
+curl -X DELETE 'http://schemaregistry:8082/subjects/test-rw-sink-plain-avro-value'
+
+system ok
+curl -X DELETE 'http://schemaregistry:8082/subjects/test-rw-sink-plain-avro-value?permanent=true'
+
+system ok
+rpk topic delete test-rw-sink-plain-avro
+
 query TTTRRIITTTTTTTT
 select
   bool_field,
@@ -72,7 +193,7 @@ create sink sink_err from into_kafka with (
   properties.bootstrap.server = 'message_queue:29092',
   primary_key = 'int32_field,string_field')
 format upsert encode avro (
-  schema.registry = 'http://message_queue:8081');
+  schema.registry = 'http://schemaregistry:8082');
 
 statement error field not in avro
 create sink sink_err as select 1 as extra_column, * from into_kafka with (
@@ -81,7 +202,7 @@ create sink sink_err as select 1 as extra_column, * from into_kafka with (
   properties.bootstrap.server = 'message_queue:29092',
   primary_key = 'int32_field,string_field')
 format upsert encode avro (
-  schema.registry = 'http://message_queue:8081');
+  schema.registry = 'http://schemaregistry:8082');
 
 statement error unrecognized
 create sink sink_err from into_kafka with (
@@ -90,7 +211,7 @@ create sink sink_err from into_kafka with (
   properties.bootstrap.server = 'message_queue:29092',
   primary_key = 'int32_field,string_field')
 format upsert encode avro (
-  schema.registry = 'http://message_queue:8081',
+  schema.registry = 'http://schemaregistry:8082',
   schema.registry.name.strategy = 'typo');
 
 statement error empty field key.message
@@ -100,7 +221,7 @@ create sink sink_err from into_kafka with (
   properties.bootstrap.server = 'message_queue:29092',
   primary_key = 'int32_field,string_field')
 format upsert encode avro (
-  schema.registry = 'http://message_queue:8081',
+  schema.registry = 'http://schemaregistry:8082',
   schema.registry.name.strategy = 'record_name_strategy');
 
 statement ok
diff --git a/e2e_test/sink/kafka/create_sink.slt b/e2e_test/sink/kafka/create_sink.slt
index 1cdb4771ffd47..f5e2e0333bc35 100644
--- a/e2e_test/sink/kafka/create_sink.slt
+++ b/e2e_test/sink/kafka/create_sink.slt
@@ -1,6 +1,9 @@
 statement ok
 set rw_implicit_flush=true;
 
+statement ok
+set sink_decouple = false;
+
 statement ok
 create table t_kafka (
     id integer primary key,
@@ -183,6 +186,61 @@ create sink invalid_pk_column from t_kafka with (
     primary_key = 'id,invalid'
 );
 
+### Test sink with key encode ###
+
+statement error sink key encode unsupported: JSON, only TEXT supported
+create sink sink_text_error from t_kafka with (
+  connector = 'kafka',
+  properties.bootstrap.server = 'message_queue:29092',
+  topic = 'test-rw-sink-text-key-id',
+  primary_key = 'id')
+format plain encode json (
+  force_append_only='true'
+) key encode json ;
+
+statement error
+# The key encode is TEXT, but the primary key has 2 columns. The key encode TEXT requires the primary key to be a single column.s
+create sink sink_text_error from t_kafka with (
+  connector = 'kafka',
+  properties.bootstrap.server = 'message_queue:29092',
+  topic = 'test-rw-sink-text-key-id',
+  primary_key = 'id, v_varchar')
+format plain encode json (
+  force_append_only='true'
+) key encode text ;
+
+statement error
+# The key encode is TEXT, but the primary key column v_bytea has type bytea. The key encode TEXT requires the primary key column to be of type varchar, bool, small int, int, or big int.
+create sink sink_text_error from t_kafka with (
+  connector = 'kafka',
+  properties.bootstrap.server = 'message_queue:29092',
+  topic = 'test-rw-sink-text-key-id',
+  primary_key = 'v_bytea')
+format plain encode json (
+  force_append_only='true'
+) key encode text ;
+
+statement ok
+create sink sink_text_id from t_kafka with (
+  connector = 'kafka',
+  properties.bootstrap.server = 'message_queue:29092',
+  topic = 'test-rw-sink-text-key-id',
+  primary_key = 'id')
+format plain encode json (
+  force_append_only='true'
+) key encode text ;
+
+statement ok
+create table t_sink_text_id (id int)
+include key as rw_key
+with (
+  connector = 'kafka',
+  properties.bootstrap.server = 'message_queue:29092',
+  topic = 'test-rw-sink-text-key-id',
+) format plain encode json;
+
+#======
+
 statement ok
 insert into t_kafka values
     (1, '8DfUFencLe', 31031, 1872, 1872, 26261.416, 23956.39329760601, '2023-04-14 06:27:14.104742', '\x00', '0 second', '0001-01-01', '00:00:01.123456', '0001-01-01 00:00:00.123456'::timestamptz, '{}'),
@@ -193,7 +251,6 @@ insert into t_kafka values
     (6, 'V4y71v4Gip', 4014, 10844, 28842, 5885.368, 11210.458724794062, '2023-04-13 10:42:02.137742', '\xCAFEBABE', '4 hour', '0001-01-01', '00:00:01.123456', '0001-01-01 00:00:00.123456'::timestamptz, '{}'),
     (7, 'YIVLnWxHyf', 10324, 12652, 15914, 3946.7434, 10967.182297153104, '2023-04-14 04:41:03.083742', '\xBABEC0DE', '3 day', '0001-01-01', '01:00:01.123456', '0001-01-01 00:00:00.123456'::timestamptz, '{}');
 
-
 statement error
 create sink si_kafka_without_snapshot as select * from t_kafka with (
     connector = 'kafka',
@@ -216,8 +273,24 @@ create sink si_kafka_without_snapshot from t_kafka with (
     snapshot = 'false',
 );
 
+sleep 1s
+
+query T
+select rw_key from t_sink_text_id order by rw_key
+----
+\x31
+\x32
+\x33
+\x34
+\x35
+\x36
+\x37
+
 statement ok
 insert into t_kafka values
     (8, 'lv7Eq3g8hx', 194, 19036, 28641, 13652.073, 993.408963466774, '2023-04-13 13:52:09.356742', '\xDEADBABE', '04:00:00.1234', '1970-01-01', '00:00:01.123456', '0001-01-01 00:00:00.123456'::timestamptz, '{}'),
     (9, 'nwRq4zejSQ', 10028, 20090, 24837, 20699.559, 11615.276406159757, '2023-04-13 12:40:42.487742', '\xDEADBABE', '05:01:00.123456', '1970-01-01', '00:00:01.123456', '0001-01-01 00:00:00.123456'::timestamptz, '{}'),
-    (10, '0oVqRIHqkb', 26951, 20674, 20674, 19387.238, 9042.404483827515, '2023-04-13 16:40:58.888742', '\x00', '05:01:00.1234567', '1970-01-01', '00:00:01.123456', '1970-01-01 00:00:00.123456'::timestamptz, '{}');
\ No newline at end of file
+    (10, '0oVqRIHqkb', 26951, 20674, 20674, 19387.238, 9042.404483827515, '2023-04-13 16:40:58.888742', '\x00', '05:01:00.1234567', '1970-01-01', '00:00:01.123456', '1970-01-01 00:00:00.123456'::timestamptz, '{}');
+
+statement ok
+drop table t_sink_text_id;
\ No newline at end of file
diff --git a/e2e_test/sink/kafka/debezium.py b/e2e_test/sink/kafka/debezium.py
index 4ec5e31281d88..5dbac1ae80216 100644
--- a/e2e_test/sink/kafka/debezium.py
+++ b/e2e_test/sink/kafka/debezium.py
@@ -11,9 +11,9 @@
         # debezium sink sends k/v pair
         kv = line.split()
         key = json.loads(kv[0])
-        # kafka consumer outputs string "null" for null payload
-        if kv[1] == "null":
-            value = kv[1]
+        # rpk output nothing for null payload
+        if len(kv) == 1:
+            value = None
         else:
             value = json.loads(kv[1])
             # The `ts_ms` field may vary, so we delete it from the json object
@@ -26,12 +26,10 @@
 with open(test_output_file) as file:
     for line in file:
         kv = line.split()
-        if len(kv) != 2:
-            print(line)
-        assert(len(kv) == 2)
         key = json.loads(kv[0])
-        if kv[1] == "null":
-            value = kv[1]
+        # rpk output nothing for null payload
+        if len(kv) == 1:
+            value = None
         else:
             value = json.loads(kv[1])
             # Assert `ts_ms` is an integer here.
diff --git a/e2e_test/sink/kafka/debezium3.result b/e2e_test/sink/kafka/debezium3.result
index 196037c88b33b..b2d0235c2cfd2 100644
--- a/e2e_test/sink/kafka/debezium3.result
+++ b/e2e_test/sink/kafka/debezium3.result
@@ -1,5 +1,5 @@
 {"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":10,"v_bigint":20674,"v_bytea":"AA==","v_date":0,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_interval":"P0Y0M0DT5H1M0.123457S","v_jsonb":"{}","v_smallint":26951,"v_time":1000,"v_timestamp":1681404058888,"v_timestamptz":"1970-01-01T00:00:00.123456Z","v_varchar":"0oVqRIHqkb"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka","ts_ms":1700210120041},"ts_ms":1700210120041},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","name":"org.apache.kafka.connect.data.Timestamp","optional":true,"type":"int64"},{"field":"v_bytea","optional":true,"type":"bytes"},{"field":"v_interval","name":"io.debezium.time.Interval","optional":true,"type":"string"},{"field":"v_date","name":"org.apache.kafka.connect.data.Date","optional":true,"type":"int32"},{"field":"v_time","name":"org.apache.kafka.connect.data.Time","optional":true,"type":"int64"},{"field":"v_timestamptz","name":"io.debezium.time.ZonedTimestamp","optional":true,"type":"string"},{"field":"v_jsonb","name":"io.debezium.data.Json","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","name":"org.apache.kafka.connect.data.Timestamp","optional":true,"type":"int64"},{"field":"v_bytea","optional":true,"type":"bytes"},{"field":"v_interval","name":"io.debezium.time.Interval","optional":true,"type":"string"},{"field":"v_date","name":"org.apache.kafka.connect.data.Date","optional":true,"type":"int32"},{"field":"v_time","name":"org.apache.kafka.connect.data.Time","optional":true,"type":"int64"},{"field":"v_timestamptz","name":"io.debezium.time.ZonedTimestamp","optional":true,"type":"string"},{"field":"v_jsonb","name":"io.debezium.data.Json","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
-{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	null
+{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}
 {"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":null,"before":{"id":1,"v_bigint":0,"v_bytea":"AA==","v_date":-719162,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_interval":"P0Y0M0DT0H0M0S","v_jsonb":"{}","v_smallint":0,"v_time":1000,"v_timestamp":0,"v_timestamptz":"0001-01-01T00:00:00.123456Z","v_varchar":""},"op":"d","source":{"db":"dev","table":"t_kafka","ts_ms":1700210132903},"ts_ms":1700210132903},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","name":"org.apache.kafka.connect.data.Timestamp","optional":true,"type":"int64"},{"field":"v_bytea","optional":true,"type":"bytes"},{"field":"v_interval","name":"io.debezium.time.Interval","optional":true,"type":"string"},{"field":"v_date","name":"org.apache.kafka.connect.data.Date","optional":true,"type":"int32"},{"field":"v_time","name":"org.apache.kafka.connect.data.Time","optional":true,"type":"int64"},{"field":"v_timestamptz","name":"io.debezium.time.ZonedTimestamp","optional":true,"type":"string"},{"field":"v_jsonb","name":"io.debezium.data.Json","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","name":"org.apache.kafka.connect.data.Timestamp","optional":true,"type":"int64"},{"field":"v_bytea","optional":true,"type":"bytes"},{"field":"v_interval","name":"io.debezium.time.Interval","optional":true,"type":"string"},{"field":"v_date","name":"org.apache.kafka.connect.data.Date","optional":true,"type":"int32"},{"field":"v_time","name":"org.apache.kafka.connect.data.Time","optional":true,"type":"int64"},{"field":"v_timestamptz","name":"io.debezium.time.ZonedTimestamp","optional":true,"type":"string"},{"field":"v_jsonb","name":"io.debezium.data.Json","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
 {"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":1,"v_bigint":0,"v_bytea":"AA==","v_date":-719162,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_interval":"P0Y0M0DT0H0M0S","v_jsonb":"{}","v_smallint":0,"v_time":1000,"v_timestamp":0,"v_timestamptz":"0001-01-01T00:00:00.123456Z","v_varchar":""},"before":{"id":1,"v_bigint":1872,"v_bytea":"AA==","v_date":-719162,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_interval":"P0Y0M0DT0H0M0S","v_jsonb":"{}","v_smallint":31031,"v_time":1000,"v_timestamp":1681453634104,"v_timestamptz":"0001-01-01T00:00:00.123456Z","v_varchar":"8DfUFencLe"},"op":"u","source":{"db":"dev","table":"t_kafka","ts_ms":1700210127905},"ts_ms":1700210127905},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","name":"org.apache.kafka.connect.data.Timestamp","optional":true,"type":"int64"},{"field":"v_bytea","optional":true,"type":"bytes"},{"field":"v_interval","name":"io.debezium.time.Interval","optional":true,"type":"string"},{"field":"v_date","name":"org.apache.kafka.connect.data.Date","optional":true,"type":"int32"},{"field":"v_time","name":"org.apache.kafka.connect.data.Time","optional":true,"type":"int64"},{"field":"v_timestamptz","name":"io.debezium.time.ZonedTimestamp","optional":true,"type":"string"},{"field":"v_jsonb","name":"io.debezium.data.Json","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","name":"org.apache.kafka.connect.data.Timestamp","optional":true,"type":"int64"},{"field":"v_bytea","optional":true,"type":"bytes"},{"field":"v_interval","name":"io.debezium.time.Interval","optional":true,"type":"string"},{"field":"v_date","name":"org.apache.kafka.connect.data.Date","optional":true,"type":"int32"},{"field":"v_time","name":"org.apache.kafka.connect.data.Time","optional":true,"type":"int64"},{"field":"v_timestamptz","name":"io.debezium.time.ZonedTimestamp","optional":true,"type":"string"},{"field":"v_jsonb","name":"io.debezium.data.Json","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
 {"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"RisingWave.dev.t_kafka.Key","optional":false,"type":"struct"}}	{"payload":{"after":{"id":1,"v_bigint":1872,"v_bytea":"AA==","v_date":-719162,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_interval":"P0Y0M0DT0H0M0S","v_jsonb":"{}","v_smallint":31031,"v_time":1000,"v_timestamp":1681453634104,"v_timestamptz":"0001-01-01T00:00:00.123456Z","v_varchar":"8DfUFencLe"},"before":null,"op":"c","source":{"db":"dev","table":"t_kafka","ts_ms":1700210120041},"ts_ms":1700210120041},"schema":{"fields":[{"field":"before","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","name":"org.apache.kafka.connect.data.Timestamp","optional":true,"type":"int64"},{"field":"v_bytea","optional":true,"type":"bytes"},{"field":"v_interval","name":"io.debezium.time.Interval","optional":true,"type":"string"},{"field":"v_date","name":"org.apache.kafka.connect.data.Date","optional":true,"type":"int32"},{"field":"v_time","name":"org.apache.kafka.connect.data.Time","optional":true,"type":"int64"},{"field":"v_timestamptz","name":"io.debezium.time.ZonedTimestamp","optional":true,"type":"string"},{"field":"v_jsonb","name":"io.debezium.data.Json","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"after","fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","name":"org.apache.kafka.connect.data.Timestamp","optional":true,"type":"int64"},{"field":"v_bytea","optional":true,"type":"bytes"},{"field":"v_interval","name":"io.debezium.time.Interval","optional":true,"type":"string"},{"field":"v_date","name":"org.apache.kafka.connect.data.Date","optional":true,"type":"int32"},{"field":"v_time","name":"org.apache.kafka.connect.data.Time","optional":true,"type":"int64"},{"field":"v_timestamptz","name":"io.debezium.time.ZonedTimestamp","optional":true,"type":"string"},{"field":"v_jsonb","name":"io.debezium.data.Json","optional":true,"type":"string"}],"name":"RisingWave.dev.t_kafka.Key","optional":true,"type":"struct"},{"field":"source","fields":[{"field":"db","optional":false,"type":"string"},{"field":"table","optional":true,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Source","optional":false,"type":"struct"},{"field":"op","optional":false,"type":"string"},{"field":"ts_ms","optional":false,"type":"int64"}],"name":"RisingWave.dev.t_kafka.Envelope","optional":false,"type":"struct"}}
diff --git a/e2e_test/sink/kafka/drop_sink.slt b/e2e_test/sink/kafka/drop_sink.slt
index db599e80ec6d1..7bd370bcb1264 100644
--- a/e2e_test/sink/kafka/drop_sink.slt
+++ b/e2e_test/sink/kafka/drop_sink.slt
@@ -13,5 +13,8 @@ drop sink si_kafka_upsert_schema;
 statement ok
 drop sink si_kafka_without_snapshot;
 
+statement ok
+drop sink sink_text_id;
+
 statement ok
 drop table t_kafka;
diff --git a/e2e_test/sink/kafka/protobuf.slt b/e2e_test/sink/kafka/protobuf.slt
index 0c74cc8a0b369..61a91435567da 100644
--- a/e2e_test/sink/kafka/protobuf.slt
+++ b/e2e_test/sink/kafka/protobuf.slt
@@ -1,3 +1,6 @@
+statement ok
+set sink_decouple = false;
+
 statement ok
 create table from_kafka with (
   connector = 'kafka',
@@ -13,7 +16,7 @@ create table from_kafka_csr_trivial with (
   topic = 'test-rw-sink-append-only-protobuf-csr-a',
   properties.bootstrap.server = 'message_queue:29092')
 format plain encode protobuf (
-  schema.registry = 'http://message_queue:8081',
+  schema.registry = 'http://schemaregistry:8082',
   message = 'test.package.MessageA');
 
 statement ok
@@ -22,7 +25,7 @@ create table from_kafka_csr_nested with (
   topic = 'test-rw-sink-append-only-protobuf-csr-hi',
   properties.bootstrap.server = 'message_queue:29092')
 format plain encode protobuf (
-  schema.registry = 'http://message_queue:8081',
+  schema.registry = 'http://schemaregistry:8082',
   message = 'test.package.MessageH.MessageI');
 
 statement ok
@@ -68,7 +71,7 @@ create sink sink_csr_trivial as select string_field as field_a from into_kafka w
   properties.bootstrap.server = 'message_queue:29092')
 format plain encode protobuf (
   force_append_only = true,
-  schema.registry = 'http://message_queue:8081',
+  schema.registry = 'http://schemaregistry:8082',
   message = 'test.package.MessageA');
 
 statement ok
@@ -78,7 +81,7 @@ create sink sink_csr_nested as select sint32_field as field_i from into_kafka wi
   properties.bootstrap.server = 'message_queue:29092')
 format plain encode protobuf (
   force_append_only = true,
-  schema.registry = 'http://message_queue:8081',
+  schema.registry = 'http://schemaregistry:8082',
   message = 'test.package.MessageH.MessageI');
 
 sleep 2s
diff --git a/e2e_test/sink/kafka/upsert3.result b/e2e_test/sink/kafka/upsert3.result
index dde36e5f444eb..fe845aff27301 100644
--- a/e2e_test/sink/kafka/upsert3.result
+++ b/e2e_test/sink/kafka/upsert3.result
@@ -1,5 +1,5 @@
 {"id":10}	{"id":10,"v_bigint":20674,"v_bytea":"AA==","v_date":719163,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_interval":"P0Y0M0DT5H1M0.123457S","v_jsonb":"{}","v_smallint":26951,"v_time":1000,"v_timestamp":1681404058888,"v_timestamptz":"1970-01-01 00:00:00.123456","v_varchar":"0oVqRIHqkb"}
-{"id":1}	null
+{"id":1}
 {"id":1}	{"id":1,"v_bigint":0,"v_bytea":"AA==","v_date":1,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_interval":"P0Y0M0DT0H0M0S","v_jsonb":"{}","v_smallint":0,"v_time":1000,"v_timestamp":0,"v_timestamptz":"0001-01-01 00:00:00.123456","v_varchar":""}
 {"id":1}	{"id":1,"v_bigint":1872,"v_bytea":"AA==","v_date":1,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_interval":"P0Y0M0DT0H0M0S","v_jsonb":"{}","v_smallint":31031,"v_time":1000,"v_timestamp":1681453634104,"v_timestamptz":"0001-01-01 00:00:00.123456","v_varchar":"8DfUFencLe"}
 {"id":2}	{"id":2,"v_bigint":4598,"v_bytea":"AA==","v_date":719163,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_interval":"P0Y0M0DT4H0M0S","v_jsonb":"{}","v_smallint":22690,"v_time":1000,"v_timestamp":1681429444869,"v_timestamptz":"0001-01-01 00:00:00.123456","v_varchar":"sIo1XXVeHZ"}
diff --git a/e2e_test/sink/kafka/upsert_schema3.result b/e2e_test/sink/kafka/upsert_schema3.result
index 18675dcbb76d3..ccbb7fe542b37 100644
--- a/e2e_test/sink/kafka/upsert_schema3.result
+++ b/e2e_test/sink/kafka/upsert_schema3.result
@@ -1,5 +1,5 @@
 {"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}}	{"payload":{"id":10,"v_bigint":20674,"v_bytea":"AA==","v_date":719163,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_interval":"P0Y0M0DT5H1M0.123457S","v_jsonb":"{}","v_smallint":26951,"v_time":1000,"v_timestamp":1681404058888,"v_timestamptz":"1970-01-01T00:00:00.123456Z","v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"},{"field":"v_bytea","optional":true,"type":"bytes"},{"field":"v_interval","optional":true,"type":"string"},{"field":"v_date","optional":true,"type":"int32"},{"field":"v_time","optional":true,"type":"int64"},{"field":"v_timestamptz","optional":true,"type":"string"},{"field":"v_jsonb","optional":true,"type":"string"}],"name":"dev.t_kafka","optional":false,"type":"struct"}}
-{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}}	null
+{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}}
 {"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}}	{"payload":{"id":1,"v_bigint":0,"v_bytea":"AA==","v_date":1,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_interval":"P0Y0M0DT0H0M0S","v_jsonb":"{}","v_smallint":0,"v_time":1000,"v_timestamp":0,"v_timestamptz":"0001-01-01T00:00:00.123456Z","v_varchar":""},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"},{"field":"v_bytea","optional":true,"type":"bytes"},{"field":"v_interval","optional":true,"type":"string"},{"field":"v_date","optional":true,"type":"int32"},{"field":"v_time","optional":true,"type":"int64"},{"field":"v_timestamptz","optional":true,"type":"string"},{"field":"v_jsonb","optional":true,"type":"string"}],"name":"dev.t_kafka","optional":false,"type":"struct"}}
 {"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}}	{"payload":{"id":1,"v_bigint":1872,"v_bytea":"AA==","v_date":1,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_interval":"P0Y0M0DT0H0M0S","v_jsonb":"{}","v_smallint":31031,"v_time":1000,"v_timestamp":1681453634104,"v_timestamptz":"0001-01-01T00:00:00.123456Z","v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"},{"field":"v_bytea","optional":true,"type":"bytes"},{"field":"v_interval","optional":true,"type":"string"},{"field":"v_date","optional":true,"type":"int32"},{"field":"v_time","optional":true,"type":"int64"},{"field":"v_timestamptz","optional":true,"type":"string"},{"field":"v_jsonb","optional":true,"type":"string"}],"name":"dev.t_kafka","optional":false,"type":"struct"}}
 {"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}}	{"payload":{"id":2,"v_bigint":4598,"v_bytea":"AA==","v_date":719163,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_interval":"P0Y0M0DT4H0M0S","v_jsonb":"{}","v_smallint":22690,"v_time":1000,"v_timestamp":1681429444869,"v_timestamptz":"0001-01-01T00:00:00.123456Z","v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"},{"field":"v_bytea","optional":true,"type":"bytes"},{"field":"v_interval","optional":true,"type":"string"},{"field":"v_date","optional":true,"type":"int32"},{"field":"v_time","optional":true,"type":"int64"},{"field":"v_timestamptz","optional":true,"type":"string"},{"field":"v_jsonb","optional":true,"type":"string"}],"name":"dev.t_kafka","optional":false,"type":"struct"}}
diff --git a/e2e_test/sink/mqtt_sink.slt b/e2e_test/sink/mqtt_sink.slt
index d19addb024c97..2602d2ddc6198 100644
--- a/e2e_test/sink/mqtt_sink.slt
+++ b/e2e_test/sink/mqtt_sink.slt
@@ -1,3 +1,6 @@
+statement ok
+set sink_decouple = false;
+
 statement ok
 CREATE TABLE mqtt (
   device_id varchar,
diff --git a/e2e_test/sink/pulsar_sink.slt b/e2e_test/sink/pulsar_sink.slt
index 2284f9c1877e3..f8d6aa1aaff4c 100644
--- a/e2e_test/sink/pulsar_sink.slt
+++ b/e2e_test/sink/pulsar_sink.slt
@@ -1,3 +1,6 @@
+statement ok
+set sink_decouple = false;
+
 statement ok
 CREATE TABLE pulsar (
     id BIGINT,
diff --git a/e2e_test/sink/redis_cluster_sink.slt b/e2e_test/sink/redis_cluster_sink.slt
index 5d2d84b773367..03d197485777a 100644
--- a/e2e_test/sink/redis_cluster_sink.slt
+++ b/e2e_test/sink/redis_cluster_sink.slt
@@ -14,13 +14,7 @@ FROM
 )FORMAT PLAIN ENCODE JSON(force_append_only='true');
 
 statement ok
-INSERT INTO t6 VALUES (1, 1);
-
-statement ok
-INSERT INTO t6 VALUES (2, 2);
-
-statement ok
-INSERT INTO t6 VALUES (3, 3);
+INSERT INTO t6 VALUES (1, 1),(2, 2),(3, 3),(4, 4),(5, 5),(6, 6),(7, 7),(8, 8),(9, 9),(10, 10),(11, 11),(12, 12),(13, 13),(14, 14),(15, 15);
 
 statement ok
 FLUSH;
diff --git a/e2e_test/sink/remote/jdbc.check.pg.slt b/e2e_test/sink/remote/jdbc.check.pg.slt
index 0c29969445d2f..1ec8c827d939b 100644
--- a/e2e_test/sink/remote/jdbc.check.pg.slt
+++ b/e2e_test/sink/remote/jdbc.check.pg.slt
@@ -44,3 +44,9 @@ query IT
 select * from t1_uuid;
 ----
 221 74605c5a-a7bb-4b3b-8742-2a12e9709dea hello world
+
+
+query TIT
+select * from sk_t1_uuid
+----
+21189447-8736-44bd-b254-26b5dec91da9 2 bb
diff --git a/e2e_test/sink/remote/jdbc.load.slt b/e2e_test/sink/remote/jdbc.load.slt
index a453c76feeb70..9a4ede4e032ed 100644
--- a/e2e_test/sink/remote/jdbc.load.slt
+++ b/e2e_test/sink/remote/jdbc.load.slt
@@ -164,6 +164,26 @@ CREATE SINK s1_uuid FROM t1_uuid WITH (
 statement ok
 INSERT INTO t1_uuid values (221, '74605c5a-a7bb-4b3b-8742-2a12e9709dea', 'hello world');
 
+
+statement ok
+CREATE TABLE t1_test_uuid_delete (id varchar, v1 int, v2 varchar, primary key(id, v2));
+
+statement ok
+INSERT INTO t1_test_uuid_delete VALUES ('fb48ecc1-917f-4f4b-ab6d-d8e37809caf8', 1, 'aa'), ('21189447-8736-44bd-b254-26b5dec91da9', 2, 'bb');
+
+statement ok
+CREATE SINK sk_t1_uuid FROM t1_test_uuid_delete WITH (
+    connector='jdbc',
+    jdbc.url='jdbc:postgresql://db:5432/test?user=test&password=connector',
+    table.name='sk_t1_uuid',
+    primary_key='id, v2',
+    type='upsert'
+);
+
+statement ok
+DELETE FROM t1_test_uuid_delete WHERE id='fb48ecc1-917f-4f4b-ab6d-d8e37809caf8' AND v2='aa';
+
+
 statement ok
 INSERT INTO tt2 VALUES
     (1),
diff --git a/e2e_test/sink/remote/pg_create_table.sql b/e2e_test/sink/remote/pg_create_table.sql
index 3677dbb067131..ee272ef747a7a 100644
--- a/e2e_test/sink/remote/pg_create_table.sql
+++ b/e2e_test/sink/remote/pg_create_table.sql
@@ -83,3 +83,5 @@ CREATE TABLE biz.t_types (
 CREATE TABLE biz.t2 (
     "aBc" INTEGER PRIMARY KEY
 );
+
+CREATE TABLE sk_t1_uuid (id uuid, v1 int, v2 varchar, primary key(id, v2));
diff --git a/e2e_test/sink/sink_into_table/basic.slt b/e2e_test/sink/sink_into_table/basic.slt
index 179f659e4c113..59e43773560f0 100644
--- a/e2e_test/sink/sink_into_table/basic.slt
+++ b/e2e_test/sink/sink_into_table/basic.slt
@@ -327,7 +327,6 @@ drop table t_primary_key;
 statement ok
 drop table t_s3;
 
-
 # multi sinks
 
 statement ok
@@ -423,3 +422,42 @@ drop table t_c;
 
 statement ok
 drop table t_m;
+
+# from view
+
+statement ok
+create table t_a(v int);
+
+statement ok
+insert into t_a values (1), (2), (3);
+
+statement ok
+create view v_a as select v from t_a;
+
+statement ok
+create table t_m(v int primary key);
+
+statement ok
+create sink s_a into t_m as select v from v_a;
+
+statement ok
+flush;
+
+query I
+select * from t_m order by v;
+----
+1
+2
+3
+
+statement ok
+drop sink s_a;
+
+statement ok
+drop view v_a;
+
+statement ok
+drop table t_m;
+
+statement ok
+drop table t_a;
diff --git a/e2e_test/sink/sink_into_table/specify_column.slt b/e2e_test/sink/sink_into_table/specify_column.slt
new file mode 100644
index 0000000000000..2eefd8abf8fb0
--- /dev/null
+++ b/e2e_test/sink/sink_into_table/specify_column.slt
@@ -0,0 +1,54 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+create table s (a int, b int, c int) append only;
+
+statement ok
+create table t (a int, b int default 900, c int default 9000);
+
+statement error
+create sink ss into t(aaa) as select a from s with(type = 'append-only');
+
+statement error
+create sink ss into t(a) as select a, b from s with(type = 'append-only');
+
+statement error
+create sink ss into t(a, b) as select b from s with(type = 'append-only');
+
+statement error
+create sink ss into t(a, b, c, a) as select a, b from s with(type = 'append-only');
+
+statement ok
+create sink s1 into t(a,B,c) as select c, b, a from s with(type = 'append-only');
+
+statement ok
+create sink s2 into t(a,B) as select 2*c, 2*b from s with(type = 'append-only');
+
+statement ok
+create sink s3 into t(c) as select 3*a from s with(type = 'append-only');
+
+statement ok
+insert into s values(10, 100, 1000);
+
+query III rowsort
+select * from t order by a;
+----
+1000	100	10
+2000	200	9000
+NULL	900	30
+
+statement ok
+drop sink s1;
+
+statement ok
+drop sink s2;
+
+statement ok
+drop sink s3;
+
+statement ok
+drop table s;
+
+statement ok
+drop table t;
diff --git a/e2e_test/sink/sqlserver_sink.slt b/e2e_test/sink/sqlserver_sink.slt
new file mode 100644
index 0000000000000..156b8b865ffc8
--- /dev/null
+++ b/e2e_test/sink/sqlserver_sink.slt
@@ -0,0 +1,59 @@
+statement ok
+create table t_many_data_type_rw (
+  k1 int, k2 int,
+  c_boolean bool,
+  c_int16 smallint,
+  c_int32 int,
+  c_int64 bigint,
+  c_float32 float,
+  c_float64 double,
+  c_decimal decimal,
+  c_date date,
+  c_time time,
+  c_timestamp timestamp,
+  c_timestampz timestamp,
+  c_nvarchar string,
+  c_varbinary bytea);
+
+statement ok
+create sink s_many_data_type from t_many_data_type_rw with (
+  connector = 'sqlserver',
+  type = 'upsert',
+  sqlserver.host = 'sqlserver-server',
+  sqlserver.port = 1433,
+  sqlserver.user = 'SA',
+  sqlserver.password = 'SomeTestOnly@SA',
+  sqlserver.database = 'SinkTest',
+  sqlserver.table = 't_many_data_type',
+  primary_key = 'k1,k2',
+);
+
+statement ok
+insert into t_many_data_type_rw values
+(0,0,false,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL),
+(1,1,false,1,1,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf'),
+(1,2,false,2,2,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf'),
+(1,3,false,2,2,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf'),
+(1,4,false,2,2,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf'),
+(1,1,false,2,2,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf');
+flush;
+
+statement ok
+delete from t_many_data_type_rw where k1=1 and k2=2;
+delete from t_many_data_type_rw where k1=1 and k2=3;
+flush;
+
+statement ok
+insert into t_many_data_type_rw values
+(1,1,false,55,55,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf'),
+(1,2,false,66,66,1,1.0,1.0,1.0,date '2022-04-08',time '18:20:49','2022-03-13 01:00:00'::timestamp,'2022-03-13 01:00:00Z'::timestamptz,'Hello World!','\xDe00BeEf');
+flush;
+
+statement ok
+FLUSH;
+
+statement ok
+DROP SINK s_many_data_type;
+
+statement ok
+DROP TABLE t_many_data_type_rw;
diff --git a/e2e_test/slow_tests/backfill/rate_limit/slow-udf.slt b/e2e_test/slow_tests/backfill/rate_limit/slow-udf.slt
new file mode 100644
index 0000000000000..a2b1a6fc63130
--- /dev/null
+++ b/e2e_test/slow_tests/backfill/rate_limit/slow-udf.slt
@@ -0,0 +1,49 @@
+statement ok
+create table t(v1 int);
+
+statement ok
+insert into t select 2 from generate_series(1, 1000000);
+
+statement ok
+set streaming_rate_limit=1;
+
+statement ok
+set background_ddl=true;
+
+statement ok
+CREATE FUNCTION delay(secs int) RETURNS int LANGUAGE python AS $$
+def delay(n):
+    import time
+    time.sleep(n)
+    return n
+$$;
+
+statement ok
+create sink m1 as select delay(v1) from t with (connector='blackhole');
+
+statement ok
+set background_ddl = false;
+
+statement ok
+set streaming_rate_limit=default;
+
+statement ok
+flush;
+
+statement ok
+flush;
+
+statement ok
+flush;
+
+statement ok
+flush;
+
+statement ok
+drop sink m1;
+
+statement ok
+drop function delay;
+
+statement ok
+drop table t;
\ No newline at end of file
diff --git a/e2e_test/udf/always_retry_python.slt b/e2e_test/slow_tests/udf/always_retry_python.slt
similarity index 85%
rename from e2e_test/udf/always_retry_python.slt
rename to e2e_test/slow_tests/udf/always_retry_python.slt
index 19e66dd60b07d..78bf926c32986 100644
--- a/e2e_test/udf/always_retry_python.slt
+++ b/e2e_test/slow_tests/udf/always_retry_python.slt
@@ -15,7 +15,7 @@ statement ok
 CREATE TABLE t (v1 int);
 
 statement ok
-INSERT INTO t select 0 from generate_series(1, 30);
+INSERT INTO t select 0 from generate_series(1, 60);
 
 statement ok
 flush;
@@ -33,7 +33,7 @@ CREATE MATERIALIZED VIEW mv_no_retry AS SELECT sleep_no_retry(v1) as s1 from t;
 statement ok
 CREATE MATERIALIZED VIEW mv_always_retry AS SELECT sleep_always_retry(v1) as s1 from t;
 
-# Immediately kill the server, sleep for 1minute.
+# Immediately kill the server, sleep
 system ok
 pkill -9 -i python && sleep 60
 
@@ -50,10 +50,11 @@ SELECT count(*) FROM mv_always_retry where s1 is NULL;
 ----
 0
 
-query B
-SELECT count(*) > 0 FROM mv_no_retry where s1 is NULL;
-----
-t
+# FIXME(kwannoel): Somehow this is flaky..
+# query B
+# SELECT count(*) > 0 FROM mv_no_retry where s1 is NULL;
+# ----
+# t
 
 statement ok
 SET STREAMING_RATE_LIMIT TO DEFAULT;
diff --git a/e2e_test/source/basic/alter/rate_limit_source_kafka.slt b/e2e_test/source/basic/alter/rate_limit_source_kafka.slt
new file mode 100644
index 0000000000000..7d991083345a1
--- /dev/null
+++ b/e2e_test/source/basic/alter/rate_limit_source_kafka.slt
@@ -0,0 +1,130 @@
+############## Create kafka seed data
+
+statement ok
+create table kafka_seed_data (v1 int);
+
+statement ok
+insert into kafka_seed_data select * from generate_series(1, 1000);
+
+############## Sink into kafka
+
+statement ok
+create sink kafka_sink
+from
+  kafka_seed_data with (
+    properties.bootstrap.server = 'message_queue:29092',
+    topic = 'kafka_source',
+    type = 'append-only',
+    force_append_only='true',
+    connector = 'kafka'
+);
+
+############## Source from kafka (rate_limit = 0)
+
+# Wait for the topic to create
+skipif in-memory
+sleep 5s
+
+statement ok
+create source kafka_source (v1 int) with (
+  connector = 'kafka',
+  topic = 'kafka_source',
+  properties.bootstrap.server = 'message_queue:29092',
+  scan.startup.mode = 'earliest',
+) FORMAT PLAIN ENCODE JSON
+
+statement ok
+flush;
+
+############## Check data
+
+skipif in-memory
+sleep 3s
+
+############## Create MV on source
+
+statement ok
+SET STREAMING_RATE_LIMIT=0;
+
+statement ok
+create materialized view rl_mv1 as select count(*) from kafka_source;
+
+statement ok
+create materialized view rl_mv2 as select count(*) from kafka_source;
+
+statement ok
+create materialized view rl_mv3 as select count(*) from kafka_source;
+
+statement ok
+SET STREAMING_RATE_LIMIT=default;
+
+############## MVs should have 0 records, since source has (rate_limit = 0)
+
+statement ok
+flush;
+
+query I
+select * from rl_mv1;
+----
+0
+
+query I
+select * from rl_mv2;
+----
+0
+
+query I
+select * from rl_mv3;
+----
+0
+
+############## Alter Source (rate_limit = 0 --> rate_limit = 1000)
+
+skipif in-memory
+query I
+alter source kafka_source set streaming_rate_limit to 1000;
+
+skipif in-memory
+query I
+alter source kafka_source set streaming_rate_limit to default;
+
+skipif in-memory
+sleep 3s
+
+skipif in-memory
+query I
+select count(*) > 0 from rl_mv1;
+----
+t
+
+skipif in-memory
+query I
+select count(*) > 0 from rl_mv2;
+----
+t
+
+skipif in-memory
+query I
+select count(*) > 0 from rl_mv3;
+----
+t
+
+############## Cleanup
+
+statement ok
+drop materialized view rl_mv1;
+
+statement ok
+drop materialized view rl_mv2;
+
+statement ok
+drop materialized view rl_mv3;
+
+statement ok
+drop source kafka_source;
+
+statement ok
+drop sink kafka_sink;
+
+statement ok
+drop table kafka_seed_data;
\ No newline at end of file
diff --git a/e2e_test/source/basic/alter/rate_limit_table_kafka.slt b/e2e_test/source/basic/alter/rate_limit_table_kafka.slt
new file mode 100644
index 0000000000000..bf1fd6672d6ea
--- /dev/null
+++ b/e2e_test/source/basic/alter/rate_limit_table_kafka.slt
@@ -0,0 +1,88 @@
+############## Create kafka seed data
+
+statement ok
+create table kafka_seed_data (v1 int);
+
+statement ok
+insert into kafka_seed_data select * from generate_series(1, 1000);
+
+############## Sink into kafka
+
+statement ok
+create sink kafka_sink
+from
+  kafka_seed_data with (
+    properties.bootstrap.server = 'message_queue:29092',
+    topic = 'kafka_source',
+    type = 'append-only',
+    force_append_only='true',
+    connector = 'kafka'
+);
+
+############## Source from kafka (rate_limit = 0)
+
+statement ok
+create table kafka_source (v1 int) with (
+  connector = 'kafka',
+  topic = 'kafka_source',
+  properties.bootstrap.server = 'message_queue:29092',
+  scan.startup.mode = 'earliest',
+  streaming_rate_limit = 0
+) FORMAT PLAIN ENCODE JSON
+
+statement ok
+flush;
+
+############## Check data
+
+skipif in-memory
+sleep 3s
+
+skipif in-memory
+query I
+select count(*) from kafka_source;
+----
+0
+
+############## Can still insert data when rate limit = 0
+
+statement ok
+insert into kafka_source values(1);
+
+statement ok
+flush;
+
+query I
+select count(*) from kafka_source;
+----
+1
+
+############## Alter source (rate_limit = 0 --> rate_limit = 1000)
+
+skipif in-memory
+query I
+alter table kafka_source set streaming_rate_limit to 1000;
+
+skipif in-memory
+query I
+alter table kafka_source set streaming_rate_limit to default;
+
+skipif in-memory
+sleep 3s
+
+skipif in-memory
+query I
+select count(*) > 1 from kafka_source;
+----
+t
+
+############## Cleanup
+
+statement ok
+drop table kafka_source;
+
+statement ok
+drop sink kafka_sink;
+
+statement ok
+drop table kafka_seed_data;
\ No newline at end of file
diff --git a/e2e_test/source/basic/datagen.slt b/e2e_test/source/basic/datagen.slt
index 91c51f624a1ea..d89850538a3fd 100644
--- a/e2e_test/source/basic/datagen.slt
+++ b/e2e_test/source/basic/datagen.slt
@@ -186,9 +186,9 @@ statement ok
 drop table s1;
 
 # Do NOT allow With clause to contain a comma only.
-statement error Expected identifier.*
+statement error expected identifier.*
 create table s1 (v1 int) with (,) FORMAT PLAIN ENCODE JSON;
 
 # Do NOT allow an empty With clause.
-statement error Expected identifier.*
+statement error expected identifier.*
 create table s1 (v1 int) with () FORMAT PLAIN ENCODE JSON;
diff --git a/e2e_test/source/basic/ddl.slt b/e2e_test/source/basic/ddl.slt
index 402cf129b86ba..a56d90934c149 100644
--- a/e2e_test/source/basic/ddl.slt
+++ b/e2e_test/source/basic/ddl.slt
@@ -4,8 +4,9 @@ create source s;
 db error: ERROR: Failed to run the query
 
 Caused by:
-  sql parser error: Expected description of the format, found: ; at line:1, column:17
-Near "create source s"
+  sql parser error: expected description of the format, found: ;
+LINE 1: create source s;
+                       ^
 
 
 statement error missing WITH clause
@@ -187,7 +188,7 @@ create table s (
 query T
 show sources
 ----
-
+s
 
 query T
 show tables
diff --git a/e2e_test/source/basic/kafka.slt b/e2e_test/source/basic/kafka.slt
index 08c84947846a7..dee5de1cbb539 100644
--- a/e2e_test/source/basic/kafka.slt
+++ b/e2e_test/source/basic/kafka.slt
@@ -166,7 +166,7 @@ create table s8_no_schema_field (
 statement error without schema registry
 create table s9 with (
   connector = 'kafka',
-  topic = 'avro_bin',
+  topic = 'avro_simple_schema_bin',
   properties.bootstrap.server = 'message_queue:29092',
   scan.startup.mode = 'earliest'
 ) FORMAT PLAIN ENCODE AVRO (schema.location = 'file:///risingwave/avro-simple-schema.avsc');
@@ -174,7 +174,7 @@ create table s9 with (
 statement ok
 create table s9 with (
   connector = 'kafka',
-  topic = 'avro_bin',
+  topic = 'avro_simple_schema_bin',
   properties.bootstrap.server = 'message_queue:29092',
   scan.startup.mode = 'earliest'
 ) FORMAT PLAIN ENCODE AVRO (schema.location = 'file:///risingwave/avro-simple-schema.avsc', with_deprecated_file_header = true);
@@ -182,7 +182,7 @@ create table s9 with (
 statement ok
 create table s10 with (
   connector = 'kafka',
-  topic = 'avro_c_bin',
+  topic = 'avro_complex_schema_bin',
   properties.bootstrap.server = 'message_queue:29092',
   scan.startup.mode = 'earliest'
 ) FORMAT PLAIN ENCODE AVRO (schema.location = 'file:///risingwave/avro-complex-schema.avsc', with_deprecated_file_header = true);
@@ -287,7 +287,7 @@ FORMAT PLAIN ENCODE PROTOBUF (
 statement ok
 create source s18 with (
   connector = 'kafka',
-  topic = 'avro_c_bin',
+  topic = 'avro_complex_schema_bin',
   properties.bootstrap.server = 'message_queue:29092',
   scan.startup.mode = 'earliest'
 ) FORMAT PLAIN ENCODE AVRO (schema.location = 'file:///risingwave/avro-complex-schema.avsc', with_deprecated_file_header = true);
diff --git a/e2e_test/source/basic/kafka_shared_source.slt b/e2e_test/source/basic/kafka_shared_source.slt
deleted file mode 100644
index 5245d6ea68630..0000000000000
--- a/e2e_test/source/basic/kafka_shared_source.slt
+++ /dev/null
@@ -1,58 +0,0 @@
-control substitution on
-
-statement ok
-SET rw_enable_shared_source TO true;
-
-statement ok
-create source s0 (v1 int, v2 varchar) with (
-  connector = 'kafka',
-  topic = 'kafka_4_partition_topic',
-  properties.bootstrap.server = '${KAFKA_BOOTSTRAP_SERVER:message_queue:29092}',
-  scan.startup.mode = 'earliest'
-) FORMAT PLAIN ENCODE JSON;
-
-statement ok
-create materialized view mv_1 as select * from s0;
-
-statement ok
-SET rw_enable_shared_source TO false;
-
-statement ok
-create materialized view mv_2 as select * from s0;
-
-statement ok
-flush;
-
-# Wait enough time to ensure SourceExecutor consumes all Kafka data.
-sleep 1s
-
-query IT rowsort
-select v1, v2 from s0;
-----
-1 1
-2 22
-3 333
-4 4444
-
-query IT rowsort
-select v1, v2 from mv_1;
-----
-1 1
-2 22
-3 333
-4 4444
-
-query IT rowsort
-select v1, v2 from mv_2;
-----
-1 1
-2 22
-3 333
-4 4444
-
-# TODO: add more data to the topic and re-check the data. Currently there's no good test infra to do this...
-# To test the correctness of source backfill, we might need to keep producing data during an interval, to let it go
-# through the backfill stage to the forward stage.
-
-statement ok
-drop source s0 cascade;
diff --git a/e2e_test/source/basic/nosim_kafka.slt b/e2e_test/source/basic/nosim_kafka.slt
index bc4a88e65ee71..b773126c9a7c1 100644
--- a/e2e_test/source/basic/nosim_kafka.slt
+++ b/e2e_test/source/basic/nosim_kafka.slt
@@ -1,62 +1,59 @@
-# Start with nosim to avoid running in deterministic test
+control substitution on
 
+# FIXME: does this really work??
+# Start with nosim to avoid running in deterministic test
 
-# If we cannot extract key schema, use message key as varchar primary key
-statement ok
-CREATE TABLE upsert_avro_json_default_key ( primary key (rw_key) )
-INCLUDE KEY AS rw_key
-WITH (
-	connector = 'kafka',
-  properties.bootstrap.server = 'message_queue:29092',
-	topic = 'upsert_avro_json')
-FORMAT UPSERT ENCODE AVRO (schema.registry = 'http://message_queue:8081');
 
 statement ok
 CREATE TABLE upsert_student_avro_json ( primary key (rw_key) )
 INCLUDE KEY AS rw_key
 WITH (
 	connector = 'kafka',
-  properties.bootstrap.server = 'message_queue:29092',
+  properties.bootstrap.server = '${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}',
 	topic = 'upsert_student_avro_json')
-FORMAT UPSERT ENCODE AVRO (schema.registry = 'http://message_queue:8081');
-
-
-# TODO: Uncomment this when we add test data kafka key with format `"ID":id`
-# statement ok
-# CREATE TABLE upsert_avro_json (
-#   PRIMARY KEY("ID")
-# )
-# WITH (
-# connector = 'kafka',
-#  properties.bootstrap.server = 'message_queue:29092',
-# topic = 'upsert_avro_json')
-# FORMAT UPSERT ENCODE AVRO (schema.registry = 'http://message_queue:8081');
+FORMAT UPSERT ENCODE AVRO (schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}');
 
 
 statement ok
 CREATE TABLE debezium_non_compact (PRIMARY KEY(order_id)) with (
     connector = 'kafka',
     kafka.topic = 'debezium_non_compact_avro_json',
-    kafka.brokers = 'message_queue:29092',
+    kafka.brokers = '${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}',
     kafka.scan.startup.mode = 'earliest'
-) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081');
+) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}');
 
 
 statement ok
 CREATE TABLE debezium_compact (PRIMARY KEY(order_id)) with (
     connector = 'kafka',
     kafka.topic = 'debezium_compact_avro_json',
-    kafka.brokers = 'message_queue:29092',
+    kafka.brokers = '${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}',
     kafka.scan.startup.mode = 'earliest'
-) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081');
+) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}');
 
 statement ok
 CREATE TABLE kafka_json_schema_plain with (
     connector = 'kafka',
     kafka.topic = 'kafka_json_schema',
-    kafka.brokers = 'kafka:9093',
+    kafka.brokers = '${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}',
     kafka.scan.startup.mode = 'earliest'
-) FORMAT PLAIN ENCODE JSON (schema.registry = 'http://schemaregistry:8082');
+) FORMAT PLAIN ENCODE JSON (schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}');
+
+
+query
+describe kafka_json_schema_plain;
+----
+dimensions (empty) false NULL
+map jsonb false NULL
+notMap (empty) false NULL
+price double precision false NULL
+productId bigint false NULL
+productName character varying false NULL
+tags character varying[] false NULL
+_row_id serial true NULL
+primary key _row_id NULL NULL
+distribution key _row_id NULL NULL
+table description kafka_json_schema_plain NULL NULL
 
 statement ok
 CREATE TABLE kafka_json_schema_upsert (PRIMARY KEY(rw_key))
@@ -64,9 +61,9 @@ INCLUDE KEY AS rw_key
 with (
     connector = 'kafka',
     kafka.topic = 'kafka_upsert_json_schema',
-    kafka.brokers = 'kafka:9093',
+    kafka.brokers = '${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}',
     kafka.scan.startup.mode = 'earliest'
-) FORMAT UPSERT ENCODE JSON (schema.registry = 'http://schemaregistry:8082');
+) FORMAT UPSERT ENCODE JSON (schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}');
 
 statement ok
 flush;
@@ -74,32 +71,6 @@ flush;
 # Wait enough time to ensure SourceExecutor consumes all Kafka data.
 sleep 8s
 
-query II
-SELECT
- op_type, "ID", "CLASS_ID", "ITEM_ID", "ATTR_ID", "ATTR_VALUE", "ORG_ID", "UNIT_INFO", "UPD_TIME", "DEC_VAL"
-FROM
- upsert_avro_json_default_key
-ORDER BY
- "ID";
-----
-update id1 -1 6768 6970 value9 7172 info9 2021-05-18T07:59:58.714Z -21474836.47
-delete id2 2 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 99999999.99
-delete id3 3 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 21474836.47
-delete id5 5 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 21474836.49
-
-# query II
-# SELECT
-#  *
-# FROM
-#  upsert_avro_json
-# ORDER BY
-#  "ID";
-# ----
-# update id1 -1 6768 6970 value9 7172 info9 2021-05-18T07:59:58.714Z
-# delete id2 2 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z
-# delete id3 3 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z
-# delete id5 5 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z
-
 query II
 SELECT
  "ID", "firstName", "lastName", "age", "height", "weight"
@@ -128,10 +99,10 @@ select count(*) from debezium_compact;
 
 query TFITT
 select
-  "dimensions", "price", "productId", "productName", "tags"
+  *
 from kafka_json_schema_plain
 ----
-(9.5,7,12) 12.5 1 An ice sculpture {cold,ice}
+(9.5,7,12) {"foo": "bar"} (b) 12.5 1 An ice sculpture {cold,ice}
 
 query TFITT
 select
@@ -141,12 +112,6 @@ from kafka_json_schema_upsert order by id
 (9.5,7,12) 1 23 An ice sculpture {cold,ice}
 (9.5,7,12) 2 12.5 An ice sculpture {cold,ice}
 
-statement ok
-DROP TABLE upsert_avro_json_default_key;
-
-# statement ok
-# DROP TABLE upsert_avro_json;
-
 statement ok
 DROP TABLE upsert_student_avro_json;
 
@@ -160,4 +125,4 @@ statement ok
 DROP TABLE kafka_json_schema_plain;
 
 statement ok
-DROP TABLE kafka_json_schema_upsert;
\ No newline at end of file
+DROP TABLE kafka_json_schema_upsert;
diff --git a/e2e_test/source/basic/old_row_format_syntax/datagen.slt b/e2e_test/source/basic/old_row_format_syntax/datagen.slt
index 267ae8eff4c66..6467b624d0dbb 100644
--- a/e2e_test/source/basic/old_row_format_syntax/datagen.slt
+++ b/e2e_test/source/basic/old_row_format_syntax/datagen.slt
@@ -182,9 +182,9 @@ statement ok
 drop table s1;
 
 # Do NOT allow With clause to contain a comma only.
-statement error Expected identifier.*
+statement error expected identifier.*
 create table s1 (v1 int) with (,) ROW FORMAT JSON;
 
 # Do NOT allow an empty With clause.
-statement error Expected identifier.*
+statement error expected identifier.*
 create table s1 (v1 int) with () ROW FORMAT JSON;
diff --git a/e2e_test/source/basic/old_row_format_syntax/ddl.slt b/e2e_test/source/basic/old_row_format_syntax/ddl.slt
index d5c41d4ded878..b48249ca7e393 100644
--- a/e2e_test/source/basic/old_row_format_syntax/ddl.slt
+++ b/e2e_test/source/basic/old_row_format_syntax/ddl.slt
@@ -142,7 +142,7 @@ create table s (
 query T
 show sources
 ----
-
+s
 
 query T
 show tables
diff --git a/e2e_test/source/basic/old_row_format_syntax/kafka.slt b/e2e_test/source/basic/old_row_format_syntax/kafka.slt
index 6c387aeb4523f..1f4c118f30dc5 100644
--- a/e2e_test/source/basic/old_row_format_syntax/kafka.slt
+++ b/e2e_test/source/basic/old_row_format_syntax/kafka.slt
@@ -158,7 +158,7 @@ create table s8_no_schema_field (
 statement error without schema registry
 create table s9 with (
   connector = 'kafka',
-  topic = 'avro_bin',
+  topic = 'avro_simple_schema_bin',
   properties.bootstrap.server = 'message_queue:29092',
   scan.startup.mode = 'earliest'
 ) row format avro row schema location 'file:///risingwave/avro-simple-schema.avsc'
@@ -166,7 +166,7 @@ create table s9 with (
 statement error without schema registry
 create table s10 with (
   connector = 'kafka',
-  topic = 'avro_c_bin',
+  topic = 'avro_complex_schema_bin',
   properties.bootstrap.server = 'message_queue:29092',
   scan.startup.mode = 'earliest'
 ) row format avro row schema location 'file:///risingwave/avro-complex-schema.avsc'
@@ -265,7 +265,7 @@ create source s17 with (
 statement error without schema registry
 create source s18 with (
   connector = 'kafka',
-  topic = 'avro_c_bin',
+  topic = 'avro_complex_schema_bin',
   properties.bootstrap.server = 'message_queue:29092',
   scan.startup.mode = 'earliest'
 ) row format avro row schema location 'file:///risingwave/avro-complex-schema.avsc'
diff --git a/e2e_test/source/basic/pubsub.slt b/e2e_test/source/basic/pubsub.slt
deleted file mode 100644
index b245d9b2aea89..0000000000000
--- a/e2e_test/source/basic/pubsub.slt
+++ /dev/null
@@ -1,79 +0,0 @@
-# fail with invalid emulator_host
-statement error
-CREATE TABLE s1 (v1 int, v2 varchar) WITH (
-    pubsub.subscription = 'test-subscription-1',
-    pubsub.emulator_host = 'invalid_host:5981'
-) FORMAT PLAIN ENCODE JSON;
-
-statement ok
-CREATE TABLE s1 (v1 int, v2 varchar) WITH (
-    connector = 'google_pubsub',
-    pubsub.subscription = 'test-subscription-1',
-    pubsub.emulator_host = 'localhost:5980'
-) FORMAT PLAIN ENCODE JSON;
-
-statement ok
-SELECT * FROM s1;
-
-statement ok
-DROP TABLE s1;
-
-# fail with invalid subscription
-statement error
-CREATE TABLE s2 (v1 int, v2 varchar) WITH (
-    connector = 'google_pubsub',
-    pubsub.subscription = 'test-subscription-not-2',
-    pubsub.emulator_host = 'localhost:5980'
-) FORMAT PLAIN ENCODE JSON;
-
-statement ok
-CREATE TABLE s2 (v1 int, v2 varchar) WITH (
-    connector = 'google_pubsub',
-    pubsub.subscription = 'test-subscription-2',
-    pubsub.emulator_host = 'localhost:5980'
-) FORMAT PLAIN ENCODE JSON;
-
-# fail if both start_offset and start_snapshot are provided
-statement error
-CREATE TABLE s3 (v1 int, v2 varchar) WITH (
-    connector = 'google_pubsub',
-    pubsub.subscription = 'test-subscription-3',
-    pubsub.emulator_host = 'localhost:5980',
-    pubsub.start_offset = "121212",
-    pubsub.start_snapshot = "snapshot-that-doesnt-exist"
-) FORMAT PLAIN ENCODE JSON;
-
-# wait for source
-sleep 10s
-
-# flush data into storage
-statement ok
-flush;
-
-query IT rowsort
-select v1, v2 FROM s2;
-----
-0 name5
-0 name9
-1 name0
-1 name7
-2 name0
-2 name3
-3 name2
-3 name9
-4 name6
-4 name7
-5 name3
-5 name8
-6 name3
-6 name4
-7 name0
-7 name5
-8 name8
-8 name9
-9 name2
-9 name2
-
-statement ok
-DROP TABLE s2;
-
diff --git a/e2e_test/source/basic/schema_registry.slt b/e2e_test/source/basic/schema_registry.slt
deleted file mode 100644
index 76f867b2b1d0e..0000000000000
--- a/e2e_test/source/basic/schema_registry.slt
+++ /dev/null
@@ -1,67 +0,0 @@
-# wrong strategy name
-statement error
-create source s1 () with (
-  connector = 'kafka',
-  topic = 'upsert_avro_json-record',
-  properties.bootstrap.server = 'message_queue:29092'
-) format plain encode avro (
-  schema.registry = 'http://message_queue:8081',
-  schema.registry.name.strategy = 'no sense',
-  message = 'CPLM.OBJ_ATTRIBUTE_VALUE',
-);
-
-# redundant field key.message
-statement error
-create source s1 () with (
-  connector = 'kafka',
-  topic = 'upsert_avro_json-record',
-  properties.bootstrap.server = 'message_queue:29092'
-) format plain encode avro (
-  schema.registry = 'http://message_queue:8081',
-  schema.registry.name.strategy = 'record_name_strategy',
-  message = 'CPLM.OBJ_ATTRIBUTE_VALUE',
-  key.message = 'string'
-);
-
-statement ok
-create source s1 () with (
-  connector = 'kafka',
-  topic = 'upsert_avro_json-record',
-  properties.bootstrap.server = 'message_queue:29092'
-) format plain encode avro (
-  schema.registry = 'http://message_queue:8081',
-  schema.registry.name.strategy = 'record_name_strategy',
-  message = 'CPLM.OBJ_ATTRIBUTE_VALUE',
-);
-
-# lack field key.message
-statement error
-create table t1 () with (
-    connector = 'kafka',
-    topic = 'upsert_avro_json-topic-record',
-    properties.bootstrap.server = 'message_queue:29092'
-) format upsert encode avro (
-    schema.registry = 'http://message_queue:8081',
-    schema.registry.name.strategy = 'topic_record_name_strategy',
-    message = 'CPLM.OBJ_ATTRIBUTE_VALUE'
-);
-
-statement ok
-create table t1 (primary key(rw_key))
-INCLUDE KEY AS rw_key
-with (
-    connector = 'kafka',
-    topic = 'upsert_avro_json-topic-record',
-    properties.bootstrap.server = 'message_queue:29092'
-) format upsert encode avro (
-    schema.registry = 'http://message_queue:8081',
-    schema.registry.name.strategy = 'topic_record_name_strategy',
-    message = 'CPLM.OBJ_ATTRIBUTE_VALUE',
-    key.message = 'string'
-);
-
-statement ok
-drop source s1;
-
-statement ok
-drop table t1;
diff --git a/e2e_test/source/cdc/cdc.check_new_rows.slt b/e2e_test/source/cdc/cdc.check_new_rows.slt
index 0997f6efc71a5..cac4896f3fa58 100644
--- a/e2e_test/source/cdc/cdc.check_new_rows.slt
+++ b/e2e_test/source/cdc/cdc.check_new_rows.slt
@@ -15,7 +15,7 @@ select cnt from shipments_cnt;
 4
 
 query ITTTT
-select * from person_new order by id;
+SELECT id,name,email_address,credit_card,city from person_new order by id;
 ----
 1000 vicky noris yplkvgz@qbxfg.com 7878 5821 1864 2539 cheyenne
 1001 peter white myckhsp@xpmpe.com 1781 2313 8157 6974 boise
@@ -68,7 +68,7 @@ SELECT * from orders_test_cnt
 5
 
 query ITT
-SELECT * FROM rw.products_test order by id limit 3
+SELECT id,name,description FROM rw.products_test order by id limit 3
 ----
 101 RW Small 2-wheel scooter
 102 RW 12V car battery
@@ -120,3 +120,204 @@ select * from enum_to_varchar order by id;
 1 happy
 2 ok
 3 sad
+
+query II
+select id, my_int from list_with_null_shared order by id;
+----
+1 {1,2,NULL}
+2 {NULL,3,4}
+3 {NULL,-3,-4}
+4 {-4,-5,-6}
+5 NULL
+6 NULL
+
+# my_num: varchar[]
+query II
+select id, my_num from list_with_null_shared order by id;
+----
+1 NULL
+2 {2.2,0,NULL}
+3 NULL
+4 {NULL,-99999999999999999.9999}
+5 NULL
+6 NULL
+
+# my_num1: numeric[]
+query II
+select id, my_num_1 from list_with_null_shared order by id;
+----
+1 NULL
+2 {2.2,0,NULL}
+3 NULL
+4 {NULL,-99999999999999999.9999}
+5 NULL
+6 NULL
+
+# my_num2: rw_int256[]
+query II
+select id, my_num_2 from list_with_null_shared order by id;
+----
+1 NULL
+2 NULL
+3 NULL
+4 NULL
+5 NULL
+6 NULL
+
+# Due to the bug in Debezium, if a enum list contains `NULL`, the list will be converted to `NULL`
+query II
+select id, my_mood from list_with_null_shared order by id;
+----
+1 NULL
+2 {happy,ok,sad}
+3 NULL
+4 NULL
+5 NULL
+6 NULL
+
+query II
+select id, my_uuid from list_with_null_shared order by id;
+----
+1 {bb488f9b-330d-4012-b849-12adeb49e57e,bb488f9b-330d-4012-b849-12adeb49e57f,NULL}
+2 {2de296df-eda7-4202-a81f-1036100ef4f6,2977afbc-0b12-459c-a36f-f623fc9e9840}
+3 {NULL,471acecf-a4b4-4ed3-a211-7fb2291f159f,9bc35adf-fb11-4130-944c-e7eadb96b829}
+4 {b2e4636d-fa03-4ad4-bf16-029a79dca3e2}
+5 NULL
+6 NULL
+
+query II
+select id, my_bytea from list_with_null_shared order by id;
+----
+1 {"\\x00","\\x01",NULL}
+2 {"\\x00","\\x01","\\x02"}
+3 {NULL,"\\x99","\\xaa"}
+4 {"\\x88","\\x99","\\xaa"}
+5 NULL
+6 NULL
+
+query TTTTTTT
+SELECT c_boolean, c_smallint, c_integer, c_bigint, c_decimal, c_real, c_double_precision
+FROM postgres_all_types
+ORDER BY c_boolean, c_bigint, c_date;
+----
+f -32767 -2147483647 -9223372036854775807 -10.0 -10000 -10000
+f 0      0           0                    0      0      0
+f NULL   NULL        1                    NULL   NULL   NULL
+f 1      123         1234567890           123.45 123.45 123.456
+t -32767 -2147483647 -9223372036854775807 -10.0  -10000 -10000
+t 0      0           0                    0      0      0
+t NULL   NULL        1                    NULL   NULL   NULL
+t 1      123         1234567890           123.45 123.45 123.456
+
+query TTTTTTT
+SELECT c_varchar, c_bytea
+FROM postgres_all_types
+ORDER BY c_boolean, c_bigint, c_date;
+----
+d       \x00
+(empty) \x00
+NULL    NULL
+example \xdeadbeef
+d       \x00
+(empty) \x00
+NULL    NULL
+example \xdeadbeef
+
+query TTTTTTT
+SELECT c_date, c_time, c_timestamp, c_timestamptz, c_interval
+FROM postgres_all_types
+ORDER BY c_boolean, c_bigint, c_date;
+----
+0001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 08:00:00+00:00 00:00:00
+0001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 08:00:00+00:00 00:00:00
+0024-05-19 NULL     NULL                NULL                      NULL
+0024-01-01 12:34:56 2024-05-19 12:34:56 2024-05-19 12:34:56+00:00 1 day
+0001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 08:00:00+00:00 00:00:00
+0001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 08:00:00+00:00 00:00:00
+0024-05-19 NULL     NULL                NULL                      NULL
+0024-01-01 12:34:56 2024-05-19 12:34:56 2024-05-19 12:34:56+00:00 1 day
+
+query TTTTTTT
+SELECT c_jsonb, c_uuid, c_enum
+FROM postgres_all_types
+ORDER BY c_boolean, c_bigint, c_date;
+----
+{}               bb488f9b-330d-4012-b849-12adeb49e57e happy
+{}               NULL                                 sad
+NULL             NULL                                 NULL
+{"key": "value"} 123e4567-e89b-12d3-a456-426614174000 happy
+{}               bb488f9b-330d-4012-b849-12adeb49e57e happy
+{}               NULL                                 sad
+NULL             NULL                                 NULL
+{"key": "value"} 123e4567-e89b-12d3-a456-426614174000 happy
+
+query TTTTTTT
+SELECT c_boolean_array, c_smallint_array, c_integer_array, c_bigint_array
+FROM postgres_all_types
+ORDER BY c_boolean, c_bigint, c_date;
+----
+{f}        {-32767}       {-2147483647}     {-9223372036854775807}
+{}         {}             {}                {}
+NULL       NULL           NULL              NULL
+{NULL,t}   {NULL,1}       {NULL,123}        {NULL,1234567890}
+{f}        {-32767}       {-2147483647}     {-9223372036854775807}
+{}         {}             {}                {}
+NULL       NULL           NULL              NULL
+{NULL,t}   {NULL,1}       {NULL,123}        {NULL,1234567890}
+
+
+query TTTTTTT
+SELECT c_decimal_array, c_real_array, c_double_precision_array
+FROM postgres_all_types
+ORDER BY c_boolean, c_bigint, c_date;
+----
+{-10.0}        {-10000}       {-10000}
+{}             {}             {}
+NULL           NULL           NULL
+{NULL,123.45}  {NULL,123.45}  {NULL,123.456}
+{-10.0}        {-10000}       {-10000}
+{}             {}             {}
+NULL           NULL           NULL
+{NULL,123.45}  {NULL,123.45}  {NULL,123.456}
+
+query TTTTTTT
+SELECT c_varchar_array, c_bytea_array
+FROM postgres_all_types
+ORDER BY c_boolean, c_bigint, c_date;
+----
+{""}           {"\\x00"}
+{}             {}
+NULL           NULL
+{NULL,example} {NULL,"\\xdeadbeef"}
+{""}           {"\\x00"}
+{}             {}
+NULL           NULL
+{NULL,example} {NULL,"\\xdeadbeef"}
+
+query TTTTTTT
+SELECT c_date_array, c_time_array, c_timestamp_array, c_timestamptz_array, c_interval_array
+FROM postgres_all_types
+ORDER BY c_boolean, c_bigint, c_date;
+----
+{0001-01-01}      {00:00:00}      {"2001-01-01 00:00:00"}      {"2001-01-01 08:00:00+00:00"}      NULL
+{}                {}              {}                           {}                                 NULL
+NULL              NULL            NULL                         NULL                               NULL
+{NULL,2024-05-19} {NULL,12:34:56} {NULL,"2024-05-19 12:34:56"} {NULL,"2024-05-19 12:34:56+00:00"} NULL
+{0001-01-01}      {00:00:00}      {"2001-01-01 00:00:00"}      {"2001-01-01 08:00:00+00:00"}      NULL
+{}                {}              {}                           {}                                 NULL
+NULL              NULL            NULL                         NULL                               NULL
+{NULL,2024-05-19} {NULL,12:34:56} {NULL,"2024-05-19 12:34:56"} {NULL,"2024-05-19 12:34:56+00:00"} NULL
+
+query TTTTTTT
+SELECT c_jsonb_array, c_uuid_array, c_enum_array
+FROM postgres_all_types
+ORDER BY c_boolean, c_bigint, c_date;
+----
+{"{}"}                        {bb488f9b-330d-4012-b849-12adeb49e57e}      NULL
+{}                            {}                                          {}
+NULL                          NULL                                        NULL
+{NULL,"{\"key\": \"value\"}"} {NULL,123e4567-e89b-12d3-a456-426614174000} NULL
+{"{}"}                        {bb488f9b-330d-4012-b849-12adeb49e57e}      NULL
+{}                            {}                                          {}
+NULL                          NULL                                        NULL
+{NULL,"{\"key\": \"value\"}"} {NULL,123e4567-e89b-12d3-a456-426614174000} NULL
diff --git a/e2e_test/source/cdc/cdc.create_source_job.slt b/e2e_test/source/cdc/cdc.create_source_job.slt
deleted file mode 100644
index 1f2300087110b..0000000000000
--- a/e2e_test/source/cdc/cdc.create_source_job.slt
+++ /dev/null
@@ -1,15 +0,0 @@
-control substitution on
-
-# create a cdc source job, which format fixed to `FORMAT PLAIN ENCODE JSON`
-statement ok
-create source mysql_source with (
- connector = 'mysql-cdc',
- hostname = '${MYSQL_HOST:localhost}',
- port = '${MYSQL_TCP_PORT:8306}',
- username = 'rwcdc',
- password = '${MYSQL_PWD:}',
- database.name = 'mytest',
- server.id = '5001'
-);
-
-sleep 2s
diff --git a/e2e_test/source/cdc/cdc.load.slt b/e2e_test/source/cdc/cdc.load.slt
index 6de27bfe8e5da..2c372cbd3ffdd 100644
--- a/e2e_test/source/cdc/cdc.load.slt
+++ b/e2e_test/source/cdc/cdc.load.slt
@@ -161,7 +161,7 @@ create table shipments_2 (
 # Test user-provided publication
 statement ok
 create table t1_rw (
-    v1 int primary key,
+    "V1" int primary key,
     v3 varchar
 ) with (
   connector = 'postgres-cdc',
diff --git a/e2e_test/source/cdc/cdc.share_stream.slt b/e2e_test/source/cdc/cdc.share_stream.slt
index 2258be2fbb7fc..3dc26d98c6282 100644
--- a/e2e_test/source/cdc/cdc.share_stream.slt
+++ b/e2e_test/source/cdc/cdc.share_stream.slt
@@ -27,10 +27,9 @@ statement error Should not create MATERIALIZED VIEW or SELECT directly on shared
 create materialized view mv as select * from mysql_mytest;
 
 statement error The upstream table name must contain database name prefix*
-create table products_test ( id INT,
+create table products_test ( id INT PRIMARY KEY,
    name STRING,
-   description STRING,
-   PRIMARY KEY (id)
+   description STRING
 ) from mysql_mytest table 'products';
 
 statement ok
@@ -41,7 +40,13 @@ create table rw.products_test ( id INT,
  name STRING,
  description STRING,
  PRIMARY KEY (id)
-) from mysql_mytest table 'mytest.products';
+) include timestamp as commit_ts
+include database_name as database_name
+include table_name as table_name
+from mysql_mytest table 'mytest.products';
+
+# sleep to ensure (default,'Milk','Milk is a white liquid food') is consumed from Debezium message instead of backfill.
+sleep 10s
 
 system ok
 mysql --protocol=tcp -u root mytest -e "INSERT INTO products VALUES (default,'Milk','Milk is a white liquid food');
@@ -145,12 +150,33 @@ SELECT * from orders_test_cnt
 4
 
 query ITT
-SELECT * FROM rw.products_test order by id limit 3
+SELECT id,name,description FROM rw.products_test order by id limit 3
 ----
 101 scooter Small 2-wheel scooter
 102 car battery 12V car battery
 103 12-pack drill bits 12-pack of drill bits with sizes ranging from #40 to #3
 
+query TT
+select database_name, table_name from rw.products_test limit 3;
+----
+mytest products
+mytest products
+mytest products
+
+
+# commit_ts of historical records should be '1970-01-01 00:00:00+00:00'
+query I
+SELECT count(*) as cnt from rw.products_test where commit_ts = '1970-01-01 00:00:00+00:00'
+----
+9
+
+# commit_ts of new records should greater than '1970-01-01 00:00:00+00:00'
+query TTT
+SELECT name,description FROM rw.products_test where commit_ts > '1970-01-01 00:00:00+00:00' order by id
+----
+Milk Milk is a white liquid food
+Juice 100ml Juice
+
 query ITTT
 SELECT order_id,order_date,customer_name,product_id FROM orders_test order by order_id limit 3
 ----
@@ -193,6 +219,8 @@ CREATE TABLE IF NOT EXISTS postgres_all_types(
  c_timestamptz timestamptz,
  c_interval interval,
  c_jsonb jsonb,
+ c_uuid varchar,
+ c_enum varchar,
  c_boolean_array boolean[],
  c_smallint_array smallint[],
  c_integer_array integer[],
@@ -208,18 +236,18 @@ CREATE TABLE IF NOT EXISTS postgres_all_types(
  c_timestamptz_array timestamptz[],
  c_interval_array interval[],
  c_jsonb_array jsonb[],
- c_uuid varchar,
+ c_uuid_array varchar[],
+ c_enum_array varchar[],
  PRIMARY KEY (c_boolean,c_bigint,c_date)
 ) from pg_source table 'public.postgres_all_types';
 
 statement error The upstream table name must contain schema name prefix*
 CREATE TABLE person_new (
-    id int,
+    id int PRIMARY KEY,
     name varchar,
     email_address varchar,
     credit_card varchar,
-    city varchar,
-    PRIMARY KEY (id)
+    city varchar
 ) FROM pg_source TABLE 'person';
 
 statement ok
@@ -230,7 +258,11 @@ CREATE TABLE person_new (
     credit_card varchar,
     city varchar,
     PRIMARY KEY (id)
-) FROM pg_source TABLE 'public.person';
+) INCLUDE TIMESTAMP AS commit_ts
+INCLUDE DATABASE_NAME as database_name
+INCLUDE SCHEMA_NAME as schema_name
+INCLUDE TABLE_NAME as table_name
+FROM pg_source TABLE 'public.person';
 
 statement ok
 CREATE MATERIALIZED VIEW person_new_cnt AS SELECT COUNT(*) AS cnt FROM person_new;
@@ -240,7 +272,7 @@ sleep 3s
 query TTTTTTT
 SELECT c_boolean,c_date,c_time,c_timestamp,c_jsonb,c_smallint_array,c_timestamp_array,c_uuid FROM postgres_all_types where c_bigint=-9223372036854775807
 ----
-f 0001-01-01 00:00:00 0001-01-01 00:00:00 {} {-32767} {"0001-01-01 00:00:00"} bb488f9b-330d-4012-b849-12adeb49e57e
+f 0001-01-01 00:00:00 2001-01-01 00:00:00 {} {-32767} {"2001-01-01 00:00:00"} bb488f9b-330d-4012-b849-12adeb49e57e
 
 
 # postgres streaming test
@@ -259,12 +291,36 @@ SELECT * from person_new_cnt
 ----
 6
 
+query TTT
+SELECT database_name,schema_name,table_name from person_new limit 3;
+----
+cdc_test public person
+cdc_test public person
+cdc_test public person
+
+
+query ITTTT
+SELECT id,name,email_address,credit_card,city from person_new order by id;
+----
+1000 vicky noris yplkvgz@qbxfg.com 7878 5821 1864 2539 cheyenne
+1001 peter white myckhsp@xpmpe.com 1781 2313 8157 6974 boise
+1002 sarah spencer wipvdbm@dkaap.com 3453 4987 9481 6270 los angeles
+1100 noris ypl@qbxfg.com 1864 2539 enne
+1101 white myc@xpmpe.com 8157 6974 se
+1102 spencer wip@dkaap.com 9481 6270 angeles
+
+# historical data
 query ITTTT
-SELECT * from person_new order by id;
+SELECT id,name,email_address,credit_card,city from person_new where commit_ts = '1970-01-01 00:00:00+00:00' order by id;
 ----
 1000 vicky noris yplkvgz@qbxfg.com 7878 5821 1864 2539 cheyenne
 1001 peter white myckhsp@xpmpe.com 1781 2313 8157 6974 boise
 1002 sarah spencer wipvdbm@dkaap.com 3453 4987 9481 6270 los angeles
+
+# incremental data
+query ITTTT
+SELECT id,name,email_address,credit_card,city from person_new where commit_ts > '1970-01-01 00:00:00+00:00' order by id;
+----
 1100 noris ypl@qbxfg.com 1864 2539 enne
 1101 white myc@xpmpe.com 8157 6974 se
 1102 spencer wip@dkaap.com 9481 6270 angeles
@@ -283,6 +339,13 @@ CREATE TABLE numeric_to_varchar_shared (
     PRIMARY KEY (id)
 ) FROM pg_source TABLE 'public.numeric_table';
 
+statement ok
+CREATE TABLE numeric_to_numeric_shared (
+    id int,
+    num numeric,
+    PRIMARY KEY (id)
+) FROM pg_source TABLE 'public.numeric_table';
+
 statement ok
 CREATE TABLE numeric_list_to_rw_int256_list_shared (
     id int,
@@ -297,6 +360,13 @@ CREATE TABLE numeric_list_to_varchar_list_shared (
     PRIMARY KEY (id)
 ) FROM pg_source TABLE 'public.numeric_list';
 
+statement ok
+CREATE TABLE numeric_list_to_numeric_list_shared (
+    id int,
+    num numeric[],
+    PRIMARY KEY (id)
+) FROM pg_source TABLE 'public.numeric_list';
+
 statement ok
 CREATE TABLE enum_to_varchar_shared (
     id int,
@@ -304,6 +374,20 @@ CREATE TABLE enum_to_varchar_shared (
     PRIMARY KEY (id)
 ) FROM pg_source TABLE 'public.enum_table';
 
+
+statement ok
+CREATE TABLE list_with_null_shared (
+    id int,
+    my_int int[],
+    my_num varchar[],
+    my_num_1 numeric[],
+    my_num_2 rw_int256[],
+    my_mood varchar[],
+    my_uuid varchar[],
+    my_bytea bytea[],
+    PRIMARY KEY (id)
+) FROM pg_source TABLE 'public.list_with_null';
+
 system ok
 psql -c "
 insert into numeric_table values(102, 57896044618658097711785492504343953926634992332820282019728792003956564819967);
@@ -315,6 +399,8 @@ insert into numeric_table values(105, 115792089237316195423570985008687907853269
 insert into numeric_table values(106, 'NaN'::numeric);
 insert into numeric_table values(107, 'Infinity'::numeric);
 INSERT INTO enum_table VALUES (2, 'ok');
+insert into numeric_list values(3, '{3.14, 6, 57896044618658097711785492504343953926634992332820282019728792003956564819967, 57896044618658097711785492504343953926634992332820282019728792003956564819968, 115792089237316195423570985008687907853269984665640564039457584007913129639936.555555}');
+INSERT INTO numeric_list values(4, '{nan, infinity, 524596}');
 "
 
 sleep 3s
@@ -354,6 +440,23 @@ select * from numeric_to_rw_int256_shared order by id;
 106 NULL
 107 NULL
 
+query II
+select * from numeric_to_numeric_shared order by id;
+----
+1   3.14
+2   NULL
+3   NULL
+4   NULL
+5   NULL
+6   NaN
+7   Infinity
+102 NULL
+103 NULL
+104 NULL
+105 NULL
+106 NaN
+107 Infinity
+
 system ok
 psql -c "
 DELETE FROM numeric_table WHERE id IN (102, 103, 104, 105, 106, 107);
@@ -363,13 +466,25 @@ query II
 select * from numeric_list_to_varchar_list_shared order by id;
 ----
 1 {3.14,6,57896044618658097711785492504343953926634992332820282019728792003956564819967,57896044618658097711785492504343953926634992332820282019728792003956564819968,115792089237316195423570985008687907853269984665640564039457584007913129639936.555555}
-2 {NAN,POSITIVE_INFINITY,NEGATIVE_INFINITY}
+2 NULL
+3 {3.14,6,57896044618658097711785492504343953926634992332820282019728792003956564819967,57896044618658097711785492504343953926634992332820282019728792003956564819968,115792089237316195423570985008687907853269984665640564039457584007913129639936.555555}
+4 NULL
 
 query II
 select * from numeric_list_to_rw_int256_list_shared order by id;
 ----
-1 {NULL,6,57896044618658097711785492504343953926634992332820282019728792003956564819967,NULL,NULL}
-2 {NULL,NULL,NULL}
+1 NULL
+2 NULL
+3 NULL
+4 NULL
+
+query II
+select * from numeric_list_to_numeric_list_shared order by id;
+----
+1 NULL
+2 NULL
+3 NULL
+4 NULL
 
 query II
 select * from enum_to_varchar_shared order by id;
diff --git a/e2e_test/source/cdc/cdc.validate.postgres.slt b/e2e_test/source/cdc/cdc.validate.postgres.slt
index 9ca43c82f46ba..1392357298fc3 100644
--- a/e2e_test/source/cdc/cdc.validate.postgres.slt
+++ b/e2e_test/source/cdc/cdc.validate.postgres.slt
@@ -1,4 +1,5 @@
 # CDC Postgres source validate test
+control substitution on
 
 # invalid username
 statement error
@@ -217,3 +218,28 @@ explain create table enum_to_varchar (
  table.name = 'enum_table',
  slot.name = 'enum_to_varchar'
 );
+
+# generated column
+statement ok
+create table shipments (
+  shipment_id INTEGER,
+  order_id INTEGER,
+  origin STRING,
+  destination STRING,
+  is_arrived boolean,
+  generated_c INTEGER AS shipment_id + order_id,
+ PRIMARY KEY (shipment_id)
+) with (
+ connector = 'postgres-cdc',
+ hostname = '${PGHOST:localhost}',
+ port = '${PGPORT:5432}',
+ username = '${PGUSER:$USER}',
+ password = '${PGPASSWORD:}',
+ database.name = '${PGDATABASE:postgres}',
+ schema.name = 'public',
+ table.name = 'shipments',
+ slot.name = 'shipments'
+);
+
+statement ok
+drop table shipments;
diff --git a/e2e_test/source/cdc/mongodb/mongodb_basic.slt b/e2e_test/source/cdc/mongodb/mongodb_basic.slt
index f3a815df0572b..ccbd654002984 100644
--- a/e2e_test/source/cdc/mongodb/mongodb_basic.slt
+++ b/e2e_test/source/cdc/mongodb/mongodb_basic.slt
@@ -2,7 +2,11 @@
 control substitution on
 
 statement ok
-CREATE TABLE users (_id JSONB PRIMARY KEY, payload JSONB) WITH (
+CREATE TABLE users (_id JSONB PRIMARY KEY, payload JSONB)
+INCLUDE TIMESTAMP as commit_ts
+INCLUDE DATABASE_NAME as database_name
+INCLUDE COLLECTION_NAME as collection_name
+WITH (
   connector = 'mongodb-cdc',
   mongodb.url = 'mongodb://mongodb:27017/?replicaSet=rs0',
   collection.name = 'random_data.*'
@@ -24,5 +28,18 @@ select count(*) from normalized_users;
 ----
 55
 
+# historical data
+query I
+select count(*) from users where commit_ts = '1970-01-01 00:00:00+00:00';
+----
+55
+
+query TT
+select database_name, collection_name FROM users LIMIT 2;
+----
+random_data users
+random_data users
+
+
 statement ok
 DROP TABLE users cascade
diff --git a/e2e_test/source/cdc/mysql_cdc.sql b/e2e_test/source/cdc/mysql_cdc.sql
index 0ac47fc7fbd1b..2c53e57748163 100644
--- a/e2e_test/source/cdc/mysql_cdc.sql
+++ b/e2e_test/source/cdc/mysql_cdc.sql
@@ -49,9 +49,11 @@ VALUES (1,1,'no'),
        (3,3,'no'),
        (4,4,'no');
 
+-- This user is for non-shared CDC
 CREATE USER 'dbz'@'%' IDENTIFIED BY '123456';
 GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'dbz'@'%';
 
+-- This user is for shared CDC
 CREATE USER 'rwcdc'@'%' IDENTIFIED BY '123456';
 GRANT SELECT, RELOAD, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'rwcdc'@'%';
 
diff --git a/e2e_test/source/cdc/postgres_cdc.sql b/e2e_test/source/cdc/postgres_cdc.sql
index 9344035096cf1..6579bc2683037 100644
--- a/e2e_test/source/cdc/postgres_cdc.sql
+++ b/e2e_test/source/cdc/postgres_cdc.sql
@@ -31,10 +31,11 @@ INSERT INTO person VALUES (1001, 'peter white', 'myckhsp@xpmpe.com', '1781 2313
 INSERT INTO person VALUES (1002, 'sarah spencer', 'wipvdbm@dkaap.com', '3453 4987 9481 6270', 'los angeles');
 
 create schema abs;
-create table abs.t1 (v1 int primary key, v2 double precision, v3 varchar, v4 numeric);
-create publication my_publicaton for table abs.t1 (v1, v3);
+create table abs.t1 ("V1" int primary key, v2 double precision, v3 varchar, v4 numeric);
+create publication my_publicaton for table abs.t1 ("V1", v3);
 insert into abs.t1 values (1, 1.1, 'aaa', '5431.1234');
 
+CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');
 
 CREATE TABLE IF NOT EXISTS postgres_all_types(
      c_boolean boolean,
@@ -52,6 +53,8 @@ CREATE TABLE IF NOT EXISTS postgres_all_types(
      c_timestamptz timestamptz,
      c_interval interval,
      c_jsonb jsonb,
+     c_uuid uuid,
+     c_enum mood,
      c_boolean_array boolean[],
      c_smallint_array smallint[],
      c_integer_array integer[],
@@ -67,11 +70,14 @@ CREATE TABLE IF NOT EXISTS postgres_all_types(
      c_timestamptz_array timestamptz[],
      c_interval_array interval[],
      c_jsonb_array jsonb[],
-     c_uuid uuid,
+     c_uuid_array uuid[],
+     c_enum_array mood[],
      PRIMARY KEY (c_boolean,c_bigint,c_date)
 );
-INSERT INTO postgres_all_types VALUES ( False, 0, 0, 0, 0, 0, 0, '', '\x00', '0001-01-01', '00:00:00', '0001-01-01 00:00:00'::timestamp, '0001-01-01 00:00:00'::timestamptz, interval '0 second', '{}', array[]::boolean[], array[]::smallint[], array[]::integer[], array[]::bigint[], array[]::decimal[], array[]::real[], array[]::double precision[], array[]::varchar[], array[]::bytea[], array[]::date[], array[]::time[], array[]::timestamp[], array[]::timestamptz[], array[]::interval[], array[]::jsonb[], null);
-INSERT INTO postgres_all_types VALUES ( False, -32767, -2147483647, -9223372036854775807, -10.0, -9999.999999, -10000.0, '', '\x00', '0001-01-01', '00:00:00', '0001-01-01 00:00:00'::timestamp, '0001-01-01 00:00:00'::timestamptz, interval '0 second', '{}', array[False::boolean]::boolean[], array[-32767::smallint]::smallint[], array[-2147483647::integer]::integer[], array[-9223372036854775807::bigint]::bigint[], array[-10.0::decimal]::decimal[], array[-9999.999999::real]::real[], array[-10000.0::double precision]::double precision[], array[''::varchar]::varchar[], array['\x00'::bytea]::bytea[], array['0001-01-01'::date]::date[], array['00:00:00'::time]::time[], array['0001-01-01 00:00:00'::timestamp::timestamp]::timestamp[], array['0001-01-01 00:00:00'::timestamptz::timestamptz]::timestamptz[], array[interval '0 second'::interval]::interval[], array['{}'::jsonb]::jsonb[], 'bb488f9b-330d-4012-b849-12adeb49e57e');
+INSERT INTO postgres_all_types VALUES ( False, 0, 0, 0, 0, 0, 0, '', '\x00', '0001-01-01', '00:00:00', '2001-01-01 00:00:00'::timestamp, '2001-01-01 00:00:00-8'::timestamptz, interval '0 second', '{}', null, 'sad', array[]::boolean[], array[]::smallint[], array[]::integer[], array[]::bigint[], array[]::decimal[], array[]::real[], array[]::double precision[], array[]::varchar[], array[]::bytea[], array[]::date[], array[]::time[], array[]::timestamp[], array[]::timestamptz[], array[]::interval[], array[]::jsonb[], array[]::uuid[], array[]::mood[]);
+INSERT INTO postgres_all_types VALUES ( False, -32767, -2147483647, -9223372036854775807, -10.0, -9999.999999, -10000.0, 'd', '\x00', '0001-01-01', '00:00:00', '2001-01-01 00:00:00'::timestamp, '2001-01-01 00:00:00-8'::timestamptz, interval '0 second', '{}', 'bb488f9b-330d-4012-b849-12adeb49e57e', 'happy', array[False::boolean]::boolean[], array[-32767::smallint]::smallint[], array[-2147483647::integer]::integer[], array[-9223372036854775807::bigint]::bigint[], array[-10.0::decimal]::decimal[], array[-9999.999999::real]::real[], array[-10000.0::double precision]::double precision[], array[''::varchar]::varchar[], array['\x00'::bytea]::bytea[], array['0001-01-01'::date]::date[], array['00:00:00'::time]::time[], array['2001-01-01 00:00:00'::timestamp::timestamp]::timestamp[], array['2001-01-01 00:00:00-8'::timestamptz::timestamptz]::timestamptz[], array[interval '0 second'::interval]::interval[], array['{}'::jsonb]::jsonb[], '{bb488f9b-330d-4012-b849-12adeb49e57e}', '{happy,ok,NULL}');
+INSERT INTO postgres_all_types VALUES ( False, 1, 123, 1234567890, 123.45, 123.45, 123.456, 'example', '\xDEADBEEF', '0024-01-01', '12:34:56', '2024-05-19 12:34:56', '2024-05-19 12:34:56+00', INTERVAL '1 day', '{"key": "value"}', '123e4567-e89b-12d3-a456-426614174000', 'happy', ARRAY[NULL, TRUE]::boolean[], ARRAY[NULL, 1::smallint], ARRAY[NULL, 123], ARRAY[NULL, 1234567890], ARRAY[NULL, 123.45::numeric], ARRAY[NULL, 123.45::real], ARRAY[NULL, 123.456], ARRAY[NULL, 'example'], ARRAY[NULL, '\xDEADBEEF'::bytea], ARRAY[NULL, '2024-05-19'::date], ARRAY[NULL, '12:34:56'::time], ARRAY[NULL, '2024-05-19 12:34:56'::timestamp], ARRAY[NULL, '2024-05-19 12:34:56+00'::timestamptz], ARRAY[NULL, INTERVAL '1 day'], ARRAY[NULL, '{"key": "value"}'::jsonb], ARRAY[NULL, '123e4567-e89b-12d3-a456-426614174000'::uuid], ARRAY[NULL, 'happy'::mood]);
+INSERT INTO postgres_all_types VALUES ( False, NULL, NULL, 1, NULL, NULL, NULL, NULL, NULL, '0024-05-19', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
 
 create table numeric_table(id int PRIMARY KEY, num numeric);
 insert into numeric_table values(1, 3.14);
@@ -89,10 +95,13 @@ create table numeric_list(id int primary key, num numeric[]);
 insert into numeric_list values(1, '{3.14, 6, 57896044618658097711785492504343953926634992332820282019728792003956564819967, 57896044618658097711785492504343953926634992332820282019728792003956564819968, 115792089237316195423570985008687907853269984665640564039457584007913129639936.555555}');
 insert into numeric_list values(2, '{nan, infinity, -infinity}');
 
---- for https://github.com/risingwavelabs/risingwave/issues/16392
-CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');
 CREATE TABLE enum_table (
     id int PRIMARY KEY,
     current_mood mood
 );
 INSERT INTO enum_table VALUES (1, 'happy');
+
+CREATE TABLE list_with_null(id int primary key, my_int int[], my_num numeric[], my_num_1 numeric[], my_num_2 numeric[], my_mood mood[], my_uuid uuid[], my_bytea bytea[]);
+INSERT INTO list_with_null VALUES (1, '{1,2,NULL}', '{1.1,inf,NULL}', '{1.1,inf,NULL}', '{1.1,inf,NULL}', '{happy,ok,NULL}', '{bb488f9b-330d-4012-b849-12adeb49e57e,bb488f9b-330d-4012-b849-12adeb49e57f, NULL}', '{\\x00,\\x01,NULL}');
+INSERT INTO list_with_null VALUES (2, '{NULL,3,4}', '{2.2,0,NULL}' , '{2.2,0,NULL}', '{2.2,0,NULL}', '{happy,ok,sad}', '{2de296df-eda7-4202-a81f-1036100ef4f6,2977afbc-0b12-459c-a36f-f623fc9e9840}', '{\\x00,\\x01,\\x02}');
+INSERT INTO list_with_null VALUES (5, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
diff --git a/e2e_test/source/cdc/postgres_cdc_insert.sql b/e2e_test/source/cdc/postgres_cdc_insert.sql
index 5f98b6c015f31..4c0d0dee48b42 100644
--- a/e2e_test/source/cdc/postgres_cdc_insert.sql
+++ b/e2e_test/source/cdc/postgres_cdc_insert.sql
@@ -12,6 +12,11 @@ SELECT pg_current_wal_lsn();
 select * from pg_publication_tables where pubname='rw_publication';
 select * from public.person order by id;
 
+INSERT INTO postgres_all_types VALUES ( True, 0, 0, 0, 0, 0, 0, '', '\x00', '0001-01-01', '00:00:00', '2001-01-01 00:00:00'::timestamp, '2001-01-01 00:00:00-8'::timestamptz, interval '0 second', '{}', null, 'sad', array[]::boolean[], array[]::smallint[], array[]::integer[], array[]::bigint[], array[]::decimal[], array[]::real[], array[]::double precision[], array[]::varchar[], array[]::bytea[], array[]::date[], array[]::time[], array[]::timestamp[], array[]::timestamptz[], array[]::interval[], array[]::jsonb[], array[]::uuid[], array[]::mood[]);
+INSERT INTO postgres_all_types VALUES ( True, -32767, -2147483647, -9223372036854775807, -10.0, -9999.999999, -10000.0, 'd', '\x00', '0001-01-01', '00:00:00', '2001-01-01 00:00:00'::timestamp, '2001-01-01 00:00:00-8'::timestamptz, interval '0 second', '{}', 'bb488f9b-330d-4012-b849-12adeb49e57e', 'happy', array[False::boolean]::boolean[], array[-32767::smallint]::smallint[], array[-2147483647::integer]::integer[], array[-9223372036854775807::bigint]::bigint[], array[-10.0::decimal]::decimal[], array[-9999.999999::real]::real[], array[-10000.0::double precision]::double precision[], array[''::varchar]::varchar[], array['\x00'::bytea]::bytea[], array['0001-01-01'::date]::date[], array['00:00:00'::time]::time[], array['2001-01-01 00:00:00'::timestamp::timestamp]::timestamp[], array['2001-01-01 00:00:00-8'::timestamptz::timestamptz]::timestamptz[], array[interval '0 second'::interval]::interval[], array['{}'::jsonb]::jsonb[], '{bb488f9b-330d-4012-b849-12adeb49e57e}', '{happy,ok,NULL}');
+INSERT INTO postgres_all_types VALUES ( True, 1, 123, 1234567890, 123.45, 123.45, 123.456, 'example', '\xDEADBEEF', '0024-01-01', '12:34:56', '2024-05-19 12:34:56', '2024-05-19 12:34:56+00', INTERVAL '1 day', '{"key": "value"}', '123e4567-e89b-12d3-a456-426614174000', 'happy', ARRAY[NULL, TRUE]::boolean[], ARRAY[NULL, 1::smallint], ARRAY[NULL, 123], ARRAY[NULL, 1234567890], ARRAY[NULL, 123.45::numeric], ARRAY[NULL, 123.45::real], ARRAY[NULL, 123.456], ARRAY[NULL, 'example'], ARRAY[NULL, '\xDEADBEEF'::bytea], ARRAY[NULL, '2024-05-19'::date], ARRAY[NULL, '12:34:56'::time], ARRAY[NULL, '2024-05-19 12:34:56'::timestamp], ARRAY[NULL, '2024-05-19 12:34:56+00'::timestamptz], ARRAY[NULL, INTERVAL '1 day'], ARRAY[NULL, '{"key": "value"}'::jsonb], ARRAY[NULL, '123e4567-e89b-12d3-a456-426614174000'::uuid], ARRAY[NULL, 'happy'::mood]);
+INSERT INTO postgres_all_types VALUES ( True, NULL, NULL, 1, NULL, NULL, NULL, NULL, NULL, '0024-05-19', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+
 insert into numeric_table values(102, 57896044618658097711785492504343953926634992332820282019728792003956564819967);
 --- 2^255
 insert into numeric_table values(103, 57896044618658097711785492504343953926634992332820282019728792003956564819968);
@@ -22,3 +27,8 @@ insert into numeric_table values(106, 'NaN'::numeric);
 insert into numeric_table values(107, 'Infinity'::numeric);
 
 INSERT INTO enum_table VALUES (3, 'sad');
+--- to avoid escaping issues of psql -c "", we insert this row here and check the result in check_new_rows.slt
+INSERT INTO list_with_null VALUES (3, '{NULL,-3,-4}', '{NULL,nan,-inf}', '{NULL,nan,-inf}', '{NULL,nan,-inf}', '{NULL,sad,ok}', '{NULL,471acecf-a4b4-4ed3-a211-7fb2291f159f,9bc35adf-fb11-4130-944c-e7eadb96b829}', '{NULL,\\x99,\\xAA}');
+INSERT INTO list_with_null VALUES (4, '{-4,-5,-6}', '{NULL,-99999999999999999.9999}', '{NULL,-99999999999999999.9999}', '{NULL,-99999999999999999.9999}', '{NULL,sad,ok}', '{b2e4636d-fa03-4ad4-bf16-029a79dca3e2}', '{\\x88,\\x99,\\xAA}');
+INSERT INTO list_with_null VALUES (6, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+
diff --git a/e2e_test/source/cdc_inline/auto_schema_map_mysql.slt b/e2e_test/source/cdc_inline/auto_schema_map_mysql.slt
new file mode 100644
index 0000000000000..5d27c73765e85
--- /dev/null
+++ b/e2e_test/source/cdc_inline/auto_schema_map_mysql.slt
@@ -0,0 +1,118 @@
+control substitution on
+
+# test case need to cover all data types
+system ok
+mysql -e "DROP DATABASE IF EXISTS mytest; CREATE DATABASE mytest;"
+
+system ok
+mysql --protocol=tcp -u root mytest -e "
+    DROP TABLE IF EXISTS mysql_types_test;
+    CREATE TABLE IF NOT EXISTS mysql_types_test(
+        c_boolean boolean,
+        c_bit bit,
+        c_tinyint tinyint,
+        c_smallint smallint,
+        c_mediumint mediumint,
+        c_integer integer,
+        c_Bigint bigint,
+        c_decimal decimal,
+        c_float float,
+        c_double double,
+        c_char_255 char(255),
+        c_varchar_10000 varchar(10000),
+        c_binary_255 binary(255),
+        c_varbinary_10000 varbinary(10000),
+        c_date date,
+        c_time time,
+        c_datetime datetime,
+        c_timestamp timestamp,
+        c_enum ENUM('happy','sad','ok'),
+        c_json JSON,
+        PRIMARY KEY (c_boolean,c_Bigint,c_date)
+    );
+    INSERT INTO mysql_types_test VALUES ( False, 0, null, null, -8388608, -2147483647, 9223372036854775806, -10.0, -9999.999999, -10000.0, 'c', 'd', '', '', '1001-01-01', '-838:59:59.000000', '2000-01-01 00:00:00.000000', null, 'happy', '[1,2]');
+    INSERT INTO mysql_types_test VALUES ( True, 1, -128, -32767, -8388608, -2147483647, -9223372036854775807, -10.0, -9999.999999, -10000.0, 'a', 'b', '', '', '1001-01-01', '00:00:00', '1998-01-01 00:00:00.000000', '1970-01-01 00:00:01', 'sad', '[3,4]');
+    "
+
+
+statement ok
+create source mysql_source with (
+ connector = 'mysql-cdc',
+ hostname = '${MYSQL_HOST:localhost}',
+ port = '${MYSQL_TCP_PORT:8306}',
+ username = 'root',
+ password = '${MYSQL_PWD:}',
+ database.name = 'mytest',
+ server.id = '5601'
+);
+
+
+statement ok
+create table rw_mysql_types_test (*) from mysql_source table 'mytest.mysql_types_test';
+
+sleep 3s
+
+# Name, Type, Is Hidden, Description
+query TTTT
+describe rw_mysql_types_test;
+----
+c_boolean smallint false NULL
+c_bit boolean false NULL
+c_tinyint smallint false NULL
+c_smallint smallint false NULL
+c_mediumint integer false NULL
+c_integer integer false NULL
+c_Bigint bigint false NULL
+c_decimal numeric false NULL
+c_float real false NULL
+c_double double precision false NULL
+c_char_255 character varying false NULL
+c_varchar_10000 character varying false NULL
+c_binary_255 bytea false NULL
+c_varbinary_10000 bytea false NULL
+c_date date false NULL
+c_time time without time zone false NULL
+c_datetime timestamp without time zone false NULL
+c_timestamp timestamp with time zone false NULL
+c_enum character varying false NULL
+c_json jsonb false NULL
+primary key c_boolean, c_Bigint, c_date NULL NULL
+distribution key c_boolean, c_Bigint, c_date NULL NULL
+table description rw_mysql_types_test NULL NULL
+
+query TTTTTTTTTTTTT
+SELECT
+    c_boolean,
+    c_bit,
+    c_tinyint,
+    c_smallint,
+    c_mediumint,
+    c_integer,
+    "c_Bigint",
+    c_decimal,
+    c_float,
+    c_double,
+    c_char_255,
+    c_varchar_10000,
+    c_binary_255
+FROM rw_mysql_types_test order by c_boolean;
+----
+0 NULL NULL NULL -8388608 -2147483647 9223372036854775806 -10 -10000 -10000 c d \x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+1 NULL -128 -32767 -8388608 -2147483647 -9223372036854775807 -10 -10000 -10000 a b \x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+
+query TTTTTTTT
+SELECT
+    c_varbinary_10000,
+    c_date,
+    c_time,
+    c_datetime,
+    c_timestamp,
+    c_enum,
+    c_json
+FROM rw_mysql_types_test order by c_boolean;
+----
+\x 1001-01-01 NULL 2000-01-01 00:00:00 NULL happy [1, 2]
+\x 1001-01-01 00:00:00 1998-01-01 00:00:00 1970-01-01 00:00:01+00:00 sad [3, 4]
+
+statement ok
+drop source mysql_source cascade;
diff --git a/e2e_test/source/cdc_inline/auto_schema_map_pg.slt b/e2e_test/source/cdc_inline/auto_schema_map_pg.slt
new file mode 100644
index 0000000000000..8183a617293b0
--- /dev/null
+++ b/e2e_test/source/cdc_inline/auto_schema_map_pg.slt
@@ -0,0 +1,168 @@
+control substitution on
+
+# test case need to cover all data types
+system ok
+psql -c "
+    DROP TABLE IF EXISTS postgres_types_test;
+    CREATE TABLE IF NOT EXISTS postgres_types_test(
+         c_boolean boolean,
+         c_smallint smallint,
+         c_integer integer,
+         c_bigint bigint,
+         c_decimal decimal,
+         c_real real,
+         c_double_precision double precision,
+         c_varchar varchar,
+         c_bytea bytea,
+         c_date date,
+         c_time time,
+         c_timestamp timestamp,
+         c_timestamptz timestamptz,
+         c_interval interval,
+         c_jsonb jsonb,
+         c_uuid uuid,
+         c_enum mood,
+         c_boolean_array boolean[],
+         c_smallint_array smallint[],
+         c_integer_array integer[],
+         c_bigint_array bigint[],
+         c_decimal_array decimal[],
+         c_real_array real[],
+         c_double_precision_array double precision[],
+         c_varchar_array varchar[],
+         c_bytea_array bytea[],
+         c_date_array date[],
+         c_time_array time[],
+         c_timestamp_array timestamp[],
+         c_timestamptz_array timestamptz[],
+         c_interval_array interval[],
+         c_jsonb_array jsonb[],
+         c_uuid_array uuid[],
+         c_enum_array mood[],
+         PRIMARY KEY (c_boolean,c_bigint,c_date)
+    );
+    INSERT INTO postgres_types_test VALUES ( False, 0, 0, 0, 0, 0, 0, '', '00'::bytea, '0001-01-01', '00:00:00', '2001-01-01 00:00:00'::timestamp, '2001-01-01 00:00:00-8'::timestamptz, interval '0 second', '{}', null, 'sad', array[]::boolean[], array[]::smallint[], array[]::integer[], array[]::bigint[], array[]::decimal[], array[]::real[], array[]::double precision[], array[]::varchar[], array[]::bytea[], array[]::date[], array[]::time[], array[]::timestamp[], array[]::timestamptz[], array[]::interval[], array[]::jsonb[], array[]::uuid[], array[]::mood[]);
+    INSERT INTO postgres_types_test VALUES ( False, -32767, -2147483647, -9223372036854775807, -10.0, -9999.999999, -10000.0, 'd', '00'::bytea, '0001-01-01', '00:00:00', '2001-01-01 00:00:00'::timestamp, '2001-01-01 00:00:00-8'::timestamptz, interval '0 second', '{}', 'bb488f9b-330d-4012-b849-12adeb49e57e', 'happy', array[False::boolean]::boolean[], array[-32767::smallint]::smallint[], array[-2147483647::integer]::integer[], array[-9223372036854775807::bigint]::bigint[], array[-10.0::decimal]::decimal[], array[-9999.999999::real]::real[], array[-10000.0::double precision]::double precision[], array[''::varchar]::varchar[], array['00'::bytea]::bytea[], array['0001-01-01'::date]::date[], array['00:00:00'::time]::time[], array['2001-01-01 00:00:00'::timestamp::timestamp]::timestamp[], array['2001-01-01 00:00:00-8'::timestamptz::timestamptz]::timestamptz[], array[interval '0 second'::interval]::interval[], array['{}'::jsonb]::jsonb[], '{bb488f9b-330d-4012-b849-12adeb49e57e}', '{happy,ok,sad}');
+    INSERT INTO postgres_types_test VALUES ( False, 1, 123, 1234567890, 123.45, 123.45, 123.456, 'a_varchar', 'DEADBEEF'::bytea, '0024-01-01', '12:34:56', '2024-05-19 12:34:56', '2024-05-19 12:34:56+00', INTERVAL '1 day', to_jsonb('hello'::text), '123e4567-e89b-12d3-a456-426614174000', 'happy', ARRAY[NULL, TRUE]::boolean[], ARRAY[NULL, 1::smallint], ARRAY[NULL, 123], ARRAY[NULL, 1234567890], ARRAY[NULL, 123.45::numeric], ARRAY[NULL, 123.45::real], ARRAY[NULL, 123.456], ARRAY[NULL, 'a_varchar'], ARRAY[NULL, 'DEADBEEF'::bytea], ARRAY[NULL, '2024-05-19'::date], ARRAY[NULL, '12:34:56'::time], ARRAY[NULL, '2024-05-19 12:34:56'::timestamp], ARRAY[NULL, '2024-05-19 12:34:56+00'::timestamptz], ARRAY[NULL, INTERVAL '1 day'], ARRAY[NULL, to_jsonb('hello'::text)], ARRAY[NULL, '123e4567-e89b-12d3-a456-426614174000'::uuid], ARRAY[NULL, 'happy'::mood]);
+    INSERT INTO postgres_types_test VALUES ( False, NULL, NULL, 1, NULL, NULL, NULL, NULL, NULL, '0024-05-19', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+    "
+
+statement ok
+create source pg_source with (
+ connector = 'postgres-cdc',
+ hostname = '${PGHOST:localhost}',
+ port = '${PGPORT:5432}',
+ username = '${PGUSER:$USER}',
+ password = '${PGPASSWORD:}',
+ database.name = '${PGDATABASE:postgres}',
+ slot.name = 'pg_slot'
+);
+
+
+statement ok
+create table rw_postgres_types_test (*) from pg_source table 'public.postgres_types_test';
+
+sleep 3s
+
+# Name, Type, Is Hidden, Description
+query TTTT
+describe rw_postgres_types_test;
+----
+c_boolean boolean false NULL
+c_smallint smallint false NULL
+c_integer integer false NULL
+c_bigint bigint false NULL
+c_decimal numeric false NULL
+c_real real false NULL
+c_double_precision double precision false NULL
+c_varchar character varying false NULL
+c_bytea bytea false NULL
+c_date date false NULL
+c_time time without time zone false NULL
+c_timestamp timestamp without time zone false NULL
+c_timestamptz timestamp with time zone false NULL
+c_interval interval false NULL
+c_jsonb jsonb false NULL
+c_uuid character varying false NULL
+c_enum character varying false NULL
+c_boolean_array boolean[] false NULL
+c_smallint_array smallint[] false NULL
+c_integer_array integer[] false NULL
+c_bigint_array bigint[] false NULL
+c_decimal_array numeric[] false NULL
+c_real_array real[] false NULL
+c_double_precision_array double precision[] false NULL
+c_varchar_array character varying[] false NULL
+c_bytea_array bytea[] false NULL
+c_date_array date[] false NULL
+c_time_array time without time zone[] false NULL
+c_timestamp_array timestamp without time zone[] false NULL
+c_timestamptz_array timestamp with time zone[] false NULL
+c_interval_array interval[] false NULL
+c_jsonb_array jsonb[] false NULL
+c_uuid_array character varying[] false NULL
+c_enum_array character varying[] false NULL
+primary key c_boolean, c_bigint, c_date NULL NULL
+distribution key c_boolean, c_bigint, c_date NULL NULL
+table description rw_postgres_types_test NULL NULL
+
+query TTTTTTT
+SELECT
+    c_boolean,
+    c_smallint,
+    c_integer,
+    c_bigint,
+    c_decimal,
+    c_real,
+    c_double_precision,
+    c_varchar,
+    c_bytea from rw_postgres_types_test where c_enum = 'happy' order by c_integer;
+----
+f -32767 -2147483647 -9223372036854775807 -10.0 -10000 -10000 d \x3030
+f 1 123 1234567890 123.45 123.45 123.456 a_varchar \x4445414442454546
+
+query TTTTT
+SELECT
+    c_date,
+    c_time,
+    c_timestamp,
+    c_timestamptz,
+    c_interval from rw_postgres_types_test where c_enum = 'happy' order by c_integer;
+----
+0001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 08:00:00+00:00 00:00:00
+0024-01-01 12:34:56 2024-05-19 12:34:56 2024-05-19 12:34:56+00:00 1 day
+
+query TTTTTTT
+SELECT
+    c_jsonb,
+    c_uuid,
+    c_enum,
+    c_boolean_array,
+    c_smallint_array,
+    c_integer_array,
+    c_bigint_array from rw_postgres_types_test where c_enum = 'happy' order by c_integer;
+----
+{} bb488f9b-330d-4012-b849-12adeb49e57e happy {f} {-32767} {-2147483647} {-9223372036854775807}
+"hello" 123e4567-e89b-12d3-a456-426614174000 happy {NULL,t} {NULL,1} {NULL,123} {NULL,1234567890}
+
+query TTTTTTTTTTTTT
+SELECT
+    c_decimal_array,
+    c_real_array,
+    c_double_precision_array,
+    c_varchar_array,
+    c_bytea_array,
+    c_date_array,
+    c_time_array,
+    c_timestamp_array,
+    c_timestamptz_array,
+    c_interval_array,
+    c_jsonb_array,
+    c_uuid_array,
+    c_enum_array from rw_postgres_types_test where c_enum = 'happy' order by c_integer;
+----
+{-10.0} {-10000} {-10000} {""} {"\\x3030"} {0001-01-01} {00:00:00} {"2001-01-01 00:00:00"} {"2001-01-01 08:00:00+00:00"} NULL {"{}"} {bb488f9b-330d-4012-b849-12adeb49e57e} {happy,ok,sad}
+{NULL,123.45} {NULL,123.45} {NULL,123.456} {NULL,a_varchar} {NULL,"\\x4445414442454546"} {NULL,2024-05-19} {NULL,12:34:56} {NULL,"2024-05-19 12:34:56"} {NULL,"2024-05-19 12:34:56+00:00"} NULL {NULL,"\"hello\""} {NULL,123e4567-e89b-12d3-a456-426614174000} NULL
+
+statement ok
+drop source pg_source cascade;
diff --git a/e2e_test/source/cdc_inline/mysql/mysql_create_drop.slt b/e2e_test/source/cdc_inline/mysql/mysql_create_drop.slt
deleted file mode 100644
index 59f7a4538ef2c..0000000000000
--- a/e2e_test/source/cdc_inline/mysql/mysql_create_drop.slt
+++ /dev/null
@@ -1,133 +0,0 @@
-# create and drop CDC mysql tables concurrently
-
-control substitution on
-
-statement ok
-ALTER SYSTEM SET max_concurrent_creating_streaming_jobs TO 1;
-
-system ok
-mysql --protocol=tcp -u root -e "
-    DROP DATABASE IF EXISTS testdb1; CREATE DATABASE testdb1;
-    USE testdb1;
-    CREATE TABLE tt1 (v1 int primary key, v2 timestamp);
-    INSERT INTO tt1 VALUES (1, '2023-10-23 10:00:00');
-    CREATE TABLE tt2 (v1 int primary key, v2 timestamp);
-    INSERT INTO tt2 VALUES (2, '2023-10-23 11:00:00');
-    CREATE TABLE tt3 (v1 int primary key, v2 timestamp);
-    INSERT INTO tt3 VALUES (3, '2023-10-23 12:00:00');
-    CREATE TABLE tt4 (v1 int primary key, v2 timestamp);
-    INSERT INTO tt4 VALUES (4, '2023-10-23 13:00:00');
-    CREATE TABLE tt5 (v1 int primary key, v2 timestamp);
-    INSERT INTO tt5 VALUES (5, '2023-10-23 14:00:00');"
-
-statement ok
-create table tt1 (v1 int,
-  v2 timestamptz,
-  PRIMARY KEY (v1)
-) with (
-  connector = 'mysql-cdc',
-  hostname = '${MYSQL_HOST:localhost}',
-  port = '${MYSQL_TCP_PORT:8306}',
-  username = 'dbz',
-  password = '${MYSQL_PWD:}',
-  database.name = 'testdb1',
-  table.name = 'tt1',
-);
-
-statement ok
-create table tt2 (v1 int,
-  v2 timestamptz,
-  PRIMARY KEY (v1)
-) with (
-  connector = 'mysql-cdc',
-  hostname = '${MYSQL_HOST:localhost}',
-  port = '${MYSQL_TCP_PORT:8306}',
-  username = 'dbz',
-  password = '${MYSQL_PWD:}',
-  database.name = 'testdb1',
-  table.name = 'tt2',
-);
-
-statement ok
-create table tt3 (v1 int,
-  v2 timestamptz,
-  PRIMARY KEY (v1)
-) with (
-  connector = 'mysql-cdc',
-  hostname = '${MYSQL_HOST:localhost}',
-  port = '${MYSQL_TCP_PORT:8306}',
-  username = 'dbz',
-  password = '${MYSQL_PWD:}',
-  database.name = 'testdb1',
-  table.name = 'tt3',
-);
-
-statement ok
-create table tt4 (v1 int,
-  v2 timestamptz,
-  PRIMARY KEY (v1)
-) with (
-  connector = 'mysql-cdc',
-  hostname = '${MYSQL_HOST:localhost}',
-  port = '${MYSQL_TCP_PORT:8306}',
-  username = 'dbz',
-  password = '${MYSQL_PWD:}',
-  database.name = 'testdb1',
-  table.name = 'tt4',
-);
-
-statement ok
-create table tt5 (v1 int,
-  v2 timestamptz,
-  PRIMARY KEY (v1)
-) with (
-  connector = 'mysql-cdc',
-  hostname = '${MYSQL_HOST:localhost}',
-  port = '${MYSQL_TCP_PORT:8306}',
-  username = 'dbz',
-  password = '${MYSQL_PWD:}',
-  database.name = 'testdb1',
-  table.name = 'tt5',
-);
-
-sleep 5s
-
-query IT
-select * from tt1;
-----
-1 2023-10-23 10:00:00+00:00
-
-query IT
-select * from tt2;
-----
-2 2023-10-23 11:00:00+00:00
-
-query IT
-select * from tt3;
-----
-3 2023-10-23 12:00:00+00:00
-
-query IT
-select * from tt4;
-----
-4 2023-10-23 13:00:00+00:00
-
-query IT
-select * from tt5;
-----
-5 2023-10-23 14:00:00+00:00
-
-statement ok
-drop table tt1;
-
-statement ok
-drop table tt2;
-
-statement ok
-drop table tt3;
-
-statement ok
-drop table tt4;
-
-statement ok
-drop table tt5;
diff --git a/e2e_test/source/cdc_inline/postgres_create_drop.slt b/e2e_test/source/cdc_inline/postgres_create_drop.slt
index 334f1eb2c9cce..ad7425bab3df9 100644
--- a/e2e_test/source/cdc_inline/postgres_create_drop.slt
+++ b/e2e_test/source/cdc_inline/postgres_create_drop.slt
@@ -23,6 +23,13 @@ create table tt1 (v1 int,
   slot.name = 'tt1_slot',
 );
 
+sleep 3s
+
+query IT
+SELECT * FROM tt1;
+----
+1 2023-10-23 10:00:00+00:00
+
 statement ok
 drop table tt1;
 
diff --git a/e2e_test/source/pulsar/astra-streaming.py b/e2e_test/source/pulsar/astra-streaming.py
index 299529e170232..b2e653ed5677b 100644
--- a/e2e_test/source/pulsar/astra-streaming.py
+++ b/e2e_test/source/pulsar/astra-streaming.py
@@ -20,7 +20,7 @@ def do_test(config):
     cur.execute(f'''CREATE TABLE t (v1 int, v2 varchar)
                     WITH (
                     connector='pulsar',
-                    topic='persistent://tenant0/default/topic0',
+                    topic='persistent://meetup/default/ci-test',
                     service.url='pulsar+ssl://pulsar-gcp-useast1.streaming.datastax.com:6651',
                     auth.token='{config['ASTRA_STREAMING_TOKEN']}'
                     )
diff --git a/e2e_test/source/pulsar/pulsar.slt b/e2e_test/source/pulsar/pulsar.slt
index 2072dfb6f8449..29dc30c60994b 100644
--- a/e2e_test/source/pulsar/pulsar.slt
+++ b/e2e_test/source/pulsar/pulsar.slt
@@ -1,8 +1,9 @@
 query IT nosort
 select * FROM t;
 ----
-1 name0
-2 name0
-6 name3
-0 name5
-5 name8
\ No newline at end of file
+1 name1
+2 name2
+3 name3
+4 name4
+5 name5
+6 name6
diff --git a/e2e_test/source_inline/README.md b/e2e_test/source_inline/README.md
index 3a9070639b8cb..d1a84e22099e5 100644
--- a/e2e_test/source_inline/README.md
+++ b/e2e_test/source_inline/README.md
@@ -26,5 +26,5 @@ risedev slt 'e2e_test/source_inline/**/*.slt'
 To write tests, please ensure each file is self-contained and does not depend on running external scripts to setup the environment.
 
 Use `system` command to setup instead.
-For simple cases, you can directly write a bash command;
-For more complex cases, you can write a test script (with any language like bash, python, zx), and invoke it in the `system` command.
+- For simple cases, you can directly write a bash command;
+- For more complex cases, you can write a test script. See also [e2e_test/commands/README.md](../commands/README.md)
diff --git a/e2e_test/source_inline/cdc/mysql/mysql_create_drop.slt b/e2e_test/source_inline/cdc/mysql/mysql_create_drop.slt
new file mode 100644
index 0000000000000..10854d97b6440
--- /dev/null
+++ b/e2e_test/source_inline/cdc/mysql/mysql_create_drop.slt
@@ -0,0 +1,236 @@
+# create and drop CDC mysql tables concurrently
+
+control substitution on
+
+statement ok
+ALTER SYSTEM SET max_concurrent_creating_streaming_jobs TO 1;
+
+system ok
+mysql -e "
+    SET GLOBAL time_zone = '+01:00';
+"
+
+system ok
+mysql -e "
+    DROP DATABASE IF EXISTS testdb1; CREATE DATABASE testdb1;
+    USE testdb1;
+    CREATE TABLE tt1 (v1 int primary key, v2 timestamp);
+    INSERT INTO tt1 VALUES (1, '2023-10-23 10:00:00');
+    CREATE TABLE tt2 (v1 int primary key, v2 timestamp);
+    INSERT INTO tt2 VALUES (2, '2023-10-23 11:00:00');
+    CREATE TABLE tt3 (v1 int primary key, v2 timestamp);
+    INSERT INTO tt3 VALUES (3, '2023-10-23 12:00:00');
+    CREATE TABLE tt4 (v1 int primary key, v2 timestamp);
+    INSERT INTO tt4 VALUES (4, '2023-10-23 13:00:00');
+    CREATE TABLE tt5 (v1 int primary key, v2 timestamp);
+    INSERT INTO tt5 VALUES (5, '2023-10-23 14:00:00');
+"
+
+system ok
+mysql -e "
+    DROP USER IF EXISTS 'non-shared-cdc'@'%';
+    CREATE USER 'non-shared-cdc'@'%' IDENTIFIED BY '123456';
+    GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'non-shared-cdc'@'%';
+    #
+    DROP USER IF EXISTS 'shared-cdc'@'%';
+    CREATE USER 'shared-cdc'@'%' IDENTIFIED BY 'abcdef';
+    GRANT SELECT, RELOAD, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'shared-cdc'@'%';
+    #
+    FLUSH PRIVILEGES;
+"
+
+statement ok
+create source s with (
+  ${RISEDEV_MYSQL_WITH_OPTIONS_COMMON},
+  username = 'shared-cdc',
+  password = 'abcdef',
+  database.name = 'testdb1',
+  server.id = '114514'
+);
+
+sleep 2s
+
+# At the beginning, the source is paused. It will resume after a downstream is created.
+system ok
+internal_table.mjs --name s --type '' --count
+----
+count: 0
+
+
+statement ok
+create table tt1_shared (v1 int,
+  v2 timestamptz,
+  PRIMARY KEY (v1)
+) from s table 'testdb1.tt1';
+
+sleep 2s
+
+# The source is resumed.
+# SourceExecutor does not handle historical data, and only receives new data after it's created.
+# But it can receive offset update at the beginning and periodically
+# via the heartbeat message.
+system ok
+internal_table.mjs --name s --type '' --count
+----
+count: 1
+
+
+statement ok
+create table tt1 (v1 int,
+  v2 timestamptz,
+  PRIMARY KEY (v1)
+) with (
+  ${RISEDEV_MYSQL_WITH_OPTIONS_COMMON},
+  username = 'non-shared-cdc',
+  password = '123456',
+  database.name = 'testdb1',
+  table.name = 'tt1',
+);
+
+statement ok
+create table tt2 (v1 int,
+  v2 timestamptz,
+  PRIMARY KEY (v1)
+) with (
+  ${RISEDEV_MYSQL_WITH_OPTIONS_COMMON},
+  username = 'non-shared-cdc',
+  password = '123456',
+  database.name = 'testdb1',
+  table.name = 'tt2',
+);
+
+statement ok
+create table tt3 (v1 int,
+  v2 timestamptz,
+  PRIMARY KEY (v1)
+) with (
+  ${RISEDEV_MYSQL_WITH_OPTIONS_COMMON},
+  username = 'non-shared-cdc',
+  password = '123456',
+  database.name = 'testdb1',
+  table.name = 'tt3',
+);
+
+statement ok
+create table tt4 (v1 int,
+  v2 timestamptz,
+  PRIMARY KEY (v1)
+) with (
+  ${RISEDEV_MYSQL_WITH_OPTIONS_COMMON},
+  username = 'non-shared-cdc',
+  password = '123456',
+  database.name = 'testdb1',
+  table.name = 'tt4',
+);
+
+statement ok
+create table tt5 (v1 int,
+  v2 timestamptz,
+  PRIMARY KEY (v1)
+) with (
+  ${RISEDEV_MYSQL_WITH_OPTIONS_COMMON},
+  username = 'non-shared-cdc',
+  password = '123456',
+  database.name = 'testdb1',
+  table.name = 'tt5',
+);
+
+statement ok
+create table tt2_shared (v1 int,
+  v2 timestamptz,
+  PRIMARY KEY (v1)
+) from s table 'testdb1.tt2';
+
+statement ok
+create table tt3_shared (v1 int,
+  v2 timestamptz,
+  PRIMARY KEY (v1)
+) from s table 'testdb1.tt3';
+
+statement ok
+create table tt4_shared (v1 int,
+  v2 timestamptz,
+  PRIMARY KEY (v1)
+) from s table 'testdb1.tt4';
+
+statement ok
+create table tt5_shared (v1 int,
+  v2 timestamptz,
+  PRIMARY KEY (v1)
+) from s table 'testdb1.tt5';
+
+sleep 5s
+
+query IT
+select * from tt1;
+----
+1 2023-10-23 09:00:00+00:00
+
+query IT
+select * from tt2;
+----
+2 2023-10-23 10:00:00+00:00
+
+query IT
+select * from tt3;
+----
+3 2023-10-23 11:00:00+00:00
+
+query IT
+select * from tt4;
+----
+4 2023-10-23 12:00:00+00:00
+
+query IT
+select * from tt5;
+----
+5 2023-10-23 13:00:00+00:00
+
+
+query IT
+select * from tt1_shared;
+----
+1 2023-10-23 09:00:00+00:00
+
+query IT
+select * from tt2_shared;
+----
+2 2023-10-23 10:00:00+00:00
+
+query IT
+select * from tt3_shared;
+----
+3 2023-10-23 11:00:00+00:00
+
+query IT
+select * from tt4_shared;
+----
+4 2023-10-23 12:00:00+00:00
+
+query IT
+select * from tt5_shared;
+----
+5 2023-10-23 13:00:00+00:00
+
+statement ok
+drop table tt1;
+
+statement ok
+drop table tt2;
+
+statement ok
+drop table tt3;
+
+statement ok
+drop table tt4;
+
+statement ok
+drop table tt5;
+
+statement ok
+drop source s cascade;
+
+system ok
+mysql -e "
+    SET GLOBAL time_zone = '+00:00';
+"
diff --git a/e2e_test/source_inline/commands.toml b/e2e_test/source_inline/commands.toml
index 8af865099ac7c..48342bceafd42 100644
--- a/e2e_test/source_inline/commands.toml
+++ b/e2e_test/source_inline/commands.toml
@@ -1,9 +1,5 @@
 # This file contains commands used by the tests.
 
-[tasks.source-test-hook]
-private = true
-dependencies = ["check-risedev-env-file"]
-env_files = ["${PREFIX_CONFIG}/risedev-env"]
 
 # Note about the Kafka CLI tooling:
 # - Built-in Kafka console tools:
@@ -16,19 +12,14 @@ env_files = ["${PREFIX_CONFIG}/risedev-env"]
 # - rpk:
 #       Golang based.
 #       Style example: RPK_BROKERS=localhost:9092 rpk topic create t
-[tasks.kafka-hook]
+[tasks.check-kafka]
 private = true
 description = "Check if Kafka is started by RiseDev"
-dependencies = ["source-test-hook"]
+dependencies = ["check-and-load-risedev-env-file"]
 script = '''
 #!/usr/bin/env sh
 set -e
 
-if [ ! -d "${PREFIX_BIN}/kafka" ]; then
-    echo "Kafka is not installed in ${PREFIX_BIN}/kafka. Did you enable Kafka using $(tput setaf 4)\`./risedev configure\`$(tput sgr0)?"
-    exit 1
-fi
-
 if [ -z "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" ]; then
     echo "RISEDEV_KAFKA_BOOTSTRAP_SERVERS is not set in risedev-env file. Did you start Kafka using $(tput setaf 4)\`./risedev d\`$(tput sgr0)?"
     exit 1
@@ -38,50 +29,23 @@ fi
 [tasks.clean-kafka]
 category = "RiseDev - Test - Source Test - Kafka"
 description = "Delete all kafka topics."
-dependencies = ["kafka-hook"]
-command = "rpk"
-args = ["topic", "delete", "-r", "*"]
-
-[tasks.kafka-topics]
-category = "RiseDev - Test - Source Test - Kafka"
-dependencies = ["kafka-hook"]
-script = """
-#!/usr/bin/env sh
-set -e
-${PREFIX_BIN}/kafka/bin/kafka-topics.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTSTRAP_SERVERS} "$@"
-"""
-
-[tasks.kafka-produce]
-category = "RiseDev - Test - Source Test - Kafka"
-dependencies = ["kafka-hook"]
-script = """
-#!/usr/bin/env sh
-set -e
-${PREFIX_BIN}/kafka/bin/kafka-console-producer.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTSTRAP_SERVERS} "$@"
-"""
-
-[tasks.kafka-consume]
-category = "RiseDev - Test - Source Test - Kafka"
-dependencies = ["kafka-hook"]
-script = """
+dependencies = ["check-and-load-risedev-env-file"]
+script = '''
 #!/usr/bin/env sh
 set -e
-${PREFIX_BIN}/kafka/bin/kafka-console-consumer.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTSTRAP_SERVERS} "$@"
-"""
 
-[tasks.kafka-consumer-groups]
-category = "RiseDev - Test - Source Test - Kafka"
-dependencies = ["kafka-hook"]
-script = """
-#!/usr/bin/env sh
-set -e
-${PREFIX_BIN}/kafka/bin/kafka-consumer-groups.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTSTRAP_SERVERS} "$@"
-"""
+if [ -n "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" ]; then
+    echo "Deleting all Kafka topics..."
+    rpk topic delete -r "*"
+else
+    echo "No Kafka to clean."
+fi
+'''
 
 # rpk tools
 [tasks.rpk]
 category = "RiseDev - Test - Source Test - Kafka"
-dependencies = ["kafka-hook"]
+dependencies = ["check-kafka"]
 # check https://docs.redpanda.com/current/reference/rpk/rpk-x-options/ or rpk -X help/list for options
 script = """
 #!/usr/bin/env sh
@@ -98,7 +62,7 @@ rpk "$@"
 [tasks.redpanda-console]
 category = "RiseDev - Test - Source Test - Kafka"
 description = "Start Redpanda console (Kafka GUI) at localhost:8080."
-dependencies = ["kafka-hook"]
+dependencies = ["check-kafka"]
 script = '''
 #!/usr/bin/env sh
 set -e
diff --git a/e2e_test/source_inline/kafka/avro/name_strategy.slt b/e2e_test/source_inline/kafka/avro/name_strategy.slt
new file mode 100644
index 0000000000000..737b97316cc9a
--- /dev/null
+++ b/e2e_test/source_inline/kafka/avro/name_strategy.slt
@@ -0,0 +1,181 @@
+control substitution on
+
+system ok
+rpk topic delete 'upsert_avro_json.*'
+
+
+# wrong strategy name
+statement error expect strategy name in topic_name_strategy, record_name_strategy and topic_record_name_strategy
+create source s1 () with (
+  connector = 'kafka',
+  topic = 'upsert_avro_json-record',
+  properties.bootstrap.server = '${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}'
+) format plain encode avro (
+  schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}',
+  schema.registry.name.strategy = 'no sense',
+  message = 'CPLM.OBJ_ATTRIBUTE_VALUE',
+);
+
+
+#######################
+# topic_name_strategy
+#######################
+
+# TODO: refactor the producer script and the test data format.
+# Currently we are abusing this test case to also test data types.
+
+system ok
+python3 scripts/source/schema_registry_producer.py  "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic" "avro"
+
+# If we cannot extract key schema, use message key as varchar primary key
+statement ok
+CREATE TABLE t_topic ( primary key (rw_key) )
+INCLUDE KEY AS rw_key
+WITH (
+	connector = 'kafka',
+  properties.bootstrap.server = '${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}',
+	topic = 'upsert_avro_json')
+FORMAT UPSERT ENCODE AVRO (schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}');
+
+
+#######################
+# record_name_strategy
+#######################
+
+
+## topic: upsert_avro_json-record, key subject: string, value subject: CPLM.OBJ_ATTRIBUTE_VALUE
+system ok
+python3 scripts/source/schema_registry_producer.py  "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "record" "avro"
+
+
+statement error key\.message
+create table t_record_format_plain () with (
+  connector = 'kafka',
+  topic = 'upsert_avro_json-record',
+  properties.bootstrap.server = '${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}'
+) format plain encode avro (
+  schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}',
+  schema.registry.name.strategy = 'record_name_strategy',
+  message = 'CPLM.OBJ_ATTRIBUTE_VALUE',
+  key.message = 'string'
+);
+
+# Note that FORMAT PLAIN is used here.
+statement ok
+create table t_record_format_plain () with (
+  connector = 'kafka',
+  topic = 'upsert_avro_json-record',
+  properties.bootstrap.server = '${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}'
+) format plain encode avro (
+  schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}',
+  schema.registry.name.strategy = 'record_name_strategy',
+  message = 'CPLM.OBJ_ATTRIBUTE_VALUE',
+);
+
+
+#######################
+# topic_record_name_strategy
+#######################
+
+## topic: upsert_avro_json-topic-record,
+## key subject: upsert_avro_json-topic-record-string
+## value subject: upsert_avro_json-topic-record-CPLM.OBJ_ATTRIBUTE_VALUE
+system ok
+python3 scripts/source/schema_registry_producer.py  "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic-record" "avro"
+
+
+
+statement error SCHEMA_REGISTRY_NAME_STRATEGY_TOPIC_RECORD_NAME_STRATEGY expect non-empty field key\.message
+create table t_topic_record () with (
+    connector = 'kafka',
+    topic = 'upsert_avro_json-topic-record',
+    properties.bootstrap.server = '${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}'
+) format upsert encode avro (
+    schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}',
+    schema.registry.name.strategy = 'topic_record_name_strategy',
+    message = 'CPLM.OBJ_ATTRIBUTE_VALUE'
+);
+
+statement ok
+create table t_topic_record (primary key(rw_key))
+INCLUDE KEY AS rw_key
+with (
+    connector = 'kafka',
+    topic = 'upsert_avro_json-topic-record',
+    properties.bootstrap.server = '${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}'
+) format upsert encode avro (
+    schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}',
+    schema.registry.name.strategy = 'topic_record_name_strategy',
+    message = 'CPLM.OBJ_ATTRIBUTE_VALUE',
+    key.message = 'string'
+);
+
+
+#######################
+# test result now
+#######################
+
+sleep 2s
+
+
+query II
+SELECT
+ * except (rw_key)
+FROM
+ t_topic
+ORDER BY
+ "ID";
+----
+update id1 -1 6768 6970 value9 7172 info9 2021-05-18T07:59:58.714Z -21474836.47 NULL NULL NULL
+delete id2 2 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 99999999.99 NULL NULL NULL
+delete id3 3 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 21474836.47 NULL NULL NULL
+delete id5 5 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 21474836.49 NULL NULL NULL
+NULL id6 NULL NULL NULL NULL NULL NULL NULL -0.01 NULL (abcdefg) NULL
+NULL id7 NULL NULL NULL NULL NULL NULL NULL -0.01 NULL NULL 67e55044-10b1-426f-9247-bb680e5fe0c8
+
+
+query II
+SELECT
+  *
+FROM
+ t_record_format_plain
+ORDER BY
+ "ID";
+----
+update id1 1 6768 6970 value9 7172 info9 2021-05-18T07:59:58.714Z 99999999.99 NULL NULL NULL
+update id1 -1 6768 6970 value9 7172 info9 2021-05-18T07:59:58.714Z -21474836.47 NULL NULL NULL
+delete id2 2 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 99999999.99 NULL NULL NULL
+delete id3 3 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 21474836.47 NULL NULL NULL
+delete id4 4 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 21474836.49 NULL NULL NULL
+delete id5 5 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 21474836.49 NULL NULL NULL
+NULL id6 NULL NULL NULL NULL NULL NULL NULL -0.01 NULL (abcdefg) NULL
+NULL id7 NULL NULL NULL NULL NULL NULL NULL -0.01 NULL NULL 67e55044-10b1-426f-9247-bb680e5fe0c8
+NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
+
+
+query II
+SELECT
+ * except (rw_key)
+FROM
+ t_topic_record
+ORDER BY
+ "ID";
+----
+update id1 -1 6768 6970 value9 7172 info9 2021-05-18T07:59:58.714Z -21474836.47 NULL NULL NULL
+delete id2 2 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 99999999.99 NULL NULL NULL
+delete id3 3 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 21474836.47 NULL NULL NULL
+delete id5 5 7778 7980 value10 8182 info10 2021-05-19T15:22:45.539Z 21474836.49 NULL NULL NULL
+NULL id6 NULL NULL NULL NULL NULL NULL NULL -0.01 NULL (abcdefg) NULL
+NULL id7 NULL NULL NULL NULL NULL NULL NULL -0.01 NULL NULL 67e55044-10b1-426f-9247-bb680e5fe0c8
+
+
+
+statement ok
+DROP TABLE t_topic;
+
+statement ok
+DROP TABLE t_record_format_plain;
+
+
+statement ok
+DROP TABLE t_topic_record;
diff --git a/scripts/source/test_data/upsert_avro_json.1 b/e2e_test/source_inline/kafka/avro/upsert_avro_json
similarity index 71%
rename from scripts/source/test_data/upsert_avro_json.1
rename to e2e_test/source_inline/kafka/avro/upsert_avro_json
index 44bba14c145d3..426b27c41fb98 100644
--- a/scripts/source/test_data/upsert_avro_json.1
+++ b/e2e_test/source_inline/kafka/avro/upsert_avro_json
@@ -1,4 +1,4 @@
-"string"^{"type":"record","name":"OBJ_ATTRIBUTE_VALUE","namespace":"CPLM","fields":[{"name":"op_type","type":["null","string"],"default":null},{"name":"ID","type":["null","string"],"default":null},{"name":"CLASS_ID","type":["null","string"],"default":null},{"name":"ITEM_ID","type":["null","string"], "default":null},{"name":"ATTR_ID","type":["null","string"],"default":null},{"name":"ATTR_VALUE","type":["null","string"],"default":null},{"name":"ORG_ID","type":["null","string"],"default":null},{"name":"UNIT_INFO","type":["null","string"],"default":null},{"name":"UPD_TIME","type":["null","string"],"default":null},{"name":"DEC_VAL","type":[{"type":"bytes","logicalType":"decimal","precision":10,"scale":2},"null"],"default":"\u00ff"}],"connect.name":"CPLM.OBJ_ATTRIBUTE_VALUE"}
+"string"^{"type":"record","name":"OBJ_ATTRIBUTE_VALUE","namespace":"CPLM","fields":[{"name":"op_type","type":["null","string"],"default":null},{"name":"ID","type":["null","string"],"default":null},{"name":"CLASS_ID","type":["null","string"],"default":null},{"name":"ITEM_ID","type":["null","string"],"default":null},{"name":"ATTR_ID","type":["null","string"],"default":null},{"name":"ATTR_VALUE","type":["null","string"],"default":null},{"name":"ORG_ID","type":["null","string"],"default":null},{"name":"UNIT_INFO","type":["null","string"],"default":null},{"name":"UPD_TIME","type":["null","string"],"default":null},{"name":"DEC_VAL","type":[{"type":"bytes","logicalType":"decimal","precision":10,"scale":2},"null"],"default":"\u00ff"},{"name":"REFERRED","type":["null",{"type":"record","name":"REFERRED_TYPE","fields":[{"name":"a","type":"string"}]}],"default":null},{"name":"REF","type":["null","REFERRED_TYPE"],"default":null},{"name":"uuid","type":["null",{"type":"string","logicalType":"uuid"}],"default":null}],"connect.name":"CPLM.OBJ_ATTRIBUTE_VALUE"}
 "id1"^{"op_type": {"string": "update"}, "ID": {"string": "id1"}, "CLASS_ID": {"string": "1"}, "ITEM_ID": {"string": "6768"}, "ATTR_ID": {"string": "6970"}, "ATTR_VALUE": {"string": "value9"}, "ORG_ID": {"string": "7172"}, "UNIT_INFO": {"string": "info9"}, "UPD_TIME": {"string": "2021-05-18T07:59:58.714Z"}, "DEC_VAL": {"bytes": "\u0002\u0054\u000b\u00e3\u00ff"}}
 "id2"^{"op_type": {"string": "delete"}, "ID": {"string": "id2"}, "CLASS_ID": {"string": "2"}, "ITEM_ID": {"string": "7778"}, "ATTR_ID": {"string": "7980"}, "ATTR_VALUE": {"string": "value10"}, "ORG_ID": {"string": "8182"}, "UNIT_INFO": {"string": "info10"}, "UPD_TIME": {"string": "2021-05-19T15:22:45.539Z"}, "DEC_VAL": {"bytes": "\u0002\u0054\u000b\u00e3\u00ff"}}
 "id3"^{"op_type": {"string": "delete"}, "ID": {"string": "id3"}, "CLASS_ID": {"string": "3"}, "ITEM_ID": {"string": "7778"}, "ATTR_ID": {"string": "7980"}, "ATTR_VALUE": {"string": "value10"}, "ORG_ID": {"string": "8182"}, "UNIT_INFO": {"string": "info10"}, "UPD_TIME": {"string": "2021-05-19T15:22:45.539Z"}, "DEC_VAL": {"bytes": "\u007f\u00ff\u00ff\u00ff"}}
@@ -6,3 +6,5 @@
 "id5"^{"op_type": {"string": "delete"}, "ID": {"string": "id5"}, "CLASS_ID": {"string": "5"}, "ITEM_ID": {"string": "7778"}, "ATTR_ID": {"string": "7980"}, "ATTR_VALUE": {"string": "value10"}, "ORG_ID": {"string": "8182"}, "UNIT_INFO": {"string": "info10"}, "UPD_TIME": {"string": "2021-05-19T15:22:45.539Z"}, "DEC_VAL": {"bytes": "\u0000\u0080\u0000\u0000\u0001"}}
 "id1"^{"op_type": {"string": "update"}, "ID": {"string": "id1"}, "CLASS_ID": {"string": "-1"}, "ITEM_ID": {"string": "6768"}, "ATTR_ID": {"string": "6970"}, "ATTR_VALUE": {"string": "value9"}, "ORG_ID": {"string": "7172"}, "UNIT_INFO": {"string": "info9"}, "UPD_TIME": {"string": "2021-05-18T07:59:58.714Z"}, "DEC_VAL": {"bytes": "\u0080\u0000\u0000\u0001"}}
 "id4"^
+"id6"^{"ID": {"string": "id6"},"REF": {"CPLM.REFERRED_TYPE":{"a": "abcdefg"}}}
+"id7"^{"ID": {"string": "id7"},"uuid": {"string": "67e55044-10b1-426f-9247-bb680e5fe0c8"}}
diff --git a/e2e_test/source_inline/kafka/b.sh b/e2e_test/source_inline/kafka/b.sh
new file mode 100644
index 0000000000000..75960b7d03bf8
--- /dev/null
+++ b/e2e_test/source_inline/kafka/b.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+for i in {0..9}; do
+cat <<EOF | rpk topic produce shared_source -f "%p %v\\n" -p 0
+0 {"v1": 1, "v2": "a"}
+1 {"v1": 2, "v2": "b"}
+2 {"v1": 3, "v2": "c"}
+3 {"v1": 4, "v2": "d"}
+EOF
+done
\ No newline at end of file
diff --git a/e2e_test/source_inline/kafka/consumer_group.mjs b/e2e_test/source_inline/kafka/consumer_group.mjs
index 12e906631319a..1766a903b09f5 100755
--- a/e2e_test/source_inline/kafka/consumer_group.mjs
+++ b/e2e_test/source_inline/kafka/consumer_group.mjs
@@ -3,11 +3,8 @@
 // zx: A tool for writing better scripts
 // https://google.github.io/zx/
 
-const {
-  mv: mv,
-  _: _command,
-} = minimist(process.argv.slice(3), {
-  string: ["mv", "topic"],
+const { mv: mv, _: _command } = minimist(process.argv.slice(3), {
+  string: ["mv"],
   _: ["list-members", "list-lags"],
 });
 const command = _command[0];
@@ -62,7 +59,7 @@ async function list_consumer_group_members(fragment_id) {
   const groups = await list_consumer_groups(fragment_id);
   return Promise.all(
     groups.map(async (group_name) => {
-      return (await describe_consumer_group(group_name))["MEMBERS"]
+      return (await describe_consumer_group(group_name))["MEMBERS"];
     })
   );
 }
@@ -71,14 +68,14 @@ async function list_consumer_group_lags(fragment_id) {
   const groups = await list_consumer_groups(fragment_id);
   return Promise.all(
     groups.map(async (group_name) => {
-      return (await describe_consumer_group(group_name))["TOTAL-LAG"]
+      return (await describe_consumer_group(group_name))["TOTAL-LAG"];
     })
   );
 }
 
 const fragment_id = await get_fragment_id_of_mv(mv);
 if (command == "list-groups") {
-  echo`${(await list_consumer_groups(fragment_id))}`;
+  echo`${await list_consumer_groups(fragment_id)}`;
 } else if (command == "list-members") {
   echo`${await list_consumer_group_members(fragment_id)}`;
 } else if (command == "list-lags") {
diff --git a/e2e_test/source_inline/kafka/shared_source.slt b/e2e_test/source_inline/kafka/shared_source.slt
new file mode 100644
index 0000000000000..f180e6e0d8351
--- /dev/null
+++ b/e2e_test/source_inline/kafka/shared_source.slt
@@ -0,0 +1,215 @@
+control substitution on
+
+statement ok
+SET rw_enable_shared_source TO true;
+
+system ok
+rpk topic create shared_source -p 4
+
+system ok
+cat << EOF | rpk topic produce shared_source -f "%p %v\\n" -p 0
+0 {"v1": 1, "v2": "a"}
+1 {"v1": 2, "v2": "b"}
+2 {"v1": 3, "v2": "c"}
+3 {"v1": 4, "v2": "d"}
+EOF
+
+statement ok
+create source s0 (v1 int, v2 varchar) with (
+  ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
+  topic = 'shared_source',
+  scan.startup.mode = 'earliest'
+) FORMAT PLAIN ENCODE JSON;
+
+query I
+select count(*) from rw_internal_tables where name like '%s0%';
+----
+1
+
+sleep 1s
+
+# SourceExecutor's ingestion does not start (state table is empty), even after sleep
+system ok
+internal_table.mjs --name s0 --type source
+----
+(empty)
+
+
+statement ok
+create materialized view mv_1 as select * from s0;
+
+# Wait enough time to ensure SourceExecutor consumes all Kafka data.
+sleep 2s
+
+# SourceExecutor's ingestion started, but it only starts from latest.
+system ok
+internal_table.mjs --name s0 --type source
+----
+(empty)
+
+
+# offset 0 must be backfilled, not from upstream.
+system ok
+internal_table.mjs --name mv_1 --type sourcebackfill
+----
+0,"{""Backfilling"": ""0""}"
+1,"{""Backfilling"": ""0""}"
+2,"{""Backfilling"": ""0""}"
+3,"{""Backfilling"": ""0""}"
+
+
+# This does not affect the behavior for CREATE MATERIALIZED VIEW below. It also uses the shared source, and creates SourceBackfillExecutor.
+statement ok
+SET rw_enable_shared_source TO false;
+
+statement ok
+create materialized view mv_2 as select * from s0;
+
+sleep 2s
+
+query IT rowsort
+select v1, v2 from s0;
+----
+1 a
+2 b
+3 c
+4 d
+
+query IT rowsort
+select v1, v2 from mv_1;
+----
+1 a
+2 b
+3 c
+4 d
+
+query IT rowsort
+select v1, v2 from mv_2;
+----
+1 a
+2 b
+3 c
+4 d
+
+system ok
+cat << EOF | rpk topic produce shared_source -f "%p %v\\n" -p 0
+0 {"v1": 1, "v2": "aa"}
+1 {"v1": 2, "v2": "bb"}
+2 {"v1": 3, "v2": "cc"}
+3 {"v1": 4, "v2": "dd"}
+EOF
+
+sleep 2s
+
+# SourceExecutor's finally got new data now.
+system ok
+internal_table.mjs --name s0 --type source
+----
+0,"{""split_info"": {""partition"": 0, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+1,"{""split_info"": {""partition"": 1, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+2,"{""split_info"": {""partition"": 2, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+3,"{""split_info"": {""partition"": 3, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+
+
+query IT rowsort
+select v1, v2 from s0;
+----
+1 a
+1 aa
+2 b
+2 bb
+3 c
+3 cc
+4 d
+4 dd
+
+query IT rowsort
+select v1, v2 from mv_1;
+----
+1 a
+1 aa
+2 b
+2 bb
+3 c
+3 cc
+4 d
+4 dd
+
+
+# start_offset changed to 1
+system ok
+internal_table.mjs --name s0 --type source
+----
+0,"{""split_info"": {""partition"": 0, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+1,"{""split_info"": {""partition"": 1, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+2,"{""split_info"": {""partition"": 2, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+3,"{""split_info"": {""partition"": 3, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+
+
+# The result is non-deterministic:
+# If the upstream row comes before the backfill row, it will be ignored, and the result state is "{""Backfilling"": ""1""}".
+# If the upstream row comes after the backfill row, the result state is Finished.
+# Uncomment below and run manually to see the result.
+
+# system ok
+# internal_table.mjs --name mv_1 --type sourcebackfill
+# ----
+# 0,"{""Finished""}"
+# 1,"{""Finished""}"
+# 2,"{""Finished""}"
+# 3,"{""Finished""}"
+
+
+# Note: heredoc in loop in mac's sh is ok, but not in linux's sh. So we use bash here.
+system ok
+bash -c 'for i in {0..9}; do
+sleep 0.1
+cat <<EOF | rpk topic produce shared_source -f "%p %v\\n" -p 0
+0 {"v1": 1, "v2": "a"}
+1 {"v1": 2, "v2": "b"}
+2 {"v1": 3, "v2": "c"}
+3 {"v1": 4, "v2": "d"}
+EOF
+done'
+
+sleep 3s
+
+query IT rowsort
+select v1, count(*) from s0 group by v1;
+----
+1 12
+2 12
+3 12
+4 12
+
+query IT rowsort
+select v1, count(*) from mv_1 group by v1;
+----
+1 12
+2 12
+3 12
+4 12
+
+
+# start_offset changed to 11
+system ok
+internal_table.mjs --name s0 --type source
+----
+0,"{""split_info"": {""partition"": 0, ""start_offset"": 11, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+1,"{""split_info"": {""partition"": 1, ""start_offset"": 11, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+2,"{""split_info"": {""partition"": 2, ""start_offset"": 11, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+3,"{""split_info"": {""partition"": 3, ""start_offset"": 11, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+
+
+# Now it is highly probable that all partitions have finished.
+system ok
+internal_table.mjs --name mv_1 --type sourcebackfill
+----
+0,"""Finished"""
+1,"""Finished"""
+2,"""Finished"""
+3,"""Finished"""
+
+
+statement ok
+drop source s0 cascade;
diff --git a/e2e_test/source_inline/pubsub/prepare-data.rs b/e2e_test/source_inline/pubsub/prepare-data.rs
new file mode 100755
index 0000000000000..a792c6cbb2618
--- /dev/null
+++ b/e2e_test/source_inline/pubsub/prepare-data.rs
@@ -0,0 +1,81 @@
+#!/usr/bin/env -S cargo -Zscript
+```cargo
+[dependencies]
+anyhow = "1"
+google-cloud-googleapis = { version = "0.13", features = ["pubsub"] }
+google-cloud-pubsub = "0.25"
+tokio = { version = "0.2", package = "madsim-tokio", features = [
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+    "fs",
+] }
+```
+
+use google_cloud_googleapis::pubsub::v1::PubsubMessage;
+use google_cloud_pubsub::client::{Client, ClientConfig};
+use google_cloud_pubsub::subscription::SubscriptionConfig;
+
+const TOPIC: &str = "test-topic";
+
+const SUBSCRIPTION_COUNT: usize = 50;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    let args: Vec<String> = std::env::args().collect();
+    let command = args[1].as_str();
+
+    let use_emulator = std::env::var("PUBSUB_EMULATOR_HOST").is_ok();
+    let use_cloud = std::env::var("GOOGLE_APPLICATION_CREDENTIALS_JSON").is_ok();
+    if !use_emulator && !use_cloud {
+        panic!("either PUBSUB_EMULATOR_HOST or GOOGLE_APPLICATION_CREDENTIALS_JSON must be set");
+    }
+
+    let config = ClientConfig::default().with_auth().await?;
+    let client = Client::new(config).await?;
+
+    let topic = client.topic(TOPIC);
+
+    if command == "create" {
+        // delete and create "test-topic"
+        for subscription in topic.subscriptions(None).await? {
+            subscription.delete(None).await?;
+        }
+
+        let _ = topic.delete(None).await;
+        topic.create(Some(Default::default()), None).await?;
+        for i in 0..SUBSCRIPTION_COUNT {
+            let _ = client
+                .create_subscription(
+                    format!("test-subscription-{}", i).as_str(),
+                    TOPIC,
+                    SubscriptionConfig {
+                        retain_acked_messages: false,
+                        ..Default::default()
+                    },
+                    None,
+                )
+                .await?;
+        }
+    } else if command == "publish" {
+        let publisher = topic.new_publisher(Default::default());
+        for i in 0..10 {
+            let data = format!("{{\"v1\":{i},\"v2\":\"name{i}\"}}");
+            let a = publisher
+                .publish(PubsubMessage {
+                    data: data.to_string().into_bytes(),
+                    ..Default::default()
+                })
+                .await;
+            a.get().await?;
+            println!("published {}", data);
+        }
+    } else {
+        panic!("unknown command {command}");
+    }
+
+    Ok(())
+}
diff --git a/e2e_test/source_inline/pubsub/pubsub.slt b/e2e_test/source_inline/pubsub/pubsub.slt
new file mode 100644
index 0000000000000..35c94d07b9621
--- /dev/null
+++ b/e2e_test/source_inline/pubsub/pubsub.slt
@@ -0,0 +1,155 @@
+control substitution on
+
+system ok
+e2e_test/source_inline/pubsub/prepare-data.rs create
+
+statement error missing field `pubsub.subscription`
+CREATE SOURCE s (v1 int, v2 varchar) WITH (
+    connector = 'google_pubsub'
+) FORMAT PLAIN ENCODE JSON;
+
+statement error credentials must be set if not using the pubsub emulator
+CREATE SOURCE s (v1 int, v2 varchar) WITH (
+    connector = 'google_pubsub',
+    pubsub.subscription = 'test-subscription-1'
+) FORMAT PLAIN ENCODE JSON;
+
+statement error failed to lookup address information
+CREATE TABLE s (v1 int, v2 varchar) WITH (
+    connector = 'google_pubsub',
+    pubsub.subscription = 'test-subscription-1',
+    pubsub.emulator_host = 'invalid_host:5981'
+) FORMAT PLAIN ENCODE JSON;
+
+statement error subscription test-subscription-not-exist does not exist
+CREATE TABLE s (v1 int, v2 varchar) WITH (
+    ${RISEDEV_PUBSUB_WITH_OPTIONS_COMMON},
+    pubsub.subscription = 'test-subscription-not-exist',
+) FORMAT PLAIN ENCODE JSON;
+
+# fail if both start_offset and start_snapshot are provided
+statement error specify at most one of start_offset or start_snapshot
+CREATE TABLE s (v1 int, v2 varchar) WITH (
+    ${RISEDEV_PUBSUB_WITH_OPTIONS_COMMON},
+    pubsub.subscription = 'test-subscription-3',
+    pubsub.start_offset.nanos = '121212',
+    pubsub.start_snapshot = 'snapshot-that-doesnt-exist'
+) FORMAT PLAIN ENCODE JSON;
+
+statement ok
+CREATE TABLE s1 (v1 int, v2 varchar) WITH (
+    ${RISEDEV_PUBSUB_WITH_OPTIONS_COMMON},
+    pubsub.subscription = 'test-subscription-1',
+    pubsub.parallelism = 5
+) FORMAT PLAIN ENCODE JSON;
+
+
+# We want to test Pub/Sub ack messages on checkpoint, so disable checkpointing here.
+# Note that DDL & flush will trigger checkpoint.
+statement ok
+ALTER SYSTEM SET checkpoint_frequency TO 114514;
+
+# We publish data after the tables are created, because DDL will trigger checkpoint.
+system ok
+e2e_test/source_inline/pubsub/prepare-data.rs publish
+
+# default ack timeout is 10s. Let it redeliver once.
+sleep 15s
+
+# visibility mode is checkpoint
+query IT rowsort
+select count(*) from s1;
+----
+0
+
+statement ok
+set visibility_mode = 'all';
+
+query IT rowsort
+select v1, v2, count(*) FROM s1 group by v1, v2;
+----
+0 name0 2
+1 name1 2
+2 name2 2
+3 name3 2
+4 name4 2
+5 name5 2
+6 name6 2
+7 name7 2
+8 name8 2
+9 name9 2
+
+
+statement ok
+RECOVER;
+
+sleep 1s
+
+statement ok
+flush;
+
+# After recovery, uncheckpointed data in RisingWave is lost.
+query IT rowsort
+select count(*) from s1;
+----
+0
+
+# Wait for another redelivery
+sleep 10s
+
+query IT rowsort
+select v1, v2, count(*) FROM s1 group by v1, v2;
+----
+0 name0 1
+1 name1 1
+2 name2 1
+3 name3 1
+4 name4 1
+5 name5 1
+6 name6 1
+7 name7 1
+8 name8 1
+9 name9 1
+
+# flush will force a checkpoint (and ack Pub/Sub messages on checkpoint)
+statement ok
+flush;
+
+query IT rowsort
+select v1, v2, count(*) FROM s1 group by v1, v2;
+----
+0 name0 1
+1 name1 1
+2 name2 1
+3 name3 1
+4 name4 1
+5 name5 1
+6 name6 1
+7 name7 1
+8 name8 1
+9 name9 1
+
+sleep 15s
+
+# no redelivery any more
+query IT rowsort
+select v1, v2, count(*) FROM s1 group by v1, v2;
+----
+0 name0 1
+1 name1 1
+2 name2 1
+3 name3 1
+4 name4 1
+5 name5 1
+6 name6 1
+7 name7 1
+8 name8 1
+9 name9 1
+
+
+statement ok
+DROP TABLE s1;
+
+# Restore to the value in src/config/ci-recovery.toml
+statement ok
+ALTER SYSTEM SET checkpoint_frequency TO 5;
diff --git a/e2e_test/sql_migration/check.slt b/e2e_test/sql_migration/check.slt
index b4c97bba50bf1..c79721f1145e8 100644
--- a/e2e_test/sql_migration/check.slt
+++ b/e2e_test/sql_migration/check.slt
@@ -64,6 +64,14 @@ create materialized view mv3 as select * from mv2;
 statement ok
 REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA schema1 FROM user1;
 
+query T
+show secrets;
+----
+secret_1
+
+statement ok
+drop secret secret_1;
+
 statement error Permission denied
 drop source src;
 
diff --git a/e2e_test/sql_migration/prepare.slt b/e2e_test/sql_migration/prepare.slt
index f0669a4c6b297..ecb295cf6c534 100644
--- a/e2e_test/sql_migration/prepare.slt
+++ b/e2e_test/sql_migration/prepare.slt
@@ -66,3 +66,8 @@ statement ok
 create function int_42() returns int language javascript as $$
     return 42;
 $$;
+
+statement ok
+create secret secret_1 with (
+  backend = 'meta'
+) as 'demo_secret';
diff --git a/e2e_test/streaming/bug_fixes/issue_11915.slt b/e2e_test/streaming/bug_fixes/issue_11915.slt
new file mode 100644
index 0000000000000..2f4f111358afd
--- /dev/null
+++ b/e2e_test/streaming/bug_fixes/issue_11915.slt
@@ -0,0 +1,41 @@
+# https://github.com/risingwavelabs/risingwave/issues/11915
+
+statement ok
+create table t(x int);
+
+statement ok
+create materialized view mv as select x, generate_series(1, 2, x) from t;
+
+# x = 0 causes generate_series(1, 2, x) to return an error.
+statement ok
+insert into t values (0), (1);
+
+statement ok
+flush;
+
+# Output 0 row when the set-returning function returns error.
+query II rowsort
+select * from mv;
+----
+1 1
+1 2
+
+# Delete the error row.
+statement ok
+delete from t where x = 0;
+
+statement ok
+flush;
+
+# The result should be the same as before.
+query II rowsort
+select * from mv;
+----
+1 1
+1 2
+
+statement ok
+drop materialized view mv;
+
+statement ok
+drop table t;
diff --git a/e2e_test/streaming/bug_fixes/issue_12474.slt b/e2e_test/streaming/bug_fixes/issue_12474.slt
new file mode 100644
index 0000000000000..245ca499a255d
--- /dev/null
+++ b/e2e_test/streaming/bug_fixes/issue_12474.slt
@@ -0,0 +1,26 @@
+# https://github.com/risingwavelabs/risingwave/issues/12474
+
+statement ok
+create table t(x int[]);
+
+statement ok
+create materialized view mv as select 1 / x[1] as bomb, unnest(x) as unnest from t;
+
+statement ok
+insert into t values (array[0, 1]), (array[1]);
+
+statement ok
+flush;
+
+query II rowsort
+select * from mv;
+----
+1    1
+NULL 0
+NULL 1
+
+statement ok
+drop materialized view mv;
+
+statement ok
+drop table t;
diff --git a/e2e_test/streaming/eowc/eowc_over_window.slt b/e2e_test/streaming/eowc/eowc_over_window.slt
index fe2570e93b6df..e762d54743e7f 100644
--- a/e2e_test/streaming/eowc/eowc_over_window.slt
+++ b/e2e_test/streaming/eowc/eowc_over_window.slt
@@ -42,6 +42,15 @@ select
 from t
 emit on window close;
 
+statement ok
+create materialized view mv4 as
+select
+    *,
+    first_value(tm) over (partition by bar order by tm session with gap '10 minutes') as window_start,
+    last_value(tm) over (partition by bar order by tm session with gap '10 minutes') as window_end
+from t
+emit on window close;
+
 statement ok
 insert into t values
   ('2023-05-06 16:51:00', 1, 100)
@@ -71,6 +80,12 @@ select * from mv3 order by tm;
 2023-05-06 17:30:00  3  200  1
 2023-05-06 17:35:00  5  100  3
 
+query TiiTT
+select * from mv4 order by tm;
+----
+2023-05-06 16:51:00	1	100	2023-05-06 16:51:00	2023-05-06 16:56:00
+2023-05-06 16:56:00	8	100	2023-05-06 16:51:00	2023-05-06 16:56:00
+
 statement ok
 insert into t values
   ('2023-05-06 18:10:00', 7, 100)
@@ -100,6 +115,14 @@ select * from mv3 order by tm;
 2023-05-06 17:59:00  4  100  4
 2023-05-06 18:01:00  6  200  2
 
+query TiiTT
+select * from mv4 order by tm;
+----
+2023-05-06 16:51:00	1	100	2023-05-06 16:51:00	2023-05-06 16:56:00
+2023-05-06 16:56:00	8	100	2023-05-06 16:51:00	2023-05-06 16:56:00
+2023-05-06 17:30:00	3	200	2023-05-06 17:30:00	2023-05-06 17:30:00
+2023-05-06 17:35:00	5	100	2023-05-06 17:35:00	2023-05-06 17:35:00
+
 statement ok
 drop materialized view mv1;
 
@@ -109,5 +132,8 @@ drop materialized view mv2;
 statement ok
 drop materialized view mv3;
 
+statement ok
+drop materialized view mv4;
+
 statement ok
 drop table t;
diff --git a/e2e_test/streaming/rate_limit/alter_rate_limit.slt.ignore b/e2e_test/streaming/rate_limit/alter_rate_limit.slt.ignore
new file mode 100644
index 0000000000000..7afe279a80f1e
--- /dev/null
+++ b/e2e_test/streaming/rate_limit/alter_rate_limit.slt.ignore
@@ -0,0 +1,35 @@
+-- This test is ignored until alter mv rate limit is fully supported.
+
+statement ok
+CREATE TABLE t (v1 int);
+
+statement ok
+INSERT INTO t VALUES (1);
+
+statement ok
+flush;
+
+statement ok
+SET BACKGROUND_DDL=true;
+
+statement ok
+CREATE MATERIALIZED VIEW streaming_rate_limit_0 with ( streaming_rate_limit = 0 ) AS SELECT * FROM t;
+
+skipif in-memory
+sleep 1s
+
+query I
+select progress from rw_ddl_progress;
+----
+0.00%
+
+statement ok
+ALTER MATERIALIZED VIEW streaming_rate_limit_0 SET STREAMING_RATE_LIMIT = 1;
+
+statement ok
+wait;
+
+query I
+select * from streaming_rate_limit_0;
+----
+1
\ No newline at end of file
diff --git a/e2e_test/subscription/create_table_and_subscription.slt b/e2e_test/subscription/create_table_and_subscription.slt
index 94039f98b11cc..fd43567bc52de 100644
--- a/e2e_test/subscription/create_table_and_subscription.slt
+++ b/e2e_test/subscription/create_table_and_subscription.slt
@@ -4,5 +4,8 @@ create table t1 (v1 int, v2 int);
 statement ok
 insert into t1 values (1,2);
 
+statement ok
+flush;
+
 statement ok
 create subscription sub from t1 with(retention = '1D');
\ No newline at end of file
diff --git a/e2e_test/subscription/main.py b/e2e_test/subscription/main.py
index f8e78813801f2..3ffaefd02cee6 100644
--- a/e2e_test/subscription/main.py
+++ b/e2e_test/subscription/main.py
@@ -33,15 +33,15 @@ def execute_insert(sql,conn):
     conn.commit()
     cur.close()
 
-def check_rows_data(expect_vec,rows,status):
-    row = rows[0]
+def check_rows_data(expect_vec,row,status):
+    value_len  = len(row)
     for index, value in enumerate(row):
-        if index == 0:
+        if index == value_len  - 1:
             continue
-        if index == 1:
+        if index == value_len  - 2:
             assert value == status,f"expect {value} but got {status}"
             continue
-        assert value == expect_vec[index-2],f"expect {expect_vec[index-2]} but got {value}"
+        assert value == expect_vec[index],f"expect {expect_vec[index]} but got {value}"
 
 def test_cursor_snapshot():
     print(f"test_cursor_snapshot")
@@ -55,7 +55,7 @@ def test_cursor_snapshot():
 
     execute_insert("declare cur subscription cursor for sub",conn)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([1,2],row,1)
+    check_rows_data([1,2],row[0],1)
     row = execute_query("fetch next from cur",conn)
     assert row == []
     execute_insert("close cur",conn)
@@ -74,7 +74,7 @@ def test_cursor_snapshot_log_store():
 
     execute_insert("declare cur subscription cursor for sub",conn)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([1,2],row,1)
+    check_rows_data([1,2],row[0],1)
     row = execute_query("fetch next from cur",conn)
     assert row == []
     execute_insert("insert into t1 values(4,4)",conn)
@@ -82,9 +82,9 @@ def test_cursor_snapshot_log_store():
     execute_insert("insert into t1 values(5,5)",conn)
     execute_insert("flush",conn)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([4,4],row,1)
+    check_rows_data([4,4],row[0],1)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([5,5],row,1)
+    check_rows_data([5,5],row[0],1)
     row = execute_query("fetch next from cur",conn)
     assert row == []
     execute_insert("close cur",conn)
@@ -108,11 +108,11 @@ def test_cursor_since_begin():
     execute_insert("insert into t1 values(6,6)",conn)
     execute_insert("flush",conn)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([4,4],row,1)
+    check_rows_data([4,4],row[0],1)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([5,5],row,1)
+    check_rows_data([5,5],row[0],1)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([6,6],row,1)
+    check_rows_data([6,6],row[0],1)
     row = execute_query("fetch next from cur",conn)
     assert row == []
     execute_insert("close cur",conn)
@@ -137,7 +137,7 @@ def test_cursor_since_now():
     execute_insert("insert into t1 values(6,6)",conn)
     execute_insert("flush",conn)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([6,6],row,1)
+    check_rows_data([6,6],row[0],1)
     row = execute_query("fetch next from cur",conn)
     assert row == []
     execute_insert("close cur",conn)
@@ -161,26 +161,29 @@ def test_cursor_since_rw_timestamp():
     execute_insert("insert into t1 values(6,6)",conn)
     execute_insert("flush",conn)
     row = execute_query("fetch next from cur",conn)
-    rw_timestamp_1 = row[0][0]
-    check_rows_data([4,4],row,1)
+    valuelen = len(row[0])
+    rw_timestamp_1 = row[0][valuelen - 1]
+    check_rows_data([4,4],row[0],1)
     row = execute_query("fetch next from cur",conn)
-    rw_timestamp_2 = row[0][0] - 1
-    check_rows_data([5,5],row,1)
+    valuelen = len(row[0])
+    rw_timestamp_2 = row[0][valuelen - 1] - 1
+    check_rows_data([5,5],row[0],1)
     row = execute_query("fetch next from cur",conn)
-    rw_timestamp_3 = row[0][0] + 1
-    check_rows_data([6,6],row,1)
+    valuelen = len(row[0])
+    rw_timestamp_3 = row[0][valuelen - 1] + 1
+    check_rows_data([6,6],row[0],1)
     row = execute_query("fetch next from cur",conn)
     assert row == []
     execute_insert("close cur",conn)
 
     execute_insert(f"declare cur subscription cursor for sub since {rw_timestamp_1}",conn)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([4,4],row,1)
+    check_rows_data([4,4],row[0],1)
     execute_insert("close cur",conn)
 
     execute_insert(f"declare cur subscription cursor for sub since {rw_timestamp_2}",conn)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([5,5],row,1)
+    check_rows_data([5,5],row[0],1)
     execute_insert("close cur",conn)
 
     execute_insert(f"declare cur subscription cursor for sub since {rw_timestamp_3}",conn)
@@ -202,7 +205,7 @@ def test_cursor_op():
 
     execute_insert("declare cur subscription cursor for sub",conn)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([1,2],row,1)
+    check_rows_data([1,2],row[0],1)
     row = execute_query("fetch next from cur",conn)
     assert row == []
 
@@ -211,24 +214,96 @@ def test_cursor_op():
     execute_insert("update t1 set v2 = 10 where v1 = 4",conn)
     execute_insert("flush",conn)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([4,4],row,1)
+    check_rows_data([4,4],row[0],1)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([4,4],row,4)
+    check_rows_data([4,4],row[0],4)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([4,10],row,3)
+    check_rows_data([4,10],row[0],3)
     row = execute_query("fetch next from cur",conn)
     assert row == []
 
     execute_insert("delete from t1 where v1 = 4",conn)
     execute_insert("flush",conn)
     row = execute_query("fetch next from cur",conn)
-    check_rows_data([4,10],row,2)
+    check_rows_data([4,10],row[0],2)
     row = execute_query("fetch next from cur",conn)
     assert row == []
 
     execute_insert("close cur",conn)
     drop_table_subscription()
 
+def test_cursor_with_table_alter():
+    print(f"test_cursor_with_table_alter")
+    create_table_subscription()
+    conn = psycopg2.connect(
+        host="localhost",
+        port="4566",
+        user="root",
+        database="dev"
+    )
+
+    execute_insert("declare cur subscription cursor for sub",conn)
+    execute_insert("alter table t1 add v3 int",conn)
+    execute_insert("insert into t1 values(4,4,4)",conn)
+    execute_insert("flush",conn)
+    row = execute_query("fetch next from cur",conn)
+    check_rows_data([1,2],row[0],1)
+    row = execute_query("fetch next from cur",conn)
+    check_rows_data([4,4,4],row[0],1)
+    execute_insert("insert into t1 values(5,5,5)",conn)
+    execute_insert("flush",conn)
+    row = execute_query("fetch next from cur",conn)
+    check_rows_data([5,5,5],row[0],1)
+    execute_insert("alter table t1 drop column v2",conn)
+    execute_insert("insert into t1 values(6,6)",conn)
+    execute_insert("flush",conn)
+    row = execute_query("fetch next from cur",conn)
+    check_rows_data([6,6],row[0],1)
+    drop_table_subscription()
+
+def test_cursor_fetch_n():
+    print(f"test_cursor_with_table_alter")
+    create_table_subscription()
+    conn = psycopg2.connect(
+        host="localhost",
+        port="4566",
+        user="root",
+        database="dev"
+    )
+
+    execute_insert("declare cur subscription cursor for sub",conn)
+    execute_insert("insert into t1 values(4,4)",conn)
+    execute_insert("flush",conn)
+    execute_insert("insert into t1 values(5,5)",conn)
+    execute_insert("flush",conn)
+    execute_insert("insert into t1 values(6,6)",conn)
+    execute_insert("flush",conn)
+    execute_insert("insert into t1 values(7,7)",conn)
+    execute_insert("flush",conn)
+    execute_insert("insert into t1 values(8,8)",conn)
+    execute_insert("flush",conn)
+    execute_insert("insert into t1 values(9,9)",conn)
+    execute_insert("flush",conn)
+    execute_insert("insert into t1 values(10,10)",conn)
+    execute_insert("flush",conn)
+    execute_insert("update t1 set v2 = 100 where v1 = 10",conn)
+    execute_insert("flush",conn)
+    row = execute_query("fetch 6 from cur",conn)
+    assert len(row) == 6
+    check_rows_data([1,2],row[0],1)
+    check_rows_data([4,4],row[1],1)
+    check_rows_data([5,5],row[2],1)
+    check_rows_data([6,6],row[3],1)
+    check_rows_data([7,7],row[4],1)
+    check_rows_data([8,8],row[5],1)
+    row = execute_query("fetch 6 from cur",conn)
+    assert len(row) == 4
+    check_rows_data([9,9],row[0],1)
+    check_rows_data([10,10],row[1],1)
+    check_rows_data([10,10],row[2],4)
+    check_rows_data([10,100],row[3],3)
+    drop_table_subscription()
+
 if __name__ == "__main__":
     test_cursor_snapshot()
     test_cursor_op()
@@ -236,3 +311,5 @@ def test_cursor_op():
     test_cursor_since_rw_timestamp()
     test_cursor_since_now()
     test_cursor_since_begin()
+    test_cursor_with_table_alter()
+    test_cursor_fetch_n()
diff --git a/e2e_test/udf/external_udf.slt b/e2e_test/udf/external_udf.slt
index 096a605709d67..7a38506f81563 100644
--- a/e2e_test/udf/external_udf.slt
+++ b/e2e_test/udf/external_udf.slt
@@ -1,7 +1,7 @@
 # Before running this test:
 #   python3 e2e_test/udf/test.py
 # or:
-#   cd java/udf-example && mvn package && java -jar target/risingwave-udf-example.jar
+#   cd e2e_test/udf/java && mvn package && java -jar target/risingwave-udf-example.jar
 
 # Create a function.
 statement ok
diff --git a/java/udf-example/README.md b/e2e_test/udf/java/README.md
similarity index 100%
rename from java/udf-example/README.md
rename to e2e_test/udf/java/README.md
diff --git a/java/udf-example/pom.xml b/e2e_test/udf/java/pom.xml
similarity index 86%
rename from java/udf-example/pom.xml
rename to e2e_test/udf/java/pom.xml
index 8bf51cd108128..7ecd7c54dca17 100644
--- a/java/udf-example/pom.xml
+++ b/e2e_test/udf/java/pom.xml
@@ -5,17 +5,9 @@
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
-  <!-- remove this section if you want to build the project independently -->
-  <parent>
-    <groupId>com.risingwave</groupId>
-    <artifactId>risingwave-java-root</artifactId>
-    <version>0.1.0-SNAPSHOT</version>
-    <relativePath>../pom.xml</relativePath>
-  </parent>
-
   <groupId>com.risingwave</groupId>
   <artifactId>risingwave-udf-example</artifactId>
-  <version>0.1.1-SNAPSHOT</version>
+  <version>0.1.0-SNAPSHOT</version>
 
   <name>udf-example</name>
   <url>https://docs.risingwave.com/docs/current/udf-java</url>
@@ -31,7 +23,7 @@
     <dependency>
       <groupId>com.risingwave</groupId>
       <artifactId>risingwave-udf</artifactId>
-      <version>0.1.3-SNAPSHOT</version>
+      <version>0.2.0</version>
     </dependency>
     <dependency>
       <groupId>com.google.code.gson</groupId>
diff --git a/java/udf-example/src/main/java/com/example/UdfExample.java b/e2e_test/udf/java/src/main/java/com/example/UdfExample.java
similarity index 99%
rename from java/udf-example/src/main/java/com/example/UdfExample.java
rename to e2e_test/udf/java/src/main/java/com/example/UdfExample.java
index 883dc5035514c..1702e244bf1ff 100644
--- a/java/udf-example/src/main/java/com/example/UdfExample.java
+++ b/e2e_test/udf/java/src/main/java/com/example/UdfExample.java
@@ -33,7 +33,7 @@
 
 public class UdfExample {
     public static void main(String[] args) throws IOException {
-        try (var server = new UdfServer("0.0.0.0", 8815)) {
+        try (var server = new UdfServer("localhost", 8815)) {
             server.addFunction("int_42", new Int42());
             server.addFunction("float_to_decimal", new FloatToDecimal());
             server.addFunction("sleep", new Sleep());
diff --git a/e2e_test/udf/js_udf.slt b/e2e_test/udf/js_udf.slt
index 260fd991f648f..a6cabbd487cde 100644
--- a/e2e_test/udf/js_udf.slt
+++ b/e2e_test/udf/js_udf.slt
@@ -152,3 +152,86 @@ wave   4
 
 statement ok
 drop function split;
+
+
+# aggregate function
+statement ok
+create aggregate weighted_avg(value int, weight int) returns float language javascript as $$
+    export function create_state() {
+        return {sum: 0, weight: 0};
+    }
+    export function accumulate(state, value, weight) {
+        if (value == null || weight == null) {
+            return state;
+        }
+        state.sum += value * weight;
+        state.weight += weight;
+        return state;
+    }
+    export function retract(state, value, weight) {
+        if (value == null || weight == null) {
+            return state;
+        }
+        state.sum -= value * weight;
+        state.weight -= weight;
+        return state;
+    }
+    export function finish(state) {
+        if (state.weight == 0) {
+            return null;
+        }
+        return state.sum / state.weight;
+    }
+$$;
+
+# batch
+query F
+select weighted_avg(value, weight) from (values (1, 1), (null, 2), (3, 3)) as t(value, weight);
+----
+2.5
+
+# streaming
+statement ok
+create table t(value int, weight int);
+
+statement ok
+create materialized view mv as select weighted_avg(value, weight) from t;
+
+query F
+select * from mv;
+----
+NULL
+
+statement ok
+insert into t values (1, 1), (null, 2), (3, 3);
+
+statement ok
+flush;
+
+query F
+select * from mv;
+----
+2.5
+
+statement ok
+delete from t where value = 3;
+
+statement ok
+flush;
+
+query F
+select * from mv;
+----
+1
+
+statement ok
+drop materialized view mv;
+
+statement ok
+drop table t;
+
+statement error "weighted_avg" is an aggregate function
+drop function weighted_avg;
+
+statement ok
+drop aggregate weighted_avg;
diff --git a/e2e_test/udf/python_udf.slt b/e2e_test/udf/python_udf.slt
index b8253c96de4dc..fedb9644160a5 100644
--- a/e2e_test/udf/python_udf.slt
+++ b/e2e_test/udf/python_udf.slt
@@ -170,6 +170,87 @@ wave   4
 statement ok
 drop function split;
 
+
+# aggregate function
+statement ok
+create aggregate weighted_avg(value int, weight int) returns float language python as $$
+def create_state():
+    return (0, 0)
+def accumulate(state, value, weight):
+    if value is None or weight is None:
+        return state
+    (s, w) = state
+    s += value * weight
+    w += weight
+    return (s, w)
+def retract(state, value, weight):
+    if value is None or weight is None:
+        return state
+    (s, w) = state
+    s -= value * weight
+    w -= weight
+    return (s, w)
+def finish(state):
+    (sum, weight) = state
+    if weight == 0:
+        return None
+    else:
+        return sum / weight
+$$;
+
+# batch
+query F
+select weighted_avg(value, weight) from (values (1, 1), (null, 2), (3, 3)) as t(value, weight);
+----
+2.5
+
+# streaming
+statement ok
+create table t(value int, weight int);
+
+statement ok
+create materialized view mv as select weighted_avg(value, weight) from t;
+
+query F
+select * from mv;
+----
+NULL
+
+statement ok
+insert into t values (1, 1), (null, 2), (3, 3);
+
+statement ok
+flush;
+
+query F
+select * from mv;
+----
+2.5
+
+statement ok
+delete from t where value = 3;
+
+statement ok
+flush;
+
+query F
+select * from mv;
+----
+1
+
+statement ok
+drop materialized view mv;
+
+statement ok
+drop table t;
+
+statement error "weighted_avg" is an aggregate function
+drop function weighted_avg;
+
+statement ok
+drop aggregate weighted_avg;
+
+
 statement ok
 create function mismatched_arguments() returns int language python as $$
 def mismatched_arguments(x):
diff --git a/e2e_test/udf/requirements.txt b/e2e_test/udf/requirements.txt
index 8642e2b1ec254..36688db1ed1ee 100644
--- a/e2e_test/udf/requirements.txt
+++ b/e2e_test/udf/requirements.txt
@@ -1,2 +1,3 @@
 flask
-waitress
\ No newline at end of file
+waitress
+arrow_udf==0.2.1
\ No newline at end of file
diff --git a/e2e_test/udf/sql_udf.slt b/e2e_test/udf/sql_udf.slt
index 866f27abd52ce..5f32bb6f7e024 100644
--- a/e2e_test/udf/sql_udf.slt
+++ b/e2e_test/udf/sql_udf.slt
@@ -360,7 +360,7 @@ In SQL UDF definition: `select a + b + c + not_be_displayed(c)`
                                        ^
 
 
-statement error Expected end of statement, found: 💩
+statement error expected end of statement, found: 💩
 create function call_regexp_replace() returns varchar language sql as 'select regexp_replace('💩💩💩💩💩foo🤔️bar亲爱的😭baz这不是爱情❤️‍🔥', 'baz(...)', '这是🥵', 'ic')';
 
 # Recursive definition can NOT be accepted at present due to semantic check
@@ -401,7 +401,7 @@ statement error return type mismatch detected
 create function type_mismatch(INT) returns varchar language sql as 'select $1 + 114514 + $1';
 
 # Invalid function body syntax
-statement error Expected an expression:, found: EOF at the end
+statement error expected an expression:, found: EOF at the end
 create function add_error(INT, INT) returns int language sql as $$select $1 + $2 +$$;
 
 ######################################################################
diff --git a/e2e_test/udf/test.py b/e2e_test/udf/test.py
index 6195476a80004..4443a81a6e74d 100644
--- a/e2e_test/udf/test.py
+++ b/e2e_test/udf/test.py
@@ -19,9 +19,7 @@
 from typing import Iterator, List, Optional, Tuple, Any
 from decimal import Decimal
 
-sys.path.append("src/expr/udf/python")  # noqa
-
-from risingwave.udf import udf, udtf, UdfServer
+from arrow_udf import udf, udtf, UdfServer
 
 
 @udf(input_types=[], result_type="INT")
@@ -47,13 +45,21 @@ def gcd3(x: int, y: int, z: int) -> int:
     return gcd(gcd(x, y), z)
 
 
-@udf(input_types=["BYTEA"], result_type="STRUCT<VARCHAR, VARCHAR, SMALLINT, SMALLINT>")
+@udf(
+    input_types=["BYTEA"],
+    result_type="STRUCT<src_addr: VARCHAR, dst_addr: VARCHAR, src_port: SMALLINT, dst_port: SMALLINT>",
+)
 def extract_tcp_info(tcp_packet: bytes):
     src_addr, dst_addr = struct.unpack("!4s4s", tcp_packet[12:20])
     src_port, dst_port = struct.unpack("!HH", tcp_packet[20:24])
     src_addr = socket.inet_ntoa(src_addr)
     dst_addr = socket.inet_ntoa(dst_addr)
-    return src_addr, dst_addr, src_port, dst_port
+    return {
+        "src_addr": src_addr,
+        "dst_addr": dst_addr,
+        "src_port": src_port,
+        "dst_port": dst_port,
+    }
 
 
 @udtf(input_types="INT", result_types="INT")
@@ -84,7 +90,7 @@ def hex_to_dec(hex: Optional[str]) -> Optional[Decimal]:
     return dec
 
 
-@udf(input_types=["FLOAT8"], result_type="DECIMAL")
+@udf(input_types=["FLOAT64"], result_type="DECIMAL")
 def float_to_decimal(f: float) -> Decimal:
     return Decimal(f)
 
@@ -120,21 +126,49 @@ def jsonb_array_identity(list: List[Any]) -> List[Any]:
     return list
 
 
-@udf(input_types="STRUCT<JSONB[], INT>", result_type="STRUCT<JSONB[], INT>")
+@udf(
+    input_types="STRUCT<v: JSONB[], len: INT>",
+    result_type="STRUCT<v: JSONB[], len: INT>",
+)
 def jsonb_array_struct_identity(v: Tuple[List[Any], int]) -> Tuple[List[Any], int]:
     return v
 
 
-ALL_TYPES = "BOOLEAN,SMALLINT,INT,BIGINT,FLOAT4,FLOAT8,DECIMAL,DATE,TIME,TIMESTAMP,INTERVAL,VARCHAR,BYTEA,JSONB".split(
-    ","
-) + [
-    "STRUCT<INT,INT>"
-]
-
-
 @udf(
-    input_types=ALL_TYPES,
-    result_type=f"struct<{','.join(ALL_TYPES)}>",
+    input_types=[
+        "boolean",
+        "int16",
+        "int32",
+        "int64",
+        "float32",
+        "float64",
+        "decimal",
+        "date32",
+        "time64",
+        "timestamp",
+        "interval",
+        "string",
+        "binary",
+        "json",
+        "struct<f1:int, f2:int>",
+    ],
+    result_type="""struct<
+        boolean: boolean,
+        int16: int16,
+        int32: int32,
+        int64: int64,
+        float32: float32,
+        float64: float64,
+        decimal: decimal,
+        date32: date32,
+        time64: time64,
+        timestamp: timestamp,
+        interval: interval,
+        string: string,
+        binary: binary,
+        json: json,
+        struct: struct<f1:int, f2:int>,
+    >""",
 )
 def return_all(
     bool,
@@ -153,28 +187,60 @@ def return_all(
     jsonb,
     struct,
 ):
-    return (
-        bool,
-        i16,
-        i32,
-        i64,
-        f32,
-        f64,
-        decimal,
-        date,
-        time,
-        timestamp,
-        interval,
-        varchar,
-        bytea,
-        jsonb,
-        struct,
-    )
+    return {
+        "boolean": bool,
+        "int16": i16,
+        "int32": i32,
+        "int64": i64,
+        "float32": f32,
+        "float64": f64,
+        "decimal": decimal,
+        "date32": date,
+        "time64": time,
+        "timestamp": timestamp,
+        "interval": interval,
+        "string": varchar,
+        "binary": bytea,
+        "json": jsonb,
+        "struct": struct,
+    }
 
 
 @udf(
-    input_types=[t + "[]" for t in ALL_TYPES],
-    result_type=f"struct<{','.join(t + '[]' for t in ALL_TYPES)}>",
+    input_types=[
+        "boolean[]",
+        "int16[]",
+        "int32[]",
+        "int64[]",
+        "float32[]",
+        "float64[]",
+        "decimal[]",
+        "date32[]",
+        "time64[]",
+        "timestamp[]",
+        "interval[]",
+        "string[]",
+        "binary[]",
+        "json[]",
+        "struct<f1:int, f2:int>[]",
+    ],
+    result_type="""struct<
+        boolean: boolean[],
+        int16: int16[],
+        int32: int32[],
+        int64: int64[],
+        float32: float32[],
+        float64: float64[],
+        decimal: decimal[],
+        date32: date32[],
+        time64: time64[],
+        timestamp: timestamp[],
+        interval: interval[],
+        string: string[],
+        binary: binary[],
+        json: json[],
+        struct: struct<f1:int, f2:int>[],
+    >""",
 )
 def return_all_arrays(
     bool,
@@ -193,23 +259,23 @@ def return_all_arrays(
     jsonb,
     struct,
 ):
-    return (
-        bool,
-        i16,
-        i32,
-        i64,
-        f32,
-        f64,
-        decimal,
-        date,
-        time,
-        timestamp,
-        interval,
-        varchar,
-        bytea,
-        jsonb,
-        struct,
-    )
+    return {
+        "boolean": bool,
+        "int16": i16,
+        "int32": i32,
+        "int64": i64,
+        "float32": f32,
+        "float64": f64,
+        "decimal": decimal,
+        "date32": date,
+        "time64": time,
+        "timestamp": timestamp,
+        "interval": interval,
+        "string": varchar,
+        "binary": bytea,
+        "json": jsonb,
+        "struct": struct,
+    }
 
 
 if __name__ == "__main__":
diff --git a/e2e_test/udf/wasm/Cargo.toml b/e2e_test/udf/wasm/Cargo.toml
index 250bd8132ca53..54c7da45b1af8 100644
--- a/e2e_test/udf/wasm/Cargo.toml
+++ b/e2e_test/udf/wasm/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2021"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow-udf = "0.2"
+arrow-udf = "0.3"
 genawaiter = "0.99"
 rust_decimal = "1"
 serde_json = "1"
diff --git a/grafana/common.py b/grafana/common.py
index 9a47bd0c964a9..88a31c1725b62 100644
--- a/grafana/common.py
+++ b/grafana/common.py
@@ -467,10 +467,9 @@ def timeseries_id(self, title, description, targets):
             **self.common_options,
         )
 
-    def table_info(self, title, description, targets, columns):
+    def table_info(self, title, description, targets, columns, excludeByName=dict.fromkeys(["Time", "Value"], True)):
         gridPos = self.layout.next_half_width_graph()
         column_indices = {column: index for index, column in enumerate(columns)}
-        excludeByName = dict.fromkeys(["Time", "Value"], True)
         transformations = [
             {"id": "organize", "options": {"indexByName": column_indices, "excludeByName": excludeByName}}
         ]
@@ -514,11 +513,15 @@ def table_metric(name, filter=None):
 
 def quantile(f, percentiles):
     quantile_map = {
-        "60": ["0.6", "60"],
+        "10": ["0.1", "10"],
+        "25": ["0.25", "25"],
         "50": ["0.5", "50"],
+        "60": ["0.6", "60"],
+        "75": ["0.75", "75"],
         "90": ["0.9", "90"],
         "99": ["0.99", "99"],
         "999": ["0.999", "999"],
+        "100": ["1.0", "100"],
         "max": ["1.0", "max"],
     }
     return list(
diff --git a/grafana/risingwave-dev-dashboard.dashboard.py b/grafana/risingwave-dev-dashboard.dashboard.py
index 34f58d2458764..e8a3291f98640 100644
--- a/grafana/risingwave-dev-dashboard.dashboard.py
+++ b/grafana/risingwave-dev-dashboard.dashboard.py
@@ -40,6 +40,13 @@ def section_actor_info(outer_panels):
                     [panels.table_target(f"group({metric('table_info')}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)")],
                     ["table_id", "table_name", "table_type", "materialized_view_id", "fragment_id", "compaction_group_id"],
                 ),
+                panels.table_info(
+                    "Actor Count (Group By Compute Node)",
+                    "Actor count per compute node",
+                    [panels.table_target(f"count({metric('actor_info')}) by (compute_node)")],
+                    ["table_id", "table_name", "table_type", "materialized_view_id", "fragment_id", "compaction_group_id"],
+                    dict.fromkeys(["Time"], True)
+                )
             ],
         )
     ]
@@ -82,7 +89,7 @@ def section_cluster_node(outer_panels):
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                         panels.target(
-                            f"sum(rate({metric('process_cpu_seconds_total')}[$__rate_interval])) by ({COMPONENT_LABEL}, {NODE_LABEL}) / avg({metric('process_cpu_core_num')}) by ({COMPONENT_LABEL}, {NODE_LABEL})",
+                            f"sum(rate({metric('process_cpu_seconds_total')}[$__rate_interval])) by ({COMPONENT_LABEL}, {NODE_LABEL}) / avg({metric('process_cpu_core_num')}) by ({COMPONENT_LABEL}, {NODE_LABEL}) > 0",
                             "cpu usage (avg per core) - {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -146,7 +153,7 @@ def section_recovery_node(outer_panels):
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by (le) (rate({metric('recovery_latency_sum')}[$__rate_interval])) / sum by (le) (rate({metric('recovery_latency_count')}[$__rate_interval]))",
+                            f"sum by (le) (rate({metric('recovery_latency_sum')}[$__rate_interval])) / sum by (le) (rate({metric('recovery_latency_count')}[$__rate_interval])) > 0",
                             "recovery latency avg",
                         ),
                     ],
@@ -230,7 +237,7 @@ def section_compaction(outer_panels):
                     "Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types",
                     [
                         panels.target(
-                            f"sum by(le, group, type)(irate({metric('storage_l0_compact_level_count_sum')}[$__rate_interval]))  / sum by(le, group, type)(irate({metric('storage_l0_compact_level_count_count')}[$__rate_interval]))",
+                            f"sum by(le, group, type)(irate({metric('storage_l0_compact_level_count_sum')}[$__rate_interval]))  / sum by(le, group, type)(irate({metric('storage_l0_compact_level_count_count')}[$__rate_interval])) > 0",
                             "avg cg{{group}}@{{type}}",
                         ),
                     ],
@@ -240,7 +247,7 @@ def section_compaction(outer_panels):
                     "Avg file count of the compact task, and categorize it according to different cg, levels and task types",
                     [
                         panels.target(
-                            f"sum by(le, group, type)(irate({metric('storage_compact_task_file_count_sum')}[$__rate_interval]))  / sum by(le, group, type)(irate({metric('storage_compact_task_file_count_count')}[$__rate_interval]))",
+                            f"sum by(le, group, type)(irate({metric('storage_compact_task_file_count_sum')}[$__rate_interval]))  / sum by(le, group, type)(irate({metric('storage_compact_task_file_count_count')}[$__rate_interval])) > 0",
                             "avg cg{{group}}@{{type}}",
                         ),
                     ],
@@ -315,11 +322,11 @@ def section_compaction(outer_panels):
                             "compute_apply_version_duration_p99",
                         ),
                         panels.target(
-                            f"sum by(le)(rate({metric('compactor_compact_task_duration_sum')}[$__rate_interval])) / sum by(le)(rate({metric('compactor_compact_task_duration_count')}[$__rate_interval]))",
+                            f"sum by(le)(rate({metric('compactor_compact_task_duration_sum')}[$__rate_interval])) / sum by(le)(rate({metric('compactor_compact_task_duration_count')}[$__rate_interval])) > 0",
                             "compact-task avg",
                         ),
                         panels.target(
-                            f"sum by(le)(rate({metric('state_store_compact_sst_duration_sum')}[$__rate_interval])) / sum by(le)(rate({metric('state_store_compact_sst_duration_count')}[$__rate_interval]))",
+                            f"sum by(le)(rate({metric('state_store_compact_sst_duration_sum')}[$__rate_interval])) / sum by(le)(rate({metric('state_store_compact_sst_duration_count')}[$__rate_interval])) > 0",
                             "compact-key-range avg",
                         ),
                     ],
@@ -371,7 +378,7 @@ def section_compaction(outer_panels):
                     "may read this piece of data and write it to a new SSTable, that's another write.",
                     [
                         panels.target(
-                            f"sum({metric('storage_level_compact_write')}) / sum({metric('compactor_write_build_l0_bytes')})",
+                            f"sum({metric('storage_level_compact_write')}) / sum({metric('compactor_write_build_l0_bytes')}) > 0",
                             "write amplification",
                         ),
                     ],
@@ -435,12 +442,12 @@ def section_compaction(outer_panels):
                     "Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size",
                     [
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_bloom_filter_size_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_bloom_filter_size_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_bloom_filter_size_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_bloom_filter_size_count')}[$__rate_interval])) > 0",
                             "avg_meta - {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_file_size_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_file_size_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_file_size_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_file_size_count')}[$__rate_interval])) > 0",
                             "avg_file - {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -451,7 +458,7 @@ def section_compaction(outer_panels):
                     "Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size",
                     [
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_avg_key_size_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_avg_key_size_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_avg_key_size_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_avg_key_size_count')}[$__rate_interval])) > 0",
                             "avg_key_size - {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -467,7 +474,7 @@ def section_compaction(outer_panels):
                     "Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count",
                     [
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_distinct_epoch_count_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_distinct_epoch_count_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_distinct_epoch_count_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('compactor_sstable_distinct_epoch_count_count')}[$__rate_interval])) > 0",
                             "avg_epoch_count - {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -532,8 +539,8 @@ def section_object_storage(outer_panels):
     operation_duration_blacklist = "type!~'streaming_upload_write_bytes|streaming_read'"
     write_op_filter = "type=~'upload|delete'"
     read_op_filter = "type=~'read|readv|list|metadata'"
-    request_cost_op1 = "type=~'read|streaming_read_start|delete'"
-    request_cost_op2 = "type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list'"
+    s3_request_cost_op1 = "type=~'read|streaming_read_start|streaming_read_init'"
+    s3_request_cost_op2 = "type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list'"
     return [
         outer_panels.row_collapsed(
             "Object Storage",
@@ -566,7 +573,7 @@ def section_object_storage(outer_panels):
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, type, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('object_store_operation_latency_sum', operation_duration_blacklist)}[$__rate_interval])) / sum by(le, type, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('object_store_operation_latency_count')}[$__rate_interval]))",
+                            f"sum by(le, type, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('object_store_operation_latency_sum', operation_duration_blacklist)}[$__rate_interval])) / sum by(le, type, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('object_store_operation_latency_count')}[$__rate_interval])) > 0",
                             "{{type}} avg - {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -622,12 +629,7 @@ def section_object_storage(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(irate({metric('aws_sdk_retry_counts')}[$__rate_interval])) by ({NODE_LABEL}, {COMPONENT_LABEL}, type)",
-                            "{{type}} - {{%s}} @ {{%s}}"
-                            % (COMPONENT_LABEL, NODE_LABEL),
-                        ),
-                        panels.target(
-                            f"sum(irate({metric('s3_read_request_retry_count')}[$__rate_interval])) by ({NODE_LABEL}, {COMPONENT_LABEL}, type)",
+                            f"sum(rate({metric('object_store_request_retry_count')}[$__rate_interval])) by ({NODE_LABEL}, {COMPONENT_LABEL}, type)",
                             "{{type}} - {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -646,11 +648,11 @@ def section_object_storage(outer_panels):
                             True,
                         ),
                         panels.target(
-                            f"sum({metric('object_store_operation_latency_count', request_cost_op1)}) * 0.0004 / 1000",
+                            f"sum({metric('object_store_operation_latency_count', s3_request_cost_op1)}) * 0.0004 / 1000",
                             "GET, SELECT, and all other Requests Cost",
                         ),
                         panels.target(
-                            f"sum({metric('object_store_operation_latency_count', request_cost_op2)}) * 0.005 / 1000",
+                            f"sum({metric('object_store_operation_latency_count', s3_request_cost_op2)}) * 0.005 / 1000",
                             "PUT, COPY, POST, LIST Requests Cost",
                         ),
                     ],
@@ -703,7 +705,7 @@ def section_streaming(outer_panels):
                     )
                     + [
                         panels.target(
-                            f"rate({metric('meta_barrier_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_duration_seconds_count')}[$__rate_interval])",
+                            f"rate({metric('meta_barrier_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_duration_seconds_count')}[$__rate_interval]) > 0",
                             "barrier_latency_avg",
                         ),
                     ],
@@ -735,7 +737,7 @@ def section_streaming(outer_panels):
                     [
                         panels.target(
                             f"rate({metric('source_partition_input_count')}[$__rate_interval])",
-                            "actor={{actor_id}} source={{source_id}} partition={{partition}} fragmend_id={{fragment_id}}",
+                            "actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}",
                         )
                     ],
                 ),
@@ -756,7 +758,7 @@ def section_streaming(outer_panels):
                     [
                         panels.target(
                             f"(rate({metric('source_partition_input_bytes')}[$__rate_interval]))/(1000*1000)",
-                            "actor={{actor_id}} source={{source_id}} partition={{partition}} fragmend_id={{fragment_id}}",
+                            "actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}",
                         )
                     ],
                 ),
@@ -837,8 +839,8 @@ def section_streaming(outer_panels):
                     "The figure shows the number of rows written into each materialized view per second.",
                     [
                         panels.target(
-                            f"sum(rate({metric('stream_mview_input_row_count')}[$__rate_interval])) by (fragment_id, table_id) * on(fragment_id, table_id) group_left(table_name) {metric('table_info')}",
-                            "mview {{table_id}} {{table_name}} - fragment_id {{fragment_id}}",
+                            f"rate({metric('stream_mview_input_row_count')}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) {metric('table_info')}",
+                            "mview {{table_id}} {{table_name}} - actor {{actor_id}} fragment_id {{fragment_id}}",
                         ),
                     ],
                 ),
@@ -878,7 +880,7 @@ def section_streaming(outer_panels):
                     )
                     + [
                         panels.target(
-                            f"rate({metric('meta_barrier_send_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_send_duration_seconds_count')}[$__rate_interval])",
+                            f"rate({metric('meta_barrier_send_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_send_duration_seconds_count')}[$__rate_interval]) > 0",
                             "barrier_send_latency_avg",
                         ),
                     ],
@@ -895,7 +897,7 @@ def section_streaming(outer_panels):
                     )
                     + [
                         panels.target(
-                            f"max(sum by(le, {NODE_LABEL})(rate({metric('stream_barrier_inflight_duration_seconds_sum')}[$__rate_interval]))  / sum by(le, {NODE_LABEL})(rate({metric('stream_barrier_inflight_duration_seconds_count')}[$__rate_interval])))",
+                            f"max(sum by(le, {NODE_LABEL})(rate({metric('stream_barrier_inflight_duration_seconds_sum')}[$__rate_interval]))  / sum by(le, {NODE_LABEL})(rate({metric('stream_barrier_inflight_duration_seconds_count')}[$__rate_interval]))) > 0",
                             "barrier_inflight_latency_avg",
                         ),
                     ],
@@ -913,7 +915,7 @@ def section_streaming(outer_panels):
                     )
                     + [
                         panels.target(
-                            f"sum by(le, {NODE_LABEL})(rate({metric('stream_barrier_sync_storage_duration_seconds_sum')}[$__rate_interval]))  / sum by(le, {NODE_LABEL})(rate({metric('stream_barrier_sync_storage_duration_seconds_count')}[$__rate_interval]))",
+                            f"sum by(le, {NODE_LABEL})(rate({metric('stream_barrier_sync_storage_duration_seconds_sum')}[$__rate_interval]))  / sum by(le, {NODE_LABEL})(rate({metric('stream_barrier_sync_storage_duration_seconds_count')}[$__rate_interval])) > 0",
                             "barrier_sync_latency_avg - {{%s}}" % NODE_LABEL,
                         ),
                     ],
@@ -930,7 +932,7 @@ def section_streaming(outer_panels):
                     )
                     + [
                         panels.target(
-                            f"rate({metric('meta_barrier_wait_commit_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_wait_commit_duration_seconds_count')}[$__rate_interval])",
+                            f"rate({metric('meta_barrier_wait_commit_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_wait_commit_duration_seconds_count')}[$__rate_interval]) > 0",
                             "barrier_wait_commit_avg",
                         ),
                     ],
@@ -1162,66 +1164,84 @@ def section_streaming_actors(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"(sum(rate({metric('stream_join_lookup_miss_count')}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate({metric('stream_join_lookup_total_count')}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id))",
+                            f"(sum(rate({metric('stream_join_lookup_miss_count')}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate({metric('stream_join_lookup_total_count')}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0",
                             "Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_agg_lookup_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_agg_lookup_total_count')}[$__rate_interval])) by (table_id, fragment_id))",
+                            f"(sum(rate({metric('stream_agg_lookup_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_agg_lookup_total_count')}[$__rate_interval])) by (table_id, fragment_id)) >= 0",
                             "Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_agg_distinct_cache_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_agg_distinct_total_cache_count')}[$__rate_interval])) by (table_id, fragment_id))",
+                            f"(sum(rate({metric('stream_agg_distinct_cache_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_agg_distinct_total_cache_count')}[$__rate_interval])) by (table_id, fragment_id)) >= 0",
                             "Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_group_top_n_cache_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_group_top_n_total_query_cache_count')}[$__rate_interval])) by (table_id, fragment_id))",
+                            f"(sum(rate({metric('stream_group_top_n_cache_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_group_top_n_total_query_cache_count')}[$__rate_interval])) by (table_id, fragment_id)) >= 0",
                             "Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_group_top_n_appendonly_cache_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_group_top_n_appendonly_total_query_cache_count')}[$__rate_interval])) by (table_id, fragment_id))",
+                            f"(sum(rate({metric('stream_group_top_n_appendonly_cache_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_group_top_n_appendonly_total_query_cache_count')}[$__rate_interval])) by (table_id, fragment_id)) >= 0",
                             "Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_lookup_cache_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_lookup_total_query_cache_count')}[$__rate_interval])) by (table_id, fragment_id))",
+                            f"(sum(rate({metric('stream_lookup_cache_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_lookup_total_query_cache_count')}[$__rate_interval])) by (table_id, fragment_id)) >= 0",
                             "Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_temporal_join_cache_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_temporal_join_total_query_cache_count')}[$__rate_interval])) by (table_id, fragment_id))",
+                            f"(sum(rate({metric('stream_temporal_join_cache_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_temporal_join_total_query_cache_count')}[$__rate_interval])) by (table_id, fragment_id)) >= 0",
                             "Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ",
                         ),
                         panels.target(
-                            f"1 - (sum(rate({metric('stream_materialize_cache_hit_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_materialize_cache_total_count')}[$__rate_interval])) by (table_id, fragment_id))",
+                            f"1 - (sum(rate({metric('stream_materialize_cache_hit_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_materialize_cache_total_count')}[$__rate_interval])) by (table_id, fragment_id)) >= 0",
                             "Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{%s}}"
                             % NODE_LABEL,
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_over_window_cache_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_over_window_cache_lookup_count')}[$__rate_interval])) by (table_id, fragment_id))",
+                            f"(sum(rate({metric('stream_over_window_cache_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_over_window_cache_lookup_count')}[$__rate_interval])) by (table_id, fragment_id)) >= 0",
                             "Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_over_window_range_cache_left_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_over_window_range_cache_lookup_count')}[$__rate_interval])) by (table_id, fragment_id))",
+                            f"(sum(rate({metric('stream_over_window_range_cache_left_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_over_window_range_cache_lookup_count')}[$__rate_interval])) by (table_id, fragment_id)) >= 0",
                             "Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_over_window_range_cache_right_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_over_window_range_cache_lookup_count')}[$__rate_interval])) by (table_id, fragment_id))",
+                            f"(sum(rate({metric('stream_over_window_range_cache_right_miss_count')}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate({metric('stream_over_window_range_cache_lookup_count')}[$__rate_interval])) by (table_id, fragment_id)) >= 0",
                             "Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ",
                         ),
                     ],
                 ),
                 panels.timeseries_actor_latency(
-                    "Join Executor Barrier Align",
+                    "Executor Barrier Align",
+                    "",
+                    [
+                        *quantile(
+                            lambda quantile, legend: panels.target(
+                                f"histogram_quantile({quantile}, sum(rate({metric('stream_barrier_align_duration_bucket')}[$__rate_interval])) by (le, executor, fragment_id, wait_side, {COMPONENT_LABEL}))",
+                                f"p{legend} - executor {{{{executor}}}} fragment {{{{fragment_id}}}} {{{{wait_side}}}} - {{{{{COMPONENT_LABEL}}}}}",
+                            ),
+                            [90, 99, 999, "max"],
+                        ),
+                        panels.target(
+                            f"sum by(le, executor, fragment_id, wait_side, job)(rate({metric('stream_barrier_align_duration_sum')}[$__rate_interval])) / sum by(le,executor,fragment_id,wait_side,{COMPONENT_LABEL}) (rate({metric('stream_barrier_align_duration_count')}[$__rate_interval])) > 0",
+                            "avg - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{%s}}"
+                            % COMPONENT_LABEL,
+                        ),
+                    ],
+                ),
+                panels.timeseries_actor_latency(
+                    "Merger Barrier Align",
                     "",
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('stream_join_barrier_align_duration_bucket')}[$__rate_interval])) by (le, fragment_id, wait_side, {COMPONENT_LABEL}))",
-                                f"p{legend} - fragment {{{{fragment_id}}}} {{{{wait_side}}}} - {{{{{COMPONENT_LABEL}}}}}",
+                                f"histogram_quantile({quantile}, sum(rate({metric('stream_merge_barrier_align_duration_bucket')}[$__rate_interval])) by (le, fragment_id, {COMPONENT_LABEL}))",
+                                f"p{legend} - fragment {{{{fragment_id}}}} - {{{{{COMPONENT_LABEL}}}}}",
                             ),
                             [90, 99, 999, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, fragment_id, wait_side, job)(rate({metric('stream_join_barrier_align_duration_sum')}[$__rate_interval])) / sum by(le,fragment_id,wait_side,{COMPONENT_LABEL}) (rate({metric('stream_join_barrier_align_duration_count')}[$__rate_interval]))",
-                            "avg - fragment {{fragment_id}} {{wait_side}} - {{%s}}"
+                            f"sum by(le, fragment_id, job)(rate({metric('stream_merge_barrier_align_duration_sum')}[$__rate_interval])) / sum by(le,fragment_id,{COMPONENT_LABEL}) (rate({metric('stream_merge_barrier_align_duration_count')}[$__rate_interval])) > 0",
+                            "avg - fragment {{fragment_id}} - {{%s}}"
                             % COMPONENT_LABEL,
                         ),
                     ],
@@ -1281,7 +1301,7 @@ def section_streaming_actors(outer_panels):
                             [90, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, job, actor_id, table_id) (rate({metric('stream_join_matched_join_keys_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, fragment_id, table_id) (rate({table_metric('stream_join_matched_join_keys_count')}[$__rate_interval]))",
+                            f"sum by(le, job, actor_id, table_id) (rate({metric('stream_join_matched_join_keys_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, fragment_id, table_id) (rate({table_metric('stream_join_matched_join_keys_count')}[$__rate_interval])) >= 0",
                             "avg - fragment {{fragment_id}} table_id {{table_id}} - {{%s}}"
                             % COMPONENT_LABEL,
                         ),
@@ -1766,7 +1786,7 @@ def section_batch(outer_panels):
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('batch_row_seq_scan_next_duration_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('batch_row_seq_scan_next_duration_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('batch_row_seq_scan_next_duration_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('batch_row_seq_scan_next_duration_count')}[$__rate_interval])) > 0",
                             "row_seq_scan next avg - {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -1850,17 +1870,14 @@ def section_frontend(outer_panels):
                     "Query Latency (Distributed Query Mode)",
                     "",
                     [
-                        panels.target(
-                            f"histogram_quantile(0.5, sum(rate({metric('distributed_query_latency_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
-                            "p50 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
-                        ),
-                        panels.target(
-                            f"histogram_quantile(0.9, sum(rate({metric('distributed_query_latency_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
-                            "p90 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
-                        ),
-                        panels.target(
-                            f"histogram_quantile(0.95, sum(rate({metric('distributed_query_latency_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
-                            "p99 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
+                        *quantile(
+                            lambda quantile, legend: panels.target(
+                                f"histogram_quantile({quantile}, sum(rate({metric('distributed_query_latency_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
+                                f"p{legend}"
+                                + " - {{%s}} @ {{%s}}"
+                                % (COMPONENT_LABEL, NODE_LABEL),
+                            ),
+                            [50, 90, 99, "max"],
                         ),
                     ],
                 ),
@@ -1868,17 +1885,14 @@ def section_frontend(outer_panels):
                     "Query Latency (Local Query Mode)",
                     "",
                     [
-                        panels.target(
-                            f"histogram_quantile(0.5, sum(rate({metric('frontend_latency_local_execution_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
-                            "p50 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
-                        ),
-                        panels.target(
-                            f"histogram_quantile(0.9, sum(rate({metric('frontend_latency_local_execution_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
-                            "p90 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
-                        ),
-                        panels.target(
-                            f"histogram_quantile(0.95, sum(rate({metric('frontend_latency_local_execution_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
-                            "p99 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
+                        *quantile(
+                            lambda quantile, legend: panels.target(
+                                f"histogram_quantile({quantile}, sum(rate({metric('frontend_latency_local_execution_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
+                                f"p{legend}"
+                                + " - {{%s}} @ {{%s}}"
+                                % (COMPONENT_LABEL, NODE_LABEL),
+                            ),
+                            [50, 90, 99, "max"],
                         ),
                     ],
                 ),
@@ -1930,6 +1944,11 @@ def section_hummock_read(outer_panels):
                             "data cache - {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
+                        panels.target(
+                            f"avg({metric('state_store_prefetch_memory_size')}) by ({COMPONENT_LABEL}, {NODE_LABEL})",
+                            "prefetch cache - {{%s}} @ {{%s}}"
+                            % (COMPONENT_LABEL, NODE_LABEL),
+                        ),
                     ],
                 ),
                 panels.timeseries_percentage(
@@ -1937,17 +1956,31 @@ def section_hummock_read(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"(sum(rate({table_metric('state_store_sst_store_block_request_counts', meta_miss_filter)}[$__rate_interval])) by ({COMPONENT_LABEL},{NODE_LABEL},table_id)) / (sum(rate({table_metric('state_store_sst_store_block_request_counts', meta_total_filter)}[$__rate_interval])) by ({COMPONENT_LABEL},{NODE_LABEL},table_id))",
+                            f"(sum(rate({table_metric('state_store_sst_store_block_request_counts', meta_miss_filter)}[$__rate_interval])) by ({COMPONENT_LABEL},{NODE_LABEL},table_id)) / (sum(rate({table_metric('state_store_sst_store_block_request_counts', meta_total_filter)}[$__rate_interval])) by ({COMPONENT_LABEL},{NODE_LABEL},table_id)) >= 0",
                             "meta cache miss ratio - {{table_id}} @ {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                         panels.target(
-                            f"(sum(rate({table_metric('state_store_sst_store_block_request_counts', data_miss_filter)}[$__rate_interval])) by ({COMPONENT_LABEL},{NODE_LABEL},table_id)) / (sum(rate({table_metric('state_store_sst_store_block_request_counts', data_total_filter)}[$__rate_interval])) by ({COMPONENT_LABEL},{NODE_LABEL},table_id))",
+                            f"(sum(rate({table_metric('state_store_sst_store_block_request_counts', data_miss_filter)}[$__rate_interval])) by ({COMPONENT_LABEL},{NODE_LABEL},table_id)) / (sum(rate({table_metric('state_store_sst_store_block_request_counts', data_total_filter)}[$__rate_interval])) by ({COMPONENT_LABEL},{NODE_LABEL},table_id)) >= 0",
                             "block cache miss ratio - {{table_id}} @ {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                     ],
                 ),
+                panels.timeseries_percentage(
+                    "Block Cache Efficiency",
+                    "Histogram of the estimated hit ratio of a block while in the block cache.",
+                    [
+                        *quantile(
+                            lambda quantile, legend: panels.target(
+                                f"clamp_max(histogram_quantile({quantile}, sum(rate({metric('block_efficiency_histogram_bucket')}[$__rate_interval])) by (le,{COMPONENT_LABEL},{NODE_LABEL})), 1)",
+                                f"block cache efficienfy - p{legend}"
+                                + " - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
+                            ),
+                            [10, 25, 50, 75, 90, 100],
+                        ),
+                    ]
+                ),
                 panels.timeseries_ops(
                     "Iter keys flow",
                     "",
@@ -1973,7 +2006,7 @@ def section_hummock_read(outer_panels):
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id)(rate({table_metric('state_store_iter_merge_sstable_counts_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id)(rate({table_metric('state_store_iter_merge_sstable_counts_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id)(rate({table_metric('state_store_iter_merge_sstable_counts_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id)(rate({table_metric('state_store_iter_merge_sstable_counts_count')}[$__rate_interval])) > 0",
                             "# merged ssts avg  - {{table_id}} @ {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -1993,7 +2026,7 @@ def section_hummock_read(outer_panels):
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id)(rate({table_metric('state_store_get_duration_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id) (rate({table_metric('state_store_get_duration_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id)(rate({table_metric('state_store_get_duration_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id) (rate({table_metric('state_store_get_duration_count')}[$__rate_interval])) > 0",
                             "avg - {{table_id}} {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -2006,26 +2039,26 @@ def section_hummock_read(outer_panels):
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_init_duration_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id))",
-                                f"create_iter_time p{legend} - {{{{table_id}}}} @ {{{{{COMPONENT_LABEL}}}}} @ {{{{{NODE_LABEL}}}}}",
+                                f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_init_duration_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id, iter_type))",
+                                f"create_iter_time p{legend} - {{{{iter_type}}}} {{{{table_id}}}} @ {{{{{COMPONENT_LABEL}}}}} @ {{{{{NODE_LABEL}}}}}",
                             ),
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('state_store_iter_init_duration_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_iter_init_duration_count')}[$__rate_interval]))",
-                            "create_iter_time avg - {{%s}} @ {{%s}}"
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('state_store_iter_init_duration_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, iter_type) (rate({metric('state_store_iter_init_duration_count')}[$__rate_interval])) > 0",
+                            "create_iter_time avg - {{iter_type}} {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_scan_duration_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id))",
-                                f"pure_scan_time p{legend} - {{{{table_id}}}} @ {{{{{COMPONENT_LABEL}}}}} @ {{{{{NODE_LABEL}}}}}",
+                                f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_scan_duration_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id, iter_type))",
+                                f"pure_scan_time p{legend} - {{{{iter_type}}}} {{{{table_id}}}} @ {{{{{COMPONENT_LABEL}}}}} @ {{{{{NODE_LABEL}}}}}",
                             ),
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('state_store_iter_scan_duration_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_iter_scan_duration_count')}[$__rate_interval]))",
-                            "pure_scan_time avg - {{%s}} @ {{%s}}"
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('state_store_iter_scan_duration_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, iter_type) (rate({metric('state_store_iter_scan_duration_count')}[$__rate_interval])) > 0",
+                            "pure_scan_time avg - {{iter_type}} {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                     ],
@@ -2053,7 +2086,7 @@ def section_hummock_read(outer_panels):
                     "Positive / Total",
                     [
                         panels.target(
-                            f"(sum(rate({table_metric('state_store_read_req_bloom_filter_positive_counts')}[$__rate_interval])) by (table_id,type)) / (sum(rate({table_metric('state_store_read_req_check_bloom_filter_counts')}[$__rate_interval])) by (table_id,type))",
+                            f"(sum(rate({table_metric('state_store_read_req_bloom_filter_positive_counts')}[$__rate_interval])) by (table_id,type)) / (sum(rate({table_metric('state_store_read_req_check_bloom_filter_counts')}[$__rate_interval])) by (table_id,type)) >= 0",
                             "bloom filter positive rate - {{table_id}} - {{type}}",
                         ),
                     ],
@@ -2063,7 +2096,7 @@ def section_hummock_read(outer_panels):
                     "False-Positive / Total",
                     [
                         panels.target(
-                            f"(((sum(rate({table_metric('state_store_read_req_positive_but_non_exist_counts')}[$__rate_interval])) by (table_id,type))) / (sum(rate({table_metric('state_store_read_req_check_bloom_filter_counts')}[$__rate_interval])) by (table_id,type)))",
+                            f"(((sum(rate({table_metric('state_store_read_req_positive_but_non_exist_counts')}[$__rate_interval])) by (table_id,type))) / (sum(rate({table_metric('state_store_read_req_check_bloom_filter_counts')}[$__rate_interval])) by (table_id,type))) >= 0",
                             "read req bloom filter false positive rate - {{table_id}} - {{type}}",
                         ),
                     ],
@@ -2093,8 +2126,8 @@ def section_hummock_read(outer_panels):
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                         panels.target(
-                            f"sum(rate({table_metric('state_store_iter_in_process_counts')}[$__rate_interval])) by ({COMPONENT_LABEL},{NODE_LABEL},table_id)",
-                            "iter - {{table_id}} @ {{%s}} @ {{%s}}"
+                            f"sum(rate({table_metric('state_store_iter_counts')}[$__rate_interval])) by ({COMPONENT_LABEL},{NODE_LABEL},table_id, iter_type)",
+                            "{{iter_type}} - {{table_id}} @ {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                     ],
@@ -2118,8 +2151,8 @@ def section_hummock_read(outer_panels):
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_size_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id))",
-                                f"p{legend} - {{{{table_id}}}} @ {{{{{COMPONENT_LABEL}}}}} @ {{{{{NODE_LABEL}}}}}",
+                                f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_size_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id, iter_type))",
+                                f"p{legend} - {{{{iter_type}}}} {{{{table_id}}}} @ {{{{{COMPONENT_LABEL}}}}} @ {{{{{NODE_LABEL}}}}}",
                             ),
                             [50, 99, "max"],
                         ),
@@ -2144,11 +2177,19 @@ def section_hummock_read(outer_panels):
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_item_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id))",
-                                f"p{legend} - {{{{table_id}}}} @ {{{{{COMPONENT_LABEL}}}}} @ {{{{{NODE_LABEL}}}}}",
+                                f"histogram_quantile({quantile}, sum(rate({table_metric('state_store_iter_item_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id, iter_type))",
+                                f"p{legend} - {{{{iter_type}}}} {{{{table_id}}}} @ {{{{{COMPONENT_LABEL}}}}} @ {{{{{NODE_LABEL}}}}}",
                             ),
                             [50, 99, "max"],
                         ),
+                        panels.target(
+                            f"{metric('state_store_iter_in_progress_counts')}",
+                            "Existing {{iter_type}} count @ {{table_id}}",
+                        ),
+                        panels.target(
+                            f"sum(rate({metric('state_store_iter_log_op_type_counts')}[$__rate_interval])) by (table_id, op_type)",
+                            "iter_log op count @ {{table_id}} {{op_type}}",
+                        ),
                     ],
                 ),
                 panels.timeseries_bytes_per_sec(
@@ -2188,7 +2229,7 @@ def section_hummock_read(outer_panels):
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id) (rate({table_metric('state_store_iter_fetch_meta_duration_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id) (rate({table_metric('state_store_iter_fetch_meta_duration_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id) (rate({table_metric('state_store_iter_fetch_meta_duration_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id) (rate({table_metric('state_store_iter_fetch_meta_duration_count')}[$__rate_interval])) > 0",
                             "fetch_meta_duration avg - {{table_id}} @ {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -2229,6 +2270,28 @@ def section_hummock_write(outer_panels):
                             "uploading task size - {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
+                        panels.target(
+                            f"sum({metric('state_store_uploader_imm_size')}) by ({COMPONENT_LABEL}, {NODE_LABEL})",
+                            "uploader imm size - {{%s}} @ {{%s}}"
+                            % (COMPONENT_LABEL, NODE_LABEL),
+                        ),
+                        panels.target(
+                            f"sum({metric('state_store_uploader_imm_size')}) by ({COMPONENT_LABEL}, {NODE_LABEL}) - "
+                            f"sum({metric('state_store_uploader_uploading_task_size')}) by ({COMPONENT_LABEL}, {NODE_LABEL})",
+                            "unflushed imm size - {{%s}} @ {{%s}}"
+                            % (COMPONENT_LABEL, NODE_LABEL),
+                            ),
+                        panels.target(
+                            f"sum({metric('uploading_memory_size')}) by ({COMPONENT_LABEL}, {NODE_LABEL}) - "
+                            f"sum({metric('state_store_uploader_imm_size')}) by ({COMPONENT_LABEL}, {NODE_LABEL})",
+                            "orphan imm size - {{%s}} @ {{%s}}"
+                            % (COMPONENT_LABEL, NODE_LABEL),
+                        ),
+                        panels.target(
+                            f"sum({metric('state_store_old_value_size')}) by ({COMPONENT_LABEL}, {NODE_LABEL})",
+                            "old value size - {{%s}} @ {{%s}}"
+                            % (COMPONENT_LABEL, NODE_LABEL),
+                            ),
                     ],
                 ),
                 panels.timeseries_latency(
@@ -2244,7 +2307,7 @@ def section_hummock_write(outer_panels):
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_sync_duration_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_sync_duration_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_sync_duration_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_sync_duration_count')}[$__rate_interval])) > 0",
                             "avg Sync duration - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                         *quantile(
@@ -2340,7 +2403,7 @@ def section_hummock_write(outer_panels):
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id)(rate({table_metric('state_store_write_batch_duration_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id)(rate({table_metric('state_store_write_batch_duration_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id)(rate({table_metric('state_store_write_batch_duration_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}, table_id)(rate({table_metric('state_store_write_batch_duration_count')}[$__rate_interval])) > 0",
                             "write to shared_buffer avg - {{table_id}} @ {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -2353,7 +2416,7 @@ def section_hummock_write(outer_panels):
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('state_store_write_shared_buffer_sync_time_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('state_store_write_shared_buffer_sync_time_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('state_store_write_shared_buffer_sync_time_sum')}[$__rate_interval]))  / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL})(rate({metric('state_store_write_shared_buffer_sync_time_count')}[$__rate_interval])) > 0",
                             "write to object_store - {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -2375,12 +2438,12 @@ def section_hummock_write(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({table_metric('state_store_write_batch_size_sum')}[$__rate_interval]))by({COMPONENT_LABEL},{NODE_LABEL},table_id) / sum(rate({table_metric('state_store_write_batch_size_count')}[$__rate_interval]))by({COMPONENT_LABEL},{NODE_LABEL},table_id)",
+                            f"sum(rate({table_metric('state_store_write_batch_size_sum')}[$__rate_interval]))by({COMPONENT_LABEL},{NODE_LABEL},table_id) / sum(rate({table_metric('state_store_write_batch_size_count')}[$__rate_interval]))by({COMPONENT_LABEL},{NODE_LABEL},table_id) > 0",
                             "shared_buffer - {{table_id}} @ {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                         panels.target(
-                            f"sum(rate({metric('compactor_shared_buffer_to_sstable_size_sum')}[$__rate_interval]))by({COMPONENT_LABEL}, {NODE_LABEL}) / sum(rate({metric('compactor_shared_buffer_to_sstable_size_count')}[$__rate_interval]))by({COMPONENT_LABEL}, {NODE_LABEL})",
+                            f"sum(rate({metric('compactor_shared_buffer_to_sstable_size_sum')}[$__rate_interval]))by({COMPONENT_LABEL}, {NODE_LABEL}) / sum(rate({metric('compactor_shared_buffer_to_sstable_size_count')}[$__rate_interval]))by({COMPONENT_LABEL}, {NODE_LABEL}) > 0",
                             "sync - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                     ],
@@ -2395,7 +2458,7 @@ def section_hummock_write(outer_panels):
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_write_batch_size_sum')}[$__rate_interval])) / sum by(le, table_id, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_write_batch_size_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_write_batch_size_sum')}[$__rate_interval])) / sum by(le, table_id, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_write_batch_size_count')}[$__rate_interval])) > 0",
                             "avg - {{table_id}} {{%s}} @ {{%s}}"
                             % (COMPONENT_LABEL, NODE_LABEL),
                         ),
@@ -2425,7 +2488,7 @@ def section_hummock_write(outer_panels):
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_sync_size_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_sync_size_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_sync_size_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('state_store_sync_size_count')}[$__rate_interval])) > 0",
                             "avg - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
                         ),
                     ],
@@ -2482,158 +2545,243 @@ def section_hummock_tiered_cache(outer_panels):
     block_refill_filter = 'type="block"'
     block_refill_success_filter = 'type="block",op="success"'
     block_refill_unfiltered_filter = 'type="block",op="unfiltered"'
+    cache_hit_filter = 'op="hit"'
+    cache_miss_filter = 'op="miss"'
     return [
         outer_panels.row_collapsed(
             "Hummock Tiered Cache",
             [
+                # hybrid
                 panels.timeseries_ops(
-                    "Ops",
+                    "Hybrid Cache Ops",
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('foyer_storage_op_duration_count')}[$__rate_interval])) by (foyer, op, extra, {NODE_LABEL})",
-                            "{{foyer}} file cache {{op}} {{extra}} @ {{%s}}"
+                            f"sum(rate({metric('foyer_hybrid_op_total')}[$__rate_interval])) by (name, op, {NODE_LABEL})",
+                            "{{name}} - hybrid - {{op}} @ {{%s}}"
                             % NODE_LABEL,
                         ),
                     ],
                 ),
                 panels.timeseries_latency(
-                    "Duration",
+                    "Hybrid Cache Op Duration",
                     "",
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('foyer_storage_op_duration_bucket')}[$__rate_interval])) by (le, foyer, op, extra, {NODE_LABEL}))",
+                                f"histogram_quantile({quantile}, sum(rate({metric('foyer_hybrid_op_duration_bucket')}[$__rate_interval])) by (le, name, op, {NODE_LABEL}))",
                                 f"p{legend}"
-                                + " - {{foyer}} file cache - {{op}} {{extra}} @ {{%s}}"
+                                + " - {{name}} - hybrid - {{op}} @ {{%s}}"
                                 % NODE_LABEL,
                             ),
                             [50, 90, 99, "max"],
                         ),
                     ],
                 ),
-                panels.timeseries_bytes_per_sec(
-                    "Throughput",
+                panels.timeseries_percentage(
+                    "Hybrid Cache Hit Ratio",
+                    "",
+                    [
+                        panels.target(
+                            f"sum(rate({metric('foyer_hybrid_op_total', cache_hit_filter)}[$__rate_interval])) by (name, {NODE_LABEL}) / (sum(rate({metric('foyer_hybrid_op_total', cache_hit_filter)}[$__rate_interval])) by (name, {NODE_LABEL}) + sum(rate({metric('foyer_hybrid_op_total', cache_miss_filter)}[$__rate_interval])) by (name, {NODE_LABEL}))",
+                            "{{name}} - hybrid - hit ratio @ {{%s}}" % NODE_LABEL,
+                        ),
+                    ],
+                ),
+                # memory
+                panels.timeseries_ops(
+                    "Memory Cache Ops",
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('foyer_storage_op_bytes')}[$__rate_interval])) by (foyer, op, extra, {NODE_LABEL})",
-                            "{{foyer}} file cache - {{op}} {{extra}} @ {{%s}}"
+                            f"sum(rate({metric('foyer_memory_op_total')}[$__rate_interval])) by (name, op, {NODE_LABEL})",
+                            "{{name}} - memory - {{op}} @ {{%s}}"
                             % NODE_LABEL,
                         ),
                     ],
                 ),
-                panels.timeseries_percentage(
-                    "Hit Ratio",
+                panels.timeseries_bytes(
+                    "Memory Cache Size",
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('foyer_storage_op_duration_count', file_cache_hit_filter)}[$__rate_interval])) by (foyer, {NODE_LABEL}) / (sum(rate({metric('foyer_storage_op_duration_count', file_cache_hit_filter)}[$__rate_interval])) by (foyer, {NODE_LABEL}) + sum(rate({metric('foyer_storage_op_duration_count', file_cache_miss_filter)}[$__rate_interval])) by (foyer, {NODE_LABEL}))",
-                            "{{foyer}} file cache hit ratio @ {{%s}}" % NODE_LABEL,
+                            f"sum({metric('foyer_memory_usage')}) by (name, {NODE_LABEL})",
+                            "{{name}} - memory - size @ {{%s}}" % NODE_LABEL,
                         ),
                     ],
                 ),
-                panels.timeseries_ops(
-                    "Refill Ops",
+                panels.timeseries_percentage(
+                    "Memory Cache Hit Ratio",
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('refill_duration_count')}[$__rate_interval])) by (type, op, {NODE_LABEL})",
-                            "{{type}} file cache refill - {{op}} @ {{%s}}" % NODE_LABEL,
+                            f"sum(rate({metric('foyer_memory_op_total', cache_hit_filter)}[$__rate_interval])) by (name, {NODE_LABEL}) / (sum(rate({metric('foyer_memory_op_total', cache_hit_filter)}[$__rate_interval])) by (name, {NODE_LABEL}) + sum(rate({metric('foyer_memory_op_total', cache_miss_filter)}[$__rate_interval])) by (name, {NODE_LABEL}))",
+                            "{{name}} - memory - hit ratio @ {{%s}}" % NODE_LABEL,
                         ),
+                    ],
+                ),
+                # storage
+                panels.timeseries_ops(
+                    "Storage Cache Ops",
+                    "",
+                    [
                         panels.target(
-                            f"sum(rate({metric('refill_total', refill_ops_filter)}[$__rate_interval])) by (type, op, {NODE_LABEL})",
-                            "{{type}} file cache refill - {{op}} @ {{%s}}" % NODE_LABEL,
+                            f"sum(rate({metric('foyer_storage_op_total')}[$__rate_interval])) by (name, op, {NODE_LABEL})",
+                            "{{name}} - storage - {{op}} @ {{%s}}"
+                            % NODE_LABEL,
                         ),
                     ],
                 ),
-                panels.timeseries_bytes_per_sec(
-                    "Data Refill Throughput",
+                panels.timeseries_ops(
+                    "Storage Cache Inner Ops",
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('refill_bytes')}[$__rate_interval])) by (foyer, op, {NODE_LABEL})",
-                            "{{type}} file cache - {{op}} @ {{%s}}"
+                            f"sum(rate({metric('foyer_storage_inner_op_total')}[$__rate_interval])) by (name, op, {NODE_LABEL})",
+                            "{{name}} - storage - {{op}} @ {{%s}}"
                             % NODE_LABEL,
                         ),
                     ],
                 ),
                 panels.timeseries_latency(
-                    "Refill Duration",
+                    "Storage Cache Op Duration",
                     "",
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('refill_duration_bucket')}[$__rate_interval])) by (le, foyer, op, {NODE_LABEL}))",
+                                f"histogram_quantile({quantile}, sum(rate({metric('foyer_storage_op_duration_bucket')}[$__rate_interval])) by (le, name, op, {NODE_LABEL}))",
                                 f"p{legend}"
-                                + " - {{foyer}} cache refill - {{op}} @ {{%s}}"
+                                + " - {{name}} - storage - {{op}} @ {{%s}}"
                                 % NODE_LABEL,
                             ),
                             [50, 90, 99, "max"],
                         ),
                     ],
                 ),
-                panels.timeseries_count(
-                    "Refill Queue Length",
+                panels.timeseries_latency(
+                    "Storage Cache Inner Op Duration",
+                    "",
+                    [
+                        *quantile(
+                            lambda quantile, legend: panels.target(
+                                f"histogram_quantile({quantile}, sum(rate({metric('foyer_storage_inner_op_duration_bucket')}[$__rate_interval])) by (le, name, op, {NODE_LABEL}))",
+                                f"p{legend}"
+                                + " - {{name}} - storage - {{op}} @ {{%s}}"
+                                % NODE_LABEL,
+                            ),
+                            [50, 90, 99, "max"],
+                        ),
+                    ],
+                ),
+                panels.timeseries_percentage(
+                    "Storage Cache Hit Ratio",
                     "",
                     [
                         panels.target(
-                            f"sum(refill_queue_total) by ({NODE_LABEL})",
-                            "refill queue length @ {{%s}}" % NODE_LABEL,
+                            f"sum(rate({metric('foyer_storage_op_total', cache_hit_filter)}[$__rate_interval])) by (name, {NODE_LABEL}) / (sum(rate({metric('foyer_storage_op_total', cache_hit_filter)}[$__rate_interval])) by (name, {NODE_LABEL}) + sum(rate({metric('foyer_storage_op_total', cache_miss_filter)}[$__rate_interval])) by (name, {NODE_LABEL}))",
+                            "{{name}} - storage - hit ratio @ {{%s}}" % NODE_LABEL,
                         ),
                     ],
                 ),
                 panels.timeseries_bytes(
-                    "Size",
+                    "Storage Region Size",
                     "",
                     [
                         panels.target(
-                            f"sum({metric('foyer_storage_total_bytes')}) by (foyer, {NODE_LABEL})",
-                            "{{foyer}} size @ {{%s}}" % NODE_LABEL,
+                            f"sum({metric('foyer_storage_region')}) by (name, type, {NODE_LABEL}) * on(name, {NODE_LABEL}) group_left() avg({metric('foyer_storage_region_size_bytes')}) by (name, type, {NODE_LABEL})",
+                            "{{name}} - {{type}} region - size @ {{%s}}" % NODE_LABEL,
+                        ),
+                    ],
+                ),
+                # disk
+                panels.timeseries_ops(
+                    "Disk Ops",
+                    "",
+                    [
+                        panels.target(
+                            f"sum(rate({metric('foyer_storage_disk_io_total')}[$__rate_interval])) by (name, op, {NODE_LABEL})",
+                            "{{name}} - disk - {{op}} @ {{%s}}"
+                            % NODE_LABEL,
                         ),
                     ],
                 ),
                 panels.timeseries_latency(
-                    "Inner Op Duration",
+                    "Disk Op Duration",
                     "",
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('foyer_storage_inner_op_duration_bucket')}[$__rate_interval])) by (le, foyer, op, extra, {NODE_LABEL}))",
+                                f"histogram_quantile({quantile}, sum(rate({metric('foyer_storage_disk_io_duration_bucket')}[$__rate_interval])) by (le, name, op, {NODE_LABEL}))",
                                 f"p{legend}"
-                                + " - {{foyer}} file cache - {{op}} {{extra}} @ {{%s}}"
+                                + " - {{name}} - disk - {{op}} @ {{%s}}"
                                 % NODE_LABEL,
                             ),
                             [50, 90, 99, "max"],
                         ),
                     ],
                 ),
+                panels.timeseries_bytes_per_sec(
+                    "Disk Op Throughput",
+                    "",
+                    [
+                        panels.target(
+                            f"sum(rate({metric('foyer_storage_disk_io_bytes')}[$__rate_interval])) by (name, op, {NODE_LABEL})",
+                            "{{name}} - disk - {{op}} @ {{%s}}"
+                            % NODE_LABEL,
+                        ),
+                    ],
+                ),
+                # refill
                 panels.timeseries_ops(
-                    "Slow Ops",
+                    "Refill Ops",
+                    "",
+                    [
+                        panels.target(
+                            f"sum(rate({metric('refill_duration_count')}[$__rate_interval])) by (type, op, {NODE_LABEL})",
+                            "{{type}} file cache refill - {{op}} @ {{%s}}" % NODE_LABEL,
+                        ),
+                        panels.target(
+                            f"sum(rate({metric('refill_total', refill_ops_filter)}[$__rate_interval])) by (type, op, {NODE_LABEL})",
+                            "{{type}} file cache refill - {{op}} @ {{%s}}" % NODE_LABEL,
+                        ),
+                    ],
+                ),
+                panels.timeseries_bytes_per_sec(
+                    "Data Refill Throughput",
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('foyer_storage_slow_op_duration_count')}[$__rate_interval])) by (foyer, op, extra, {NODE_LABEL})",
-                            "{{foyer}} file cache {{op}} {{extra}} @ {{%s}}"
+                            f"sum(rate({metric('refill_bytes')}[$__rate_interval])) by (foyer, op, {NODE_LABEL})",
+                            "{{type}} file cache - {{op}} @ {{%s}}"
                             % NODE_LABEL,
                         ),
                     ],
                 ),
                 panels.timeseries_latency(
-                    "Slow Op Duration",
+                    "Refill Duration",
                     "",
                     [
                         *quantile(
                             lambda quantile, legend: panels.target(
-                                f"histogram_quantile({quantile}, sum(rate({metric('foyer_storage_slow_op_duration_bucket')}[$__rate_interval])) by (le, foyer, op, extra, {NODE_LABEL}))",
+                                f"histogram_quantile({quantile}, sum(rate({metric('refill_duration_bucket')}[$__rate_interval])) by (le, foyer, op, {NODE_LABEL}))",
                                 f"p{legend}"
-                                + " - {{foyer}} file cache - {{op}} {{extra}} @ {{%s}}"
+                                + " - {{foyer}} cache refill - {{op}} @ {{%s}}"
                                 % NODE_LABEL,
                             ),
                             [50, 90, 99, "max"],
                         ),
                     ],
                 ),
+                panels.timeseries_count(
+                    "Refill Queue Length",
+                    "",
+                    [
+                        panels.target(
+                            f"sum(refill_queue_total) by ({NODE_LABEL})",
+                            "refill queue length @ {{%s}}" % NODE_LABEL,
+                        ),
+                    ],
+                ),
                 panels.timeseries_ops(
                     "Inheritance - Parent Meta Lookup Ops",
                     "",
@@ -2650,7 +2798,7 @@ def section_hummock_tiered_cache(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('refill_total', inheritance_parent_lookup_hit_filter)}[$__rate_interval])) by ({NODE_LABEL}) / (sum(rate({metric('refill_total', inheritance_parent_lookup_hit_filter)}[$__rate_interval])) by ({NODE_LABEL}) + sum(rate({metric('refill_total', inheritance_parent_lookup_miss_filter)}[$__rate_interval])) by ({NODE_LABEL}))",
+                            f"sum(rate({metric('refill_total', inheritance_parent_lookup_hit_filter)}[$__rate_interval])) by ({NODE_LABEL}) / (sum(rate({metric('refill_total', inheritance_parent_lookup_hit_filter)}[$__rate_interval])) by ({NODE_LABEL}) + sum(rate({metric('refill_total', inheritance_parent_lookup_miss_filter)}[$__rate_interval])) by ({NODE_LABEL})) >= 0",
                             "parent meta lookup hit ratio @ {{%s}}" % NODE_LABEL,
                         ),
                     ],
@@ -2671,7 +2819,7 @@ def section_hummock_tiered_cache(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('refill_total', unit_inheritance_hit_filter)}[$__rate_interval])) by ({NODE_LABEL}) / (sum(rate({metric('refill_total', unit_inheritance_hit_filter)}[$__rate_interval])) by ({NODE_LABEL}) + sum(rate({metric('refill_total', unit_inheritance_miss_filter)}[$__rate_interval])) by ({NODE_LABEL}))",
+                            f"sum(rate({metric('refill_total', unit_inheritance_hit_filter)}[$__rate_interval])) by ({NODE_LABEL}) / (sum(rate({metric('refill_total', unit_inheritance_hit_filter)}[$__rate_interval])) by ({NODE_LABEL}) + sum(rate({metric('refill_total', unit_inheritance_miss_filter)}[$__rate_interval])) by ({NODE_LABEL})) >= 0",
                             "unit inheritance ratio @ {{%s}}" % NODE_LABEL,
                         ),
                     ],
@@ -2692,7 +2840,7 @@ def section_hummock_tiered_cache(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(rate({metric('refill_total', block_refill_success_filter)}[$__rate_interval])) by ({NODE_LABEL}) / sum(rate({metric('refill_total', block_refill_unfiltered_filter)}[$__rate_interval])) by ({NODE_LABEL})",
+                            f"sum(rate({metric('refill_total', block_refill_success_filter)}[$__rate_interval])) by ({NODE_LABEL}) / sum(rate({metric('refill_total', block_refill_unfiltered_filter)}[$__rate_interval])) by ({NODE_LABEL}) >= 0",
                             "block refill ratio @ {{%s}}" % NODE_LABEL,
                         ),
                     ],
@@ -2891,7 +3039,7 @@ def section_hummock_manager(outer_panels):
                     )
                     + [
                         panels.target(
-                            f"rate({metric('storage_version_checkpoint_latency_sum')}[$__rate_interval]) / rate({metric('storage_version_checkpoint_latency_count')}[$__rate_interval])",
+                            f"rate({metric('storage_version_checkpoint_latency_sum')}[$__rate_interval]) / rate({metric('storage_version_checkpoint_latency_count')}[$__rate_interval]) > 0",
                             "version_checkpoint_latency_avg",
                         ),
                     ],
@@ -3051,7 +3199,7 @@ def grpc_metrics_target(panels, name, filter):
                 f"{name}_p99",
             ),
             panels.target(
-                f"sum(irate({metric('meta_grpc_duration_seconds_sum', filter)}[$__rate_interval])) / sum(irate({metric('meta_grpc_duration_seconds_count', filter)}[$__rate_interval]))",
+                f"sum(irate({metric('meta_grpc_duration_seconds_sum', filter)}[$__rate_interval])) / sum(irate({metric('meta_grpc_duration_seconds_count', filter)}[$__rate_interval])) > 0",
                 f"{name}_avg",
             ),
         ],
@@ -3177,7 +3325,7 @@ def section_grpc_hummock_meta_client(outer_panels):
                             "unpin_version_before_latency_p99 - {{%s}}" % NODE_LABEL,
                         ),
                         panels.target(
-                            f"sum(irate({metric('state_store_unpin_version_before_latency_sum')}[$__rate_interval])) / sum(irate({metric('state_store_unpin_version_before_latency_count')}[$__rate_interval]))",
+                            f"sum(irate({metric('state_store_unpin_version_before_latency_sum')}[$__rate_interval])) / sum(irate({metric('state_store_unpin_version_before_latency_count')}[$__rate_interval])) > 0",
                             "unpin_version_before_latency_avg",
                         ),
                         panels.target(
@@ -3203,7 +3351,7 @@ def section_grpc_hummock_meta_client(outer_panels):
                             "pin_snapshot_latencyp90 - {{%s}}" % NODE_LABEL,
                         ),
                         panels.target(
-                            f"sum(irate({metric('state_store_pin_snapshot_latency_sum')}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))",
+                            f"sum(irate({metric('state_store_pin_snapshot_latency_sum')}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0",
                             "pin_snapshot_latency_avg",
                         ),
                         panels.target(
@@ -3215,7 +3363,7 @@ def section_grpc_hummock_meta_client(outer_panels):
                             "unpin_snapshot_latency_p99 - {{%s}}" % NODE_LABEL,
                         ),
                         panels.target(
-                            f"sum(irate({metric('state_store_unpin_snapshot_latency_sum')}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))",
+                            f"sum(irate({metric('state_store_unpin_snapshot_latency_sum')}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0",
                             "unpin_snapshot_latency_avg",
                         ),
                         panels.target(
@@ -3251,7 +3399,7 @@ def section_grpc_hummock_meta_client(outer_panels):
                             "get_new_sst_ids_latency_latency_p99 - {{%s}}" % NODE_LABEL,
                         ),
                         panels.target(
-                            f"sum(irate({metric('state_store_get_new_sst_ids_latency_sum')}[$__rate_interval])) / sum(irate({metric('state_store_get_new_sst_ids_latency_count')}[$__rate_interval]))",
+                            f"sum(irate({metric('state_store_get_new_sst_ids_latency_sum')}[$__rate_interval])) / sum(irate({metric('state_store_get_new_sst_ids_latency_count')}[$__rate_interval])) > 0",
                             "get_new_sst_ids_latency_latency_avg",
                         ),
                         panels.target(
@@ -3283,7 +3431,7 @@ def section_grpc_hummock_meta_client(outer_panels):
                             "report_compaction_task_latency_p99 - {{%s}}" % NODE_LABEL,
                         ),
                         panels.target(
-                            f"sum(irate({metric('state_store_report_compaction_task_latency_sum')}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))",
+                            f"sum(irate({metric('state_store_report_compaction_task_latency_sum')}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0",
                             "report_compaction_task_latency_avg",
                         ),
                         panels.target(
@@ -3610,7 +3758,7 @@ def section_iceberg_metrics(outer_panels):
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, sink_id)(rate({metric('iceberg_write_latency_sum')}[$__rate_interval])) / sum by(le, type, job, instance) (rate({metric('iceberg_write_latency_count')}[$__rate_interval]))",
+                            f"sum by(le, sink_id)(rate({metric('iceberg_write_latency_sum')}[$__rate_interval])) / sum by(le, type, job, instance) (rate({metric('iceberg_write_latency_count')}[$__rate_interval])) > 0",
                             "avg @ {{sink_id}}",
                         ),
                     ],
@@ -3666,22 +3814,26 @@ def section_memory_manager(outer_panels):
                         ),
                     ],
                 ),
-                panels.timeseries_count(
-                    "LRU manager watermark steps",
+                panels.timeseries(
+                    "LRU manager eviction policy",
                     "",
                     [
                         panels.target(
-                            f"{metric('lru_watermark_step')}",
+                            f"{metric('lru_eviction_policy')}",
                             "",
                         ),
                     ],
                 ),
-                panels.timeseries_ms(
-                    "LRU manager diff between watermark_time and now (ms)",
-                    "watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.",
+                panels.timeseries(
+                    "LRU manager sequence",
+                    "",
                     [
                         panels.target(
-                            f"{metric('lru_physical_now_ms')} - {metric('lru_current_watermark_time_ms')}",
+                            f"{metric('lru_latest_sequence')}",
+                            "",
+                        ),
+                        panels.target(
+                            f"{metric('lru_watermark_sequence')}",
                             "",
                         ),
                     ],
@@ -3810,7 +3962,7 @@ def section_sink_metrics(outer_panels):
                             [50, 99, "max"],
                         ),
                         panels.target(
-                            f"sum by(le, connector, sink_id)(rate({metric('sink_commit_duration_sum')}[$__rate_interval])) / sum by(le, type, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('sink_commit_duration_count')}[$__rate_interval]))",
+                            f"sum by(le, connector, sink_id)(rate({metric('sink_commit_duration_sum')}[$__rate_interval])) / sum by(le, type, {COMPONENT_LABEL}, {NODE_LABEL}) (rate({metric('sink_commit_duration_count')}[$__rate_interval])) > 0",
                             "avg - {{connector}} @ {{sink_id}}",
                         ),
                     ],
@@ -3827,6 +3979,10 @@ def section_sink_metrics(outer_panels):
                             f"{metric('log_store_latest_read_epoch')}",
                             "latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}",
                         ),
+                        panels.target(
+                            f"{metric('kv_log_store_buffer_unconsumed_min_epoch')}",
+                            "Kv log store uncomsuned min epoch @ {{connector}} {{sink_id}} {{executor_id}}",
+                        ),
                     ],
                 ),
                 panels.timeseries_latency(
@@ -3840,6 +3996,16 @@ def section_sink_metrics(outer_panels):
                         ),
                     ],
                 ),
+                panels.timeseries_percentage(
+                    "Log Store Backpressure Ratio",
+                    "",
+                    [
+                        panels.target(
+                            f"avg(rate({metric('log_store_reader_wait_new_future_duration_ns')}[$__rate_interval])) by (connector, sink_id, executor_id) / 1000000000",
+                            "Backpressure @ {{connector}} {{sink_id}} {{executor_id}}",
+                        ),
+                    ],
+                ),
                 panels.timeseries_latency(
                     "Log Store Consume Persistent Log Lag",
                     "",
@@ -3933,6 +4099,24 @@ def section_sink_metrics(outer_panels):
                         ),
                     ],
                 ),
+                panels.timeseries_count(
+                    "Kv Log Store Buffer State",
+                    "",
+                    [
+                        panels.target(
+                            f"{metric('kv_log_store_buffer_unconsumed_item_count')}",
+                            "Unconsumed item count @ {{connector}} {{sink_id}} {{executor_id}}",
+                        ),
+                        panels.target(
+                            f"{metric('kv_log_store_buffer_unconsumed_row_count')}",
+                            "Unconsumed row count @ {{connector}} {{sink_id}} {{executor_id}}",
+                        ),
+                        panels.target(
+                            f"{metric('kv_log_store_buffer_unconsumed_epoch_count')}",
+                            "Unconsumed epoch count @ {{connector}} {{sink_id}} {{executor_id}}",
+                        ),
+                    ],
+                ),
                 panels.timeseries_ops(
                     "Kv Log Store Rewind Rate",
                     "",
@@ -3953,6 +4137,16 @@ def section_sink_metrics(outer_panels):
                         ),
                     ],
                 ),
+                panels.timeseries_bytes(
+                    "Chunk Buffer Size",
+                    "Total size of chunks buffered in a barrier",
+                    [
+                        panels.target(
+                            f"sum({metric('stream_sink_chunk_buffer_size')}) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) {metric('sink_info')}",
+                            "sink {{sink_id}} {{sink_name}} - actor {{actor_id}}",
+                        ),
+                    ],
+                ),
             ],
         )
     ]
@@ -4132,7 +4326,7 @@ def section_udf(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"sum(irate({metric('udf_input_chunk_rows_sum')}[$__rate_interval])) by (link, name, fragment_id) / sum(irate({metric('udf_input_chunk_rows_count')}[$__rate_interval])) by (link, name, fragment_id)",
+                            f"sum(irate({metric('udf_input_chunk_rows_sum')}[$__rate_interval])) by (link, name, fragment_id) / sum(irate({metric('udf_input_chunk_rows_count')}[$__rate_interval])) by (link, name, fragment_id) > 0",
                             "udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}",
                         ),
                     ],
@@ -4154,7 +4348,7 @@ def section_udf(outer_panels):
                             "udf_latency_p99 - {{%s}}" % NODE_LABEL,
                         ),
                         panels.target(
-                            f"sum(irate({metric('udf_latency_sum')}[$__rate_interval])) by ({COMPONENT_LABEL}, {NODE_LABEL}) / sum(irate({metric('udf_latency_count')}[$__rate_interval])) by ({COMPONENT_LABEL}, {NODE_LABEL})",
+                            f"sum(irate({metric('udf_latency_sum')}[$__rate_interval])) by ({COMPONENT_LABEL}, {NODE_LABEL}) / sum(irate({metric('udf_latency_count')}[$__rate_interval])) by ({COMPONENT_LABEL}, {NODE_LABEL}) > 0",
                             "udf_latency_avg - {{%s}}" % NODE_LABEL,
                         ),
                         panels.target(
@@ -4162,7 +4356,7 @@ def section_udf(outer_panels):
                             "udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}",
                         ),
                         panels.target(
-                            f"sum(irate({metric('udf_latency_sum')}[$__rate_interval])) by (link, name, fragment_id) / sum(irate({metric('udf_latency_count')}[$__rate_interval])) by (link, name, fragment_id)",
+                            f"sum(irate({metric('udf_latency_sum')}[$__rate_interval])) by (link, name, fragment_id) / sum(irate({metric('udf_latency_count')}[$__rate_interval])) by (link, name, fragment_id) > 0",
                             "udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}",
                         ),
                     ],
@@ -4195,6 +4389,20 @@ def section_udf(outer_panels):
                         ),
                     ],
                 ),
+                panels.timeseries_bytes(
+                    "UDF Memory Usage (bytes)",
+                    "Currently only embedded JS UDF supports this. Others will always show 0.",
+                    [
+                        panels.target(
+                            f"sum({metric('udf_memory_usage')}) by ({COMPONENT_LABEL}, {NODE_LABEL})",
+                            "udf_memory_usage - {{%s}}" % NODE_LABEL,
+                        ),
+                        panels.target(
+                            f"sum({metric('udf_memory_usage')}) by (name, fragment_id)",
+                            "udf_memory_usage - {{name}} {{fragment_id}}",
+                        ),
+                    ],
+                ),
             ],
         )
     ]
diff --git a/grafana/risingwave-dev-dashboard.json b/grafana/risingwave-dev-dashboard.json
index 0ca7142657562..223e94e051841 100644
--- a/grafana/risingwave-dev-dashboard.json
+++ b/grafana/risingwave-dev-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration from the last committed barrier's epoch time to the current time. This metric reflects the data freshness of the system. During this time, no new data has been committed.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_pending_time","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier pending time (secs)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragmend_id={{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragmend_id={{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, table_id) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - fragment_id {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":36,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":41,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, wait_side, job)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The actor-level memory usage statistics reported by TaskLocalAlloc. (Disabled by default)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":67,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":84,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":87,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during computation. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during source data ingestion. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during data sink out. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":91,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":97,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":118,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 upload task duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 upload task duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax upload task duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"syncing epoch count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handler pending event number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 {{event_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{event_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{event_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax finished_task_wait_poll {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handle latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":133,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":159,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":168,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} size @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_slow_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":187,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":207,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":210,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":214,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":217,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":221,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":226,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":234,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":243,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":254,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":257,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":272,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka high watermark by source and partition and source latest message by partition, source and actor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"high watermark: source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest msg: source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka high watermark and source latest message","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":285,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":291,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":299,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":304,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":29},"height":null,"hideTimeOverride":false,"id":305,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":309,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":310,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":10,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":14,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration from the last committed barrier's epoch time to the current time. This metric reflects the data freshness of the system. During this time, no new data has been committed.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_pending_time","metric":"","query":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier pending time (secs)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}} fragment_id {{fragment_id}}","metric":"","query":"rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":37,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","query":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":42,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} desc: {{desc}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, executor, fragment_id, wait_side, job)(rate(stream_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,executor,fragment_id,wait_side,job) (rate(stream_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - executor {{executor}} fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"sum by(le, executor, fragment_id, wait_side, job)(rate(stream_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,executor,fragment_id,wait_side,job) (rate(stream_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} - {{job}}","metric":"","query":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Merger Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"sum by(le, job, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The actor-level memory usage statistics reported by TaskLocalAlloc. (Disabled by default)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":69,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":86,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":89,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during computation. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during source data ingestion. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during data sink out. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":93,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":99,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"prefetch cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the estimated hit ratio of a block while in the block cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p10 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p25 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p50 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p75 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p90 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p100 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Cache Efficiency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{iter_type}} - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Existing {{iter_type}} count @ {{table_id}}","metric":"","query":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter_log op count @ {{table_id}} {{op_type}}","metric":"","query":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":121,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploader imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unflushed imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"orphan imm size - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"old value size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg Sync duration - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"syncing epoch count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handler pending event number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handle latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":136,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","query":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":162,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|streaming_read_init',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|streaming_read_init',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":171,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - size @ {{instance}}","metric":"","query":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - {{type}} region - size @ {{instance}}","metric":"","query":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Region Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":197,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":217,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":220,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":224,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":227,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":231,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":236,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":244,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":253,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager eviction policy","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager sequence","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":264,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":267,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","query":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Kv log store uncomsuned min epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, executor_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Backpressure @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, executor_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Backpressure Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed item count @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed row count @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed epoch count @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Buffer State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total size of chunks buffered in a barrier","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Chunk Buffer Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":285,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka high watermark by source and partition and source latest message by partition, source and actor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"high watermark: source={{source_id}} partition={{partition}}","metric":"","query":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest msg: source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka high watermark and source latest message","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":291,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":299,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":304,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":305,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":309,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":310,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":311,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":312,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":313,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":314,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}}","metric":"","query":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":315,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":316,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":317,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":29},"height":null,"hideTimeOverride":false,"id":318,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":319,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":320,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":321,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":322,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":323,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Currently only embedded JS UDF supports this. Others will always show 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":324,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{instance}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{name}} {{fragment_id}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Memory Usage (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
diff --git a/grafana/risingwave-user-dashboard.dashboard.py b/grafana/risingwave-user-dashboard.dashboard.py
index b476de74f0f88..20ba4a857eaca 100644
--- a/grafana/risingwave-user-dashboard.dashboard.py
+++ b/grafana/risingwave-user-dashboard.dashboard.py
@@ -49,6 +49,13 @@ def section_actor_info(outer_panels):
                     [panels.table_target(f"group({metric('table_info')}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)")],
                     ["table_id", "table_name", "table_type", "materialized_view_id", "fragment_id", "compaction_group_id"],
                 ),
+                panels.table_info(
+                    "Actor Count (Group By Compute Node)",
+                    "Actor count per compute node",
+                    [panels.table_target(f"count({metric('actor_info')}) by (compute_node)")],
+                    ["table_id", "table_name", "table_type", "materialized_view_id", "fragment_id", "compaction_group_id"],
+                    dict.fromkeys(["Time"], True)
+                )
             ],
         )
     ]
@@ -113,7 +120,7 @@ def section_overview(panels):
             )
             + [
                 panels.target(
-                    f"rate({metric('meta_barrier_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_duration_seconds_count')}[$__rate_interval])",
+                    f"rate({metric('meta_barrier_duration_seconds_sum')}[$__rate_interval]) / rate({metric('meta_barrier_duration_seconds_count')}[$__rate_interval]) > 0",
                     "barrier_latency_avg",
                 ),
             ],
@@ -409,35 +416,35 @@ def section_memory(outer_panels):
                     "",
                     [
                         panels.target(
-                            f"(sum(rate({metric('stream_join_lookup_miss_count')}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate({metric('stream_join_lookup_total_count')}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))",
+                            f"(sum(rate({metric('stream_join_lookup_miss_count')}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate({metric('stream_join_lookup_total_count')}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ",
                             "join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_agg_lookup_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_agg_lookup_total_count')}[$__rate_interval])) by (table_id, actor_id))",
+                            f"(sum(rate({metric('stream_agg_lookup_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_agg_lookup_total_count')}[$__rate_interval])) by (table_id, actor_id)) >=0 ",
                             "Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_agg_distinct_cache_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_agg_distinct_total_cache_count')}[$__rate_interval])) by (table_id, actor_id))",
+                            f"(sum(rate({metric('stream_agg_distinct_cache_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_agg_distinct_total_cache_count')}[$__rate_interval])) by (table_id, actor_id)) >=0",
                             "Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_group_top_n_cache_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_group_top_n_total_query_cache_count')}[$__rate_interval])) by (table_id, actor_id))",
+                            f"(sum(rate({metric('stream_group_top_n_cache_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_group_top_n_total_query_cache_count')}[$__rate_interval])) by (table_id, actor_id)) >=0",
                             "Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_group_top_n_appendonly_cache_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_group_top_n_appendonly_total_query_cache_count')}[$__rate_interval])) by (table_id, actor_id))",
+                            f"(sum(rate({metric('stream_group_top_n_appendonly_cache_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_group_top_n_appendonly_total_query_cache_count')}[$__rate_interval])) by (table_id, actor_id)) >=0",
                             "Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_lookup_cache_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_lookup_total_query_cache_count')}[$__rate_interval])) by (table_id, actor_id))",
+                            f"(sum(rate({metric('stream_lookup_cache_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_lookup_total_query_cache_count')}[$__rate_interval])) by (table_id, actor_id)) >=0",
                             "Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ",
                         ),
                         panels.target(
-                            f"(sum(rate({metric('stream_temporal_join_cache_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_temporal_join_total_query_cache_count')}[$__rate_interval])) by (table_id, actor_id))",
+                            f"(sum(rate({metric('stream_temporal_join_cache_miss_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_temporal_join_total_query_cache_count')}[$__rate_interval])) by (table_id, actor_id)) >=0",
                             "Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ",
                         ),
                         panels.target(
-                            f"1 - (sum(rate({metric('stream_materialize_cache_hit_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_materialize_cache_total_count')}[$__rate_interval])) by (table_id, actor_id))",
+                            f"1 - (sum(rate({metric('stream_materialize_cache_hit_count')}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate({metric('stream_materialize_cache_total_count')}[$__rate_interval])) by (table_id, actor_id)) >=0",
                             "materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{%s}}"
                             % NODE_LABEL,
                         ),
@@ -648,7 +655,7 @@ def section_storage(outer_panels):
                             [50, 99],
                         ),
                         panels.target(
-                            f"sum by(le, {COMPONENT_LABEL}) (rate({metric('state_store_sync_size_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}) (rate({metric('state_store_sync_size_count')}[$__rate_interval]))",
+                            f"sum by(le, {COMPONENT_LABEL}) (rate({metric('state_store_sync_size_sum')}[$__rate_interval])) / sum by(le, {COMPONENT_LABEL}) (rate({metric('state_store_sync_size_count')}[$__rate_interval])) > 0",
                             "avg - {{%s}}" % COMPONENT_LABEL,
                         ),
                     ],
@@ -785,17 +792,14 @@ def section_batch(outer_panels):
                     "Query Latency in Distributed Execution Mode",
                     "",
                     [
-                        panels.target(
-                            f"histogram_quantile(0.5, sum(rate({metric('distributed_query_latency_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
-                            "p50 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
-                        ),
-                        panels.target(
-                            f"histogram_quantile(0.9, sum(rate({metric('distributed_query_latency_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
-                            "p90 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
-                        ),
-                        panels.target(
-                            f"histogram_quantile(0.95, sum(rate({metric('distributed_query_latency_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
-                            "p99 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
+                        *quantile(
+                            lambda quantile, legend: panels.target(
+                                f"histogram_quantile({quantile}, sum(rate({metric('distributed_query_latency_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
+                                f"p{legend}"
+                                + " - {{%s}} @ {{%s}}"
+                                % (COMPONENT_LABEL, NODE_LABEL),
+                            ),
+                            [50, 90, 99, "max"],
                         ),
                     ],
                 ),
@@ -803,17 +807,14 @@ def section_batch(outer_panels):
                     "Query Latency in Local Execution Mode",
                     "",
                     [
-                        panels.target(
-                            f"histogram_quantile(0.5, sum(rate({metric('frontend_latency_local_execution_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
-                            "p50 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
-                        ),
-                        panels.target(
-                            f"histogram_quantile(0.9, sum(rate({metric('frontend_latency_local_execution_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
-                            "p90 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
-                        ),
-                        panels.target(
-                            f"histogram_quantile(0.95, sum(rate({metric('frontend_latency_local_execution_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
-                            "p99 - {{%s}} @ {{%s}}" % (COMPONENT_LABEL, NODE_LABEL),
+                        *quantile(
+                            lambda quantile, legend: panels.target(
+                                f"histogram_quantile({quantile}, sum(rate({metric('frontend_latency_local_execution_bucket')}[$__rate_interval])) by (le, {COMPONENT_LABEL}, {NODE_LABEL}))",
+                                f"p{legend}"
+                                + " - {{%s}} @ {{%s}}"
+                                % (COMPONENT_LABEL, NODE_LABEL),
+                            ),
+                            [50, 90, 99, "max"],
                         ),
                     ],
                 ),
diff --git a/grafana/risingwave-user-dashboard.json b/grafana/risingwave-user-dashboard.json
index 5fbc9681be606..037679704e38f 100644
--- a/grafana/risingwave-user-dashboard.json
+++ b/grafana/risingwave-user-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n            - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n            - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n            - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n            - Write Stall: Compaction cannot keep up. Stall foreground write.\n            ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":15,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":18,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":27,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":31,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":38,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":45,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":51,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n            - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n            - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n            - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n            - Write Stall: Compaction cannot keep up. Stall foreground write.\n            ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","query":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","query":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","query":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","query":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","query":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","query":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":19,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","query":"rate(actor_memory_usage[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","query":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","query":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":28,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":32,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","query":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":46,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":52,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0}
diff --git a/integration_tests/ad-click/docker-compose.yml b/integration_tests/ad-click/docker-compose.yml
index f1a2bbc8419cb..2b84bb00d9950 100644
--- a/integration_tests/ad-click/docker-compose.yml
+++ b/integration_tests/ad-click/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -37,7 +36,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/ad-ctr/docker-compose.yml b/integration_tests/ad-ctr/docker-compose.yml
index fb297a8fec8ab..2bec6c35295b8 100644
--- a/integration_tests/ad-ctr/docker-compose.yml
+++ b/integration_tests/ad-ctr/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -37,7 +36,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/big-query-sink/bq_prepare.sql b/integration_tests/big-query-sink/bq_prepare.sql
index 8ba4d670bfc61..82035c834686a 100644
--- a/integration_tests/big-query-sink/bq_prepare.sql
+++ b/integration_tests/big-query-sink/bq_prepare.sql
@@ -4,3 +4,35 @@ CREATE TABLE `rwctest.bqtest.bq_sink` (
     target_id STRING,
     event_timestamp TIMESTAMP
 );
+
+DROP TABLE IF EXISTS `rwctest.bqtest.bq_sink_data_types`;
+CREATE TABLE `rwctest.bqtest.bq_sink_data_types` (
+    types_id INT64,
+    c_boolean BOOL,
+    c_smallint INT64,
+    c_integer INT64,
+    c_bigint INT64,
+    c_decimal NUMERIC,
+    c_double_precision FLOAT64,
+    c_varchar STRING,
+    c_bytea BYTES,
+    c_date DATE,
+    c_timestamp DATETIME,
+    c_timestamptz TIMESTAMP,
+    c_interval INTERVAL,
+    c_jsonb JSON,
+    c_boolean_array ARRAY<BOOL>,
+    c_smallint_array ARRAY<INT64>,
+    c_integer_array ARRAY<INT64>,
+    c_bigint_array ARRAY<INT64>,
+    c_decimal_array ARRAY<NUMERIC>,
+    c_double_precision_array ARRAY<FLOAT64>,
+    c_varchar_array ARRAY<STRING>,
+    c_bytea_array ARRAY<BYTES>,
+    c_date_array ARRAY<DATE>,
+    c_timestamp_array ARRAY<DATETIME>,
+    c_timestamptz_array ARRAY<TIMESTAMP>,
+    c_interval_array ARRAY<INTERVAL>,
+    c_jsonb_array ARRAY<JSON>,
+    c_struct STRUCT<s_int INT64, s_bool BOOL>
+);
diff --git a/integration_tests/big-query-sink/create_sink.sql b/integration_tests/big-query-sink/create_sink.sql
index 3b8ed3b3ef9d8..a41fe0243120d 100644
--- a/integration_tests/big-query-sink/create_sink.sql
+++ b/integration_tests/big-query-sink/create_sink.sql
@@ -27,3 +27,15 @@ FROM
     -- region = '${aws_region}',
     -- force_append_only='true',
 -- );
+
+CREATE SINK bq_sink_data_types_sink
+FROM
+    bq_sink_data_types WITH (
+    connector = 'bigquery',
+    type = 'append-only',
+    bigquery.local.path= '/gcp-rwctest.json',
+    bigquery.project= 'rwctest',
+    bigquery.dataset= 'bqtest',
+    bigquery.table= 'bq_sink_data_types',
+    force_append_only='true'
+);
diff --git a/integration_tests/big-query-sink/create_source.sql b/integration_tests/big-query-sink/create_source.sql
index bfc49aee69ce4..526ad45e59d54 100644
--- a/integration_tests/big-query-sink/create_source.sql
+++ b/integration_tests/big-query-sink/create_source.sql
@@ -14,3 +14,41 @@ CREATE table user_behaviors (
     fields.user_id.end = '100',
     datagen.rows.per.second = '10'
 ) FORMAT PLAIN ENCODE JSON;
+
+CREATE TABLE bq_sink_data_types (
+    types_id int,
+    c_boolean boolean,
+    c_smallint smallint,
+    c_integer integer,
+    c_bigint bigint,
+    c_decimal decimal,
+    c_double_precision double precision,
+    c_varchar varchar,
+    c_bytea bytea,
+    c_date date,
+    c_timestamp timestamp,
+    c_timestamptz timestamptz,
+    c_interval interval,
+    c_jsonb jsonb,
+    c_boolean_array boolean[],
+    c_smallint_array smallint[],
+    c_integer_array integer[],
+    c_bigint_array bigint[],
+    c_decimal_array decimal[],
+    c_double_precision_array double precision[],
+    c_varchar_array varchar[],
+    c_bytea_array bytea[],
+    c_date_array date[],
+    c_timestamp_array timestamp[],
+    c_timestamptz_array timestamptz[],
+    c_interval_array interval[],
+    c_jsonb_array jsonb[],
+    c_struct STRUCT<s_int INTEGER, s_bool boolean>,
+    PRIMARY KEY(types_id)
+);
+
+INSERT INTO bq_sink_data_types VALUES (1, False, 0, 0, 0, 0, 0, '', '\x00', '0001-01-01', '0001-01-01 00:00:00', '0001-01-01 00:00:00'::timestamptz, interval '0 second', '{}', array[]::boolean[], array[]::smallint[], array[]::integer[], array[]::bigint[], array[]::decimal[], array[]::double precision[], array[]::varchar[], array[]::bytea[], array[]::date[], array[]::timestamp[], array[]::timestamptz[], array[]::interval[], array[]::jsonb[], ROW(1, False));
+
+INSERT INTO bq_sink_data_types VALUES (2, False, -32767, -2147483647, -9223372036854775807, -10.0, -10000.0, 'aa', '\x00', '1970-01-01', '1970-01-01 00:00:00.123456', '1970-01-01 00:00:00.123456Z', interval '10 second', '{}', array[False::boolean]::boolean[], array[-32767::smallint]::smallint[], array[-2147483647::integer]::integer[], array[-9223372036854775807::bigint]::bigint[], array[-10.0::decimal]::decimal[], array[-10000.0::double precision]::double precision[], array[''::varchar]::varchar[], array['\x00'::bytea]::bytea[], array['1970-01-01'::date]::date[], array['1970-01-01 00:00:00'::timestamp]::timestamp[], array['1970-01-01 00:00:00Z'::timestamptz]::timestamptz[], array[interval '0 second'::interval]::interval[], array['{}'::jsonb]::jsonb[], ROW(2, True));
+
+INSERT INTO bq_sink_data_types VALUES (3, True, 32767, 2147483647, 9223372036854775807, -10.0, 10000.0, 'zzzzzz', '\xffffff', '9999-12-31', '9999-12-31 23:59:59', '9999-12-31 23:59:59Z', interval '9990 year', '{"whatever":"meaningless"}', array[True::boolean]::boolean[], array[32767::smallint]::smallint[], array[2147483647::integer]::integer[], array[9223372036854775807::bigint]::bigint[], array[-10.0::decimal]::decimal[], array[10000.0::double precision]::double precision[], array['zzzzzz'::varchar]::varchar[], array['\xffffff'::bytea]::bytea[], array['9999-12-31'::date]::date[], array['9999-12-31 23:59:59'::timestamp]::timestamp[], array['9999-12-31 23:59:59Z'::timestamptz]::timestamptz[], array[interval '9990 year'::interval]::interval[], array['{"whatever":"meaningless"}'::jsonb]::jsonb[], ROW(3, True));
diff --git a/integration_tests/big-query-sink/docker-compose.yml b/integration_tests/big-query-sink/docker-compose.yml
index ca16ca8f90b3b..279f43a43b217 100644
--- a/integration_tests/big-query-sink/docker-compose.yml
+++ b/integration_tests/big-query-sink/docker-compose.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
@@ -7,10 +6,10 @@ services:
       service: risingwave-standalone
     volumes:
       - "../../gcp-rwctest.json:/gcp-rwctest.json"
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -32,7 +31,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/big-query-sink/sink_check.py b/integration_tests/big-query-sink/sink_check.py
index 7388c853c11cd..f669f0336f597 100644
--- a/integration_tests/big-query-sink/sink_check.py
+++ b/integration_tests/big-query-sink/sink_check.py
@@ -2,7 +2,7 @@
 import subprocess
 import sys
 
-relations = ['rwctest.bqtest.bq_sink']
+relations = ['rwctest.bqtest.bq_sink', 'rwctest.bqtest.bq_sink_data_types']
 
 failed_cases = []
 for rel in relations:
diff --git a/integration_tests/cassandra-and-scylladb-sink/docker-compose.yml b/integration_tests/cassandra-and-scylladb-sink/docker-compose.yml
index 425e086c56ba0..9f09b203ef700 100644
--- a/integration_tests/cassandra-and-scylladb-sink/docker-compose.yml
+++ b/integration_tests/cassandra-and-scylladb-sink/docker-compose.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 services:
   cassandra:
     image: cassandra:4.0
@@ -22,10 +21,10 @@ services:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -45,7 +44,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/cdn-metrics/docker-compose.yml b/integration_tests/cdn-metrics/docker-compose.yml
index 69eb63a448eb2..05d3d786e6279 100644
--- a/integration_tests/cdn-metrics/docker-compose.yml
+++ b/integration_tests/cdn-metrics/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -37,7 +36,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/citus-cdc/docker-compose.yml b/integration_tests/citus-cdc/docker-compose.yml
index d77e0e80dbfa9..8afb665e02cd1 100644
--- a/integration_tests/citus-cdc/docker-compose.yml
+++ b/integration_tests/citus-cdc/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -90,7 +89,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/clickhouse-sink/docker-compose.yml b/integration_tests/clickhouse-sink/docker-compose.yml
index 76b0f7fe607f5..beb2ee1254739 100644
--- a/integration_tests/clickhouse-sink/docker-compose.yml
+++ b/integration_tests/clickhouse-sink/docker-compose.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 services:
   clickhouse-server:
     image: clickhouse/clickhouse-server:23.3.8.21-alpine
@@ -17,10 +16,10 @@ services:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -36,7 +35,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/clickstream/docker-compose.yml b/integration_tests/clickstream/docker-compose.yml
index faf66b770af7d..4015a3a976ced 100644
--- a/integration_tests/clickstream/docker-compose.yml
+++ b/integration_tests/clickstream/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -37,7 +36,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/client-library/client_test.py b/integration_tests/client-library/client_test.py
index a970ace57bbcf..f5621b46b0648 100644
--- a/integration_tests/client-library/client_test.py
+++ b/integration_tests/client-library/client_test.py
@@ -4,42 +4,45 @@
 
 
 def check_go():
-    subprocess.run(["docker", "compose", "exec", "go-lang", "bash", "-c", "cd /go-client && ./run.sh"], check=True)
+    subprocess.run(["docker", "compose", "exec", "go-lang", "bash", "-c", "cd /go-client && ./run.sh"],
+                   cwd="integration_tests/client-library", check=True)
 
 
 def check_python():
-    subprocess.run(["docker", "compose", "exec", "python", "bash", "-c",
-                    "cd /python-client && pip3 install -r requirements.txt && pytest"], check=True)
+    subprocess.run(["docker", "compose", "exec", "python", "bash", "-c", "cd /python-client && pip3 install -r requirements.txt && pytest"],
+                   cwd="integration_tests/client-library", check=True)
 
 
 def check_java():
     subprocess.run(["docker", "compose", "exec", "java", "bash", "-c", "apt-get update && apt-get install -y maven"],
-                   check=True)
-    subprocess.run(["docker", "compose", "exec", "java", "bash", "-c", "cd /java-client && mvn clean test"], check=True)
+                   cwd="integration_tests/client-library", check=True)
+    subprocess.run(["docker", "compose", "exec", "java", "bash", "-c", "cd /java-client && mvn clean test"],
+                   cwd="integration_tests/client-library", check=True)
 
 
 def check_nodejs():
     subprocess.run(
         ["docker", "compose", "exec", "nodejs", "bash", "-c", "cd /nodejs-client && npm install && npm test"],
-        check=True)
+        cwd="integration_tests/client-library", check=True)
 
 
 def check_php():
     subprocess.run(
         ["docker", "compose", "exec", "php", "bash", "-c", "cd /php-client && phpunit tests/RWClientTest.php"],
-        check=True)
+        cwd="integration_tests/client-library", check=True)
 
 
 def check_ruby():
     subprocess.run(["docker", "compose", "exec", "ruby", "bash", "-c", "cd /ruby-client && ruby test/crud_test.rb"],
-                   check=True)
+                   cwd="integration_tests/client-library", check=True)
 
 
 def check_spring_boot():
-    subprocess.run(["docker", "compose", "exec", "spring-boot", "bash", "-c",
-                    "cd /spring-boot-client && mvn spring-boot:run"], check=True)
+    subprocess.run(["docker", "compose", "exec", "spring-boot", "bash", "-c", "cd /spring-boot-client && mvn spring-boot:run"],
+                   cwd="integration_tests/client-library", check=True)
 
-subprocess.run(["docker", "compose", "up", "-d"], check=True)
+subprocess.run(["docker", "compose", "up", "-d"],
+               cwd="integration_tests/client-library", check=True)
 sleep(10)
 
 failed_cases = []
@@ -68,6 +71,3 @@ def check_spring_boot():
 if len(failed_cases) != 0:
     print(f"--- client check failed for case\n{failed_cases}")
     sys.exit(1)
-
-print("--- docker compose down")
-subprocess.run(["docker", "compose", "down", "--remove-orphans", "-v"], check=True)
diff --git a/integration_tests/client-library/csharp/csharp.csproj b/integration_tests/client-library/csharp/csharp.csproj
index 2d7ae2b7e2f48..f57b4028aabe9 100644
--- a/integration_tests/client-library/csharp/csharp.csproj
+++ b/integration_tests/client-library/csharp/csharp.csproj
@@ -12,7 +12,7 @@
   <ItemGroup>
     <PackageReference Include="coverlet.collector" Version="6.0.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
-    <PackageReference Include="Npgsql" Version="8.0.2" />
+    <PackageReference Include="Npgsql" Version="8.0.3" />
     <PackageReference Include="xunit" Version="2.5.3" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" />
   </ItemGroup>
diff --git a/integration_tests/client-library/docker-compose.yml b/integration_tests/client-library/docker-compose.yml
index d2e4663fdc378..c8a03d353b18e 100644
--- a/integration_tests/client-library/docker-compose.yml
+++ b/integration_tests/client-library/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -63,7 +62,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/cockroach-sink/docker-compose.yml b/integration_tests/cockroach-sink/docker-compose.yml
index a205dca9e19cf..d325c57865baf 100644
--- a/integration_tests/cockroach-sink/docker-compose.yml
+++ b/integration_tests/cockroach-sink/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -38,7 +37,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/datagen/go.mod b/integration_tests/datagen/go.mod
index 5056ca6e82848..e9bb8cda78a8d 100644
--- a/integration_tests/datagen/go.mod
+++ b/integration_tests/datagen/go.mod
@@ -70,12 +70,12 @@ require (
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/stretchr/testify v1.8.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
-	golang.org/x/crypto v0.18.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 // indirect
-	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect
-	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/term v0.16.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
diff --git a/integration_tests/datagen/go.sum b/integration_tests/datagen/go.sum
index a35ed04f01a33..b65f55e0e9854 100644
--- a/integration_tests/datagen/go.sum
+++ b/integration_tests/datagen/go.sum
@@ -455,8 +455,8 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
-golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -538,8 +538,8 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -626,12 +626,12 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
-golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/integration_tests/debezium-mongo/docker-compose.yaml b/integration_tests/debezium-mongo/docker-compose.yaml
index 4ae90ea22eb94..886e4622ad6ac 100644
--- a/integration_tests/debezium-mongo/docker-compose.yaml
+++ b/integration_tests/debezium-mongo/docker-compose.yaml
@@ -5,10 +5,10 @@ services:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
diff --git a/integration_tests/debezium-mysql/docker-compose.yml b/integration_tests/debezium-mysql/docker-compose.yml
index 77ff7689b78f8..6cb577ac23886 100644
--- a/integration_tests/debezium-mysql/docker-compose.yml
+++ b/integration_tests/debezium-mysql/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -78,7 +77,7 @@ volumes:
     external: false
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/debezium-postgres/docker-compose.yml b/integration_tests/debezium-postgres/docker-compose.yml
index ef9094e432f6f..327cb44d6db7c 100644
--- a/integration_tests/debezium-postgres/docker-compose.yml
+++ b/integration_tests/debezium-postgres/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -88,7 +87,7 @@ volumes:
     external: false
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/debezium-sqlserver/docker-compose.yml b/integration_tests/debezium-sqlserver/docker-compose.yml
index 848d2dc5393ab..9d4bbbf0a5bb6 100644
--- a/integration_tests/debezium-sqlserver/docker-compose.yml
+++ b/integration_tests/debezium-sqlserver/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -76,7 +75,7 @@ volumes:
     external: false
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/deltalake-sink/docker-compose.yml b/integration_tests/deltalake-sink/docker-compose.yml
index a486e2336b502..2a799f9fcf45b 100644
--- a/integration_tests/deltalake-sink/docker-compose.yml
+++ b/integration_tests/deltalake-sink/docker-compose.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 services:
   spark:
     image: apache/spark:3.3.1
@@ -13,10 +12,10 @@ services:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -32,7 +31,7 @@ services:
 volumes:
   compute-node-0:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/doris-sink/docker-compose.yml b/integration_tests/doris-sink/docker-compose.yml
index fc7cfd751e989..4b43632f51319 100644
--- a/integration_tests/doris-sink/docker-compose.yml
+++ b/integration_tests/doris-sink/docker-compose.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 services:
   fe:
     platform: linux/amd64
@@ -35,10 +34,10 @@ services:
     networks:
       mynetwork:
         ipv4_address: 172.21.0.4
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
     networks:
       mynetwork:
         ipv4_address: 172.21.0.5
@@ -86,7 +85,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/dynamodb/create_sink.sql b/integration_tests/dynamodb/create_sink.sql
new file mode 100644
index 0000000000000..6de71404a9da1
--- /dev/null
+++ b/integration_tests/dynamodb/create_sink.sql
@@ -0,0 +1,13 @@
+CREATE SINK dyn_sink
+FROM
+  movies
+WITH
+(
+  connector = 'dynamodb',
+  table = 'Movies',
+  primary_key = 'year,title',
+  endpoint = 'http://dynamodb:8000',
+  region = 'us',
+  access_key = 'ac',
+  secret_key = 'sk'
+);
diff --git a/integration_tests/dynamodb/create_source.sql b/integration_tests/dynamodb/create_source.sql
new file mode 100644
index 0000000000000..24cd3ed52e3fd
--- /dev/null
+++ b/integration_tests/dynamodb/create_source.sql
@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS movies (
+    year integer,
+    title varchar,
+    description varchar,
+    primary key (year, title)
+);
diff --git a/integration_tests/dynamodb/create_table_request.json b/integration_tests/dynamodb/create_table_request.json
new file mode 100644
index 0000000000000..b10747e1116d5
--- /dev/null
+++ b/integration_tests/dynamodb/create_table_request.json
@@ -0,0 +1,15 @@
+{
+  "TableName": "Movies",
+  "KeySchema": [
+    { "AttributeName": "year", "KeyType": "HASH" },
+    { "AttributeName": "title", "KeyType": "RANGE" }
+  ],
+  "AttributeDefinitions": [
+    { "AttributeName": "year", "AttributeType": "N" },
+    { "AttributeName": "title", "AttributeType": "S" }
+  ],
+  "ProvisionedThroughput": {
+    "ReadCapacityUnits": 5,
+    "WriteCapacityUnits": 5
+  }
+}
diff --git a/integration_tests/dynamodb/data_check b/integration_tests/dynamodb/data_check
new file mode 100644
index 0000000000000..991948c9a7971
--- /dev/null
+++ b/integration_tests/dynamodb/data_check
@@ -0,0 +1 @@
+movies
diff --git a/integration_tests/dynamodb/docker-compose.yml b/integration_tests/dynamodb/docker-compose.yml
new file mode 100644
index 0000000000000..0bc6a50849976
--- /dev/null
+++ b/integration_tests/dynamodb/docker-compose.yml
@@ -0,0 +1,46 @@
+---
+version: "3"
+services:
+  dynamodb:
+    image: amazon/dynamodb-local
+    ports:
+      - "8000:8000"
+    command: "-jar DynamoDBLocal.jar -sharedDb -inMemory -port 8000"
+  risingwave-standalone:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: risingwave-standalone
+  postgres-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: postgres-0
+  grafana-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: grafana-0
+  minio-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: minio-0
+  prometheus-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: prometheus-0
+  message_queue:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: message_queue
+volumes:
+  risingwave-standalone:
+    external: false
+  postgres-0:
+    external: false
+  grafana-0:
+    external: false
+  minio-0:
+    external: false
+  prometheus-0:
+    external: false
+  message_queue:
+    external: false
+name: risingwave-compose
diff --git a/integration_tests/dynamodb/prepare.sh b/integration_tests/dynamodb/prepare.sh
new file mode 100644
index 0000000000000..29a8009b1635a
--- /dev/null
+++ b/integration_tests/dynamodb/prepare.sh
@@ -0,0 +1,8 @@
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+curl -X POST http://localhost:8000/ \
+            -H "Accept-Encoding: identity" \
+            -H "X-Amz-Target: DynamoDB_20120810.CreateTable" \
+            -H "Content-Type: application/x-amz-json-1.0" \
+            -H "Authorization: AWS4-HMAC-SHA256 Credential=DUMMY/20240601/us/dynamodb/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=DUMMY" \
+            --data "@$SCRIPT_DIR/create_table_request.json"
diff --git a/integration_tests/dynamodb/query.sql b/integration_tests/dynamodb/query.sql
new file mode 100644
index 0000000000000..8ae514743ae97
--- /dev/null
+++ b/integration_tests/dynamodb/query.sql
@@ -0,0 +1,42 @@
+INSERT INTO
+    movies (year, title, description)
+VALUES
+    (2020, 'The Emoji Movie', 'a'),
+    (2019, 'Avengers: Endgame', 'b'),
+    (2018, 'Black Panther', 'c'),
+    (2017, 'Wonder Woman', 'd');
+
+INSERT INTO
+    movies (year, title, description)
+VALUES
+    (2023, 'Beautiful beauty', 'ABC DUMMY');
+
+FLUSH;
+
+DELETE FROM
+    movies
+WHERE
+    title = 'Beautiful beauty';
+
+DELETE FROM
+    movies
+WHERE
+    year = 2017;
+
+FLUSH;
+
+INSERT INTO
+    movies (year, title, description)
+VALUES
+    (2017, 'Beautiful beauty', 'ABC DUMMY');
+
+FLUSH;
+
+UPDATE
+    movies
+SET
+    description = 'ABC'
+WHERE
+    year = 2017;
+
+FLUSH;
diff --git a/integration_tests/dynamodb/sink_check.py b/integration_tests/dynamodb/sink_check.py
new file mode 100644
index 0000000000000..2bfb9b6fe1352
--- /dev/null
+++ b/integration_tests/dynamodb/sink_check.py
@@ -0,0 +1,48 @@
+import sys
+import subprocess
+import json
+
+curl = """
+curl -X POST http://localhost:8000/ \
+            -H "Accept-Encoding: identity" \
+            -H "X-Amz-Target: DynamoDB_20120810.Scan" \
+            -H "Content-Type: application/x-amz-json-1.0" \
+            -H "Authorization: AWS4-HMAC-SHA256 Credential=DUMMY/20240601/us/dynamodb/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=DUMMY" \
+            --data '{"TableName": "Movies"}'
+"""
+
+output = subprocess.Popen(
+    ["bash", "-c", curl,],
+    stdout=subprocess.PIPE,
+)
+msgs = json.loads(output.stdout.read())
+print(msgs)
+
+item_count = msgs["Count"]
+items = msgs["Items"]
+print("item count: ", item_count)
+print("items: ", items)
+
+output.stdout.close()
+output.wait()
+
+if item_count != 4:
+    print("Data check failed")
+    sys.exit(1)
+
+for item in items:
+    title = item["title"]["S"]
+    year = item["year"]["N"]
+    description = item["description"]["S"]
+
+    if title not in ["Black Panther", "Avengers: Endgame", "Beautiful beauty", "The Emoji Movie"]:
+        print(f"Data check failed: unexcept title: {title}")
+        sys.exit(1)
+
+    if int(year) not in range(2017, 2021):
+        print(f"Data check failed: unexcept year: {year}")
+        sys.exit(1)
+
+    if description not in ["a", "b", "c", "ABC"]:
+        print(f"Data check failed: unexcept description: {description}")
+        sys.exit(1)
diff --git a/integration_tests/elasticsearch-sink/docker-compose.yml b/integration_tests/elasticsearch-sink/docker-compose.yml
index 195e8d0070eb8..097de4beb5490 100644
--- a/integration_tests/elasticsearch-sink/docker-compose.yml
+++ b/integration_tests/elasticsearch-sink/docker-compose.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 services:
   elasticsearch7:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0
@@ -29,10 +28,10 @@ services:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -48,7 +47,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/feature-store/docker-compose.yml b/integration_tests/feature-store/docker-compose.yml
index b1767f8d04864..77de22a8c3522 100644
--- a/integration_tests/feature-store/docker-compose.yml
+++ b/integration_tests/feature-store/docker-compose.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 services:
   kafka:
     image: confluentinc/cp-kafka:7.1.0
@@ -73,10 +72,10 @@ services:
       [kafka,meta-node-0,frontend-node-0]
     volumes:
       - ".log:/opt/feature-store/.log"
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose-distributed.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose-distributed.yml
@@ -98,7 +97,7 @@ services:
       file: ../../docker/docker-compose-distributed.yml
       service: prometheus-0
 volumes:
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/http-sink/docker-compose.yml b/integration_tests/http-sink/docker-compose.yml
index 8fba5ff352dc0..9a7c42b1443e0 100644
--- a/integration_tests/http-sink/docker-compose.yml
+++ b/integration_tests/http-sink/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -24,7 +23,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/iceberg-cdc/docker-compose.yml b/integration_tests/iceberg-cdc/docker-compose.yml
index e483003907746..703571c2f10a5 100644
--- a/integration_tests/iceberg-cdc/docker-compose.yml
+++ b/integration_tests/iceberg-cdc/docker-compose.yml
@@ -5,10 +5,10 @@ services:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -127,7 +127,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/iceberg-sink/docker-compose.yml b/integration_tests/iceberg-sink/docker-compose.yml
index c7182b873d85b..84bda01b21ceb 100644
--- a/integration_tests/iceberg-sink/docker-compose.yml
+++ b/integration_tests/iceberg-sink/docker-compose.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 x-airflow-common:
   &airflow-common
   image: apache/airflow:2.6.2-python3.10
@@ -53,10 +52,10 @@ services:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -161,7 +160,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/iceberg-sink2/docker/hive/docker-compose.yml b/integration_tests/iceberg-sink2/docker/hive/docker-compose.yml
index 3314083c1077b..449d6043e6e95 100644
--- a/integration_tests/iceberg-sink2/docker/hive/docker-compose.yml
+++ b/integration_tests/iceberg-sink2/docker/hive/docker-compose.yml
@@ -75,10 +75,10 @@ services:
     networks:
       iceberg_net:
 
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
     networks:
       iceberg_net:
 
@@ -106,7 +106,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   minio-0:
     external: false
diff --git a/integration_tests/iceberg-sink2/docker/jdbc/docker-compose.yml b/integration_tests/iceberg-sink2/docker/jdbc/docker-compose.yml
index 3f2bb75479563..714fcdf6e0b95 100644
--- a/integration_tests/iceberg-sink2/docker/jdbc/docker-compose.yml
+++ b/integration_tests/iceberg-sink2/docker/jdbc/docker-compose.yml
@@ -76,17 +76,17 @@ services:
     networks:
       iceberg_net:
 
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
     networks:
       iceberg_net:
 
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   minio-0:
     external: false
diff --git a/integration_tests/iceberg-sink2/docker/rest/docker-compose.yml b/integration_tests/iceberg-sink2/docker/rest/docker-compose.yml
index 025db74c23cae..ee3e1da9b62da 100644
--- a/integration_tests/iceberg-sink2/docker/rest/docker-compose.yml
+++ b/integration_tests/iceberg-sink2/docker/rest/docker-compose.yml
@@ -80,17 +80,17 @@ services:
     networks:
       iceberg_net:
 
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
     networks:
       iceberg_net:
 
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   minio-0:
     external: false
diff --git a/integration_tests/iceberg-sink2/docker/storage/docker-compose.yml b/integration_tests/iceberg-sink2/docker/storage/docker-compose.yml
index 8d6d6f72d53c5..4e0ec11eff500 100644
--- a/integration_tests/iceberg-sink2/docker/storage/docker-compose.yml
+++ b/integration_tests/iceberg-sink2/docker/storage/docker-compose.yml
@@ -56,17 +56,17 @@ services:
     networks:
       iceberg_net:
 
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
     networks:
       iceberg_net:
 
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   minio-0:
     external: false
diff --git a/integration_tests/iceberg-source/docker/hive/docker-compose.yml b/integration_tests/iceberg-source/docker/hive/docker-compose.yml
index 3314083c1077b..449d6043e6e95 100644
--- a/integration_tests/iceberg-source/docker/hive/docker-compose.yml
+++ b/integration_tests/iceberg-source/docker/hive/docker-compose.yml
@@ -75,10 +75,10 @@ services:
     networks:
       iceberg_net:
 
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
     networks:
       iceberg_net:
 
@@ -106,7 +106,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   minio-0:
     external: false
diff --git a/integration_tests/iceberg-source/docker/jdbc/docker-compose.yml b/integration_tests/iceberg-source/docker/jdbc/docker-compose.yml
index 3f2bb75479563..714fcdf6e0b95 100644
--- a/integration_tests/iceberg-source/docker/jdbc/docker-compose.yml
+++ b/integration_tests/iceberg-source/docker/jdbc/docker-compose.yml
@@ -76,17 +76,17 @@ services:
     networks:
       iceberg_net:
 
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
     networks:
       iceberg_net:
 
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   minio-0:
     external: false
diff --git a/integration_tests/iceberg-source/docker/rest/docker-compose.yml b/integration_tests/iceberg-source/docker/rest/docker-compose.yml
index 025db74c23cae..ee3e1da9b62da 100644
--- a/integration_tests/iceberg-source/docker/rest/docker-compose.yml
+++ b/integration_tests/iceberg-source/docker/rest/docker-compose.yml
@@ -80,17 +80,17 @@ services:
     networks:
       iceberg_net:
 
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
     networks:
       iceberg_net:
 
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   minio-0:
     external: false
diff --git a/integration_tests/iceberg-source/docker/storage/docker-compose.yml b/integration_tests/iceberg-source/docker/storage/docker-compose.yml
index 8d6d6f72d53c5..4e0ec11eff500 100644
--- a/integration_tests/iceberg-source/docker/storage/docker-compose.yml
+++ b/integration_tests/iceberg-source/docker/storage/docker-compose.yml
@@ -56,17 +56,17 @@ services:
     networks:
       iceberg_net:
 
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
     networks:
       iceberg_net:
 
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   minio-0:
     external: false
diff --git a/integration_tests/kafka-cdc-sink/docker-compose.yml b/integration_tests/kafka-cdc-sink/docker-compose.yml
index c8c31d7762cf6..1cebe9b73f284 100644
--- a/integration_tests/kafka-cdc-sink/docker-compose.yml
+++ b/integration_tests/kafka-cdc-sink/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -133,7 +132,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/kafka-cdc/docker-compose.yml b/integration_tests/kafka-cdc/docker-compose.yml
index d62fa2acd9df3..6eaa5b5ead7ab 100644
--- a/integration_tests/kafka-cdc/docker-compose.yml
+++ b/integration_tests/kafka-cdc/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -38,7 +37,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/kinesis-s3-source/data_check b/integration_tests/kinesis-s3-source/data_check
index 9708f5cb617c3..b95837955a510 100644
--- a/integration_tests/kinesis-s3-source/data_check
+++ b/integration_tests/kinesis-s3-source/data_check
@@ -1 +1 @@
-ad_impression,ad_click,ad_ctr,ad_ctr_5min
\ No newline at end of file
+ad_ctr,ad_ctr_5min
\ No newline at end of file
diff --git a/integration_tests/kinesis-s3-source/docker-compose.yml b/integration_tests/kinesis-s3-source/docker-compose.yml
index 9108537309bd7..74dabde96f7ba 100644
--- a/integration_tests/kinesis-s3-source/docker-compose.yml
+++ b/integration_tests/kinesis-s3-source/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -50,7 +49,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/livestream/docker-compose.yml b/integration_tests/livestream/docker-compose.yml
index 1f3e0736a4def..e263b704bc90d 100644
--- a/integration_tests/livestream/docker-compose.yml
+++ b/integration_tests/livestream/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -37,7 +36,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/mindsdb/docker-compose.yml b/integration_tests/mindsdb/docker-compose.yml
index c6eeec5c75ee6..0cd82b10a6529 100644
--- a/integration_tests/mindsdb/docker-compose.yml
+++ b/integration_tests/mindsdb/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -52,7 +51,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/mongodb-cdc/docker-compose.yaml b/integration_tests/mongodb-cdc/docker-compose.yaml
index 60d477945b38b..de09a204d991b 100644
--- a/integration_tests/mongodb-cdc/docker-compose.yaml
+++ b/integration_tests/mongodb-cdc/docker-compose.yaml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -37,7 +36,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/mongodb/docker-compose.yaml b/integration_tests/mongodb/docker-compose.yaml
index 59ac89215ec14..a2855c200e6b0 100644
--- a/integration_tests/mongodb/docker-compose.yaml
+++ b/integration_tests/mongodb/docker-compose.yaml
@@ -1,4 +1,3 @@
-version: "3"
 services:
   mongodb:
     image: mongo:4.4
diff --git a/integration_tests/mqtt/docker-compose.yml b/integration_tests/mqtt/docker-compose.yml
index 9db7e7c04f8fc..b91ddd482509c 100644
--- a/integration_tests/mqtt/docker-compose.yml
+++ b/integration_tests/mqtt/docker-compose.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
@@ -13,10 +12,10 @@ services:
     - echo "running command"; printf 'allow_anonymous true\nlistener 1883 0.0.0.0' > /mosquitto/config/mosquitto.conf; echo "starting service..."; cat /mosquitto/config/mosquitto.conf;/docker-entrypoint.sh;/usr/sbin/mosquitto -c /mosquitto/config/mosquitto.conf
     ports:
       - 1883:1883
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -36,7 +35,7 @@ services:
 volumes:
   compute-node-0:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/mysql-cdc/docker-compose.yml b/integration_tests/mysql-cdc/docker-compose.yml
index 5cd3003669d7b..b2779c42c05b6 100644
--- a/integration_tests/mysql-cdc/docker-compose.yml
+++ b/integration_tests/mysql-cdc/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -70,7 +69,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/mysql-sink/docker-compose.yml b/integration_tests/mysql-sink/docker-compose.yml
index 97d3d78ce4cb0..8f8c4f9aa4336 100644
--- a/integration_tests/mysql-sink/docker-compose.yml
+++ b/integration_tests/mysql-sink/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -41,7 +40,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/nats/docker-compose.yml b/integration_tests/nats/docker-compose.yml
index 2010ec2d8b4ed..930f1a719fd7f 100644
--- a/integration_tests/nats/docker-compose.yml
+++ b/integration_tests/nats/docker-compose.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
@@ -10,10 +9,10 @@ services:
     ports:
       - "4222:4222"
     command: -js
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -50,7 +49,7 @@ services:
 volumes:
   compute-node-0:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/pinot-sink/docker-compose.yml b/integration_tests/pinot-sink/docker-compose.yml
index 222c1d7e39735..c7d08dcc005e9 100644
--- a/integration_tests/pinot-sink/docker-compose.yml
+++ b/integration_tests/pinot-sink/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -84,7 +83,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/postgres-cdc/docker-compose.yml b/integration_tests/postgres-cdc/docker-compose.yml
index 1898d20a8c9ae..333ee2f4080c3 100644
--- a/integration_tests/postgres-cdc/docker-compose.yml
+++ b/integration_tests/postgres-cdc/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -30,7 +29,7 @@ services:
       - POSTGRES_PASSWORD=123456
       - POSTGRES_DB=mydb
     ports:
-      - 8432:5432
+      - 5432:5432
     healthcheck:
       test: [ "CMD-SHELL", "pg_isready --username=myuser --dbname=mydb" ]
       interval: 5s
@@ -78,7 +77,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/postgres-sink/docker-compose.yml b/integration_tests/postgres-sink/docker-compose.yml
index e443965c2e5be..6f5a16db64c24 100644
--- a/integration_tests/postgres-sink/docker-compose.yml
+++ b/integration_tests/postgres-sink/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -44,7 +43,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/presto-trino/docker-compose.yml b/integration_tests/presto-trino/docker-compose.yml
index b12785139ebad..5de9dc34eb78a 100644
--- a/integration_tests/presto-trino/docker-compose.yml
+++ b/integration_tests/presto-trino/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -48,7 +47,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/prometheus/docker-compose.yml b/integration_tests/prometheus/docker-compose.yml
index 0beb75839d4cd..cd840807ea1ac 100644
--- a/integration_tests/prometheus/docker-compose.yml
+++ b/integration_tests/prometheus/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -67,7 +66,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/prometheus/prometheus.yaml b/integration_tests/prometheus/prometheus.yaml
index c70ddd434064e..7d955a835b32d 100644
--- a/integration_tests/prometheus/prometheus.yaml
+++ b/integration_tests/prometheus/prometheus.yaml
@@ -27,7 +27,7 @@ scrape_configs:
 
   - job_name: etcd
     static_configs:
-      - targets: ["etcd-0:2379"]
+      - targets: ["postgres-0:2379"]
 
   - job_name: redpanda
     static_configs:
diff --git a/integration_tests/pubsub/create_sink.sql b/integration_tests/pubsub/create_sink.sql
new file mode 100644
index 0000000000000..82016040c3cbf
--- /dev/null
+++ b/integration_tests/pubsub/create_sink.sql
@@ -0,0 +1,14 @@
+CREATE SINK pubsub_sink
+FROM
+  personnel
+WITH
+(
+    connector = 'google_pubsub',
+    pubsub.endpoint = 'pubsub-emulator:8900',
+    pubsub.emulator_host = 'pubsub-emulator:8900',
+    pubsub.project_id = 'demo',
+    pubsub.topic = 'test',
+) FORMAT PLAIN ENCODE JSON (
+    force_append_only='true',
+);
+
diff --git a/integration_tests/pubsub/create_source.sql b/integration_tests/pubsub/create_source.sql
new file mode 100644
index 0000000000000..5856abfabbf47
--- /dev/null
+++ b/integration_tests/pubsub/create_source.sql
@@ -0,0 +1 @@
+CREATE TABLE IF NOT EXISTS personnel (id integer, name varchar);
diff --git a/integration_tests/pubsub/data_check b/integration_tests/pubsub/data_check
new file mode 100644
index 0000000000000..0528e08f7601a
--- /dev/null
+++ b/integration_tests/pubsub/data_check
@@ -0,0 +1 @@
+personnel
diff --git a/integration_tests/pubsub/docker-compose.yml b/integration_tests/pubsub/docker-compose.yml
new file mode 100644
index 0000000000000..e04def6dd88cd
--- /dev/null
+++ b/integration_tests/pubsub/docker-compose.yml
@@ -0,0 +1,47 @@
+---
+version: "3"
+services:
+  pubsub-emulator:
+    image: google/cloud-sdk:latest
+    command: >
+      gcloud beta emulators pubsub start --project=demo --host-port=0.0.0.0:8900
+    ports:
+      - "8900:8900"
+  risingwave-standalone:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: risingwave-standalone
+  postgres-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: postgres-0
+  grafana-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: grafana-0
+  minio-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: minio-0
+  prometheus-0:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: prometheus-0
+  message_queue:
+    extends:
+      file: ../../docker/docker-compose.yml
+      service: message_queue
+volumes:
+  risingwave-standalone:
+    external: false
+  postgres-0:
+    external: false
+  grafana-0:
+    external: false
+  minio-0:
+    external: false
+  prometheus-0:
+    external: false
+  message_queue:
+    external: false
+name: risingwave-compose
diff --git a/integration_tests/pubsub/prepare.sh b/integration_tests/pubsub/prepare.sh
new file mode 100644
index 0000000000000..eabf514e151ca
--- /dev/null
+++ b/integration_tests/pubsub/prepare.sh
@@ -0,0 +1,9 @@
+export PUBSUB_EMULATOR_HOST=localhost:8900
+
+curl -X PUT http://localhost:8900/v1/projects/demo/topics/test
+curl -X PUT http://localhost:8900/v1/projects/demo/subscriptions/sub \
+    -H 'content-type: application/json' \
+    --data '{"topic":"projects/demo/topics/test"}'
+
+curl -X GET http://localhost:8900/v1/projects/demo/topics
+curl -X GET http://localhost:8900/v1/projects/demo/subscriptions
diff --git a/integration_tests/pubsub/query.sql b/integration_tests/pubsub/query.sql
new file mode 100644
index 0000000000000..124ce9133634e
--- /dev/null
+++ b/integration_tests/pubsub/query.sql
@@ -0,0 +1,13 @@
+INSERT INTO
+  personnel
+VALUES
+  (1, 'Alice'),
+  (2, 'Bob'),
+  (3, 'Tom'),
+  (4, 'Jerry'),
+  (5, 'Araminta'),
+  (6, 'Clover'),
+  (7, 'Posey'),
+  (8, 'Waverly');
+
+FLUSH;
diff --git a/integration_tests/pubsub/sink_check.py b/integration_tests/pubsub/sink_check.py
new file mode 100644
index 0000000000000..3957d99817542
--- /dev/null
+++ b/integration_tests/pubsub/sink_check.py
@@ -0,0 +1,26 @@
+import sys
+import subprocess
+import json
+
+curl = """
+PUBSUB_EMULATOR_HOST=localhost:8900 curl -s -X POST 'http://localhost:8900/v1/projects/demo/subscriptions/sub:pull' \
+    -H 'content-type: application/json' \
+    --data '{"maxMessages":10}'
+"""
+
+output = subprocess.Popen(
+    ["bash", "-c", curl,],
+    stdout=subprocess.PIPE,
+)
+msgs = json.loads(output.stdout.read())
+print(msgs)
+
+msg_count = len(msgs["receivedMessages"])
+print("msg count", msg_count)
+
+output.stdout.close()
+output.wait()
+
+if msg_count != 8:
+    print("Data check failed")
+    sys.exit(1)
diff --git a/integration_tests/redis-sink/docker-compose.yml b/integration_tests/redis-sink/docker-compose.yml
index 0fd33048a29bf..8f3c3eb9cdd85 100644
--- a/integration_tests/redis-sink/docker-compose.yml
+++ b/integration_tests/redis-sink/docker-compose.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 services:
   redis:
     image: 'redis:latest'
@@ -16,10 +15,10 @@ services:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -39,7 +38,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/schema-registry/docker-compose.yml b/integration_tests/schema-registry/docker-compose.yml
index 3020a96790fbf..2209fb7e20aee 100644
--- a/integration_tests/schema-registry/docker-compose.yml
+++ b/integration_tests/schema-registry/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -62,7 +61,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/starrocks-sink/docker-compose.yml b/integration_tests/starrocks-sink/docker-compose.yml
index 81ef7c277dad0..e3a06cc33587a 100644
--- a/integration_tests/starrocks-sink/docker-compose.yml
+++ b/integration_tests/starrocks-sink/docker-compose.yml
@@ -1,5 +1,4 @@
 ---
-version: "3"
 services:
   starrocks-fe:
     image: starrocks/fe-ubuntu:3.1.7
@@ -37,10 +36,10 @@ services:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -62,7 +61,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/superset/docker-compose.yml b/integration_tests/superset/docker-compose.yml
index 271e79755aab3..3d7b9ed2494ca 100644
--- a/integration_tests/superset/docker-compose.yml
+++ b/integration_tests/superset/docker-compose.yml
@@ -9,16 +9,15 @@ x-superset-volumes:
   - ./docker:/app/docker
   - superset_home:/app/superset_home
 
-version: "3.7"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -112,7 +111,7 @@ volumes:
     external: false
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/tidb-cdc-sink/docker-compose.yml b/integration_tests/tidb-cdc-sink/docker-compose.yml
index 9177936d47ab8..f1b4cb0ebdd7b 100644
--- a/integration_tests/tidb-cdc-sink/docker-compose.yml
+++ b/integration_tests/tidb-cdc-sink/docker-compose.yml
@@ -5,10 +5,10 @@ services:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -204,7 +204,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/twitter-pulsar/docker-compose.yml b/integration_tests/twitter-pulsar/docker-compose.yml
index 8061425d48d38..a3e91c9f8751a 100644
--- a/integration_tests/twitter-pulsar/docker-compose.yml
+++ b/integration_tests/twitter-pulsar/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -42,7 +41,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/twitter/docker-compose.yml b/integration_tests/twitter/docker-compose.yml
index e5637b7f2664f..e59e71b3839ce 100644
--- a/integration_tests/twitter/docker-compose.yml
+++ b/integration_tests/twitter/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -37,7 +36,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/integration_tests/upsert-avro/docker-compose.yml b/integration_tests/upsert-avro/docker-compose.yml
index 695c8f8fcb043..9176ca053ba4e 100644
--- a/integration_tests/upsert-avro/docker-compose.yml
+++ b/integration_tests/upsert-avro/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose.yml
@@ -43,7 +42,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
@@ -53,4 +52,4 @@ volumes:
     external: false
   message_queue:
     external: false
-name: risingwave-compose
\ No newline at end of file
+name: risingwave-compose
diff --git a/integration_tests/vector/docker-compose.yml b/integration_tests/vector/docker-compose.yml
index 2179cd66542c4..13101925ac4a3 100644
--- a/integration_tests/vector/docker-compose.yml
+++ b/integration_tests/vector/docker-compose.yml
@@ -1,14 +1,13 @@
 ---
-version: "3"
 services:
   risingwave-standalone:
     extends:
       file: ../../docker/docker-compose.yml
       service: risingwave-standalone
-  etcd-0:
+  postgres-0:
     extends:
       file: ../../docker/docker-compose-distributed.yml
-      service: etcd-0
+      service: postgres-0
   grafana-0:
     extends:
       file: ../../docker/docker-compose-distributed.yml
@@ -34,7 +33,7 @@ services:
 volumes:
   risingwave-standalone:
     external: false
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/Monitor.java b/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/Monitor.java
new file mode 100644
index 0000000000000..5b89cfeb6e5df
--- /dev/null
+++ b/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/Monitor.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2024 RisingWave Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.risingwave.connector.api;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+
+public class Monitor {
+
+    public static String dumpStackTrace() {
+        StringBuilder builder = new StringBuilder();
+        ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean();
+        for (ThreadInfo ti : threadMxBean.dumpAllThreads(true, true)) {
+            builder.append(ti.toString());
+        }
+        return builder.toString();
+    }
+}
diff --git a/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/TableSchema.java b/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/TableSchema.java
index ded49e003d19a..c7e9f7035f1e7 100644
--- a/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/TableSchema.java
+++ b/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/TableSchema.java
@@ -20,12 +20,18 @@
 import com.risingwave.proto.Data.DataType.TypeName;
 import com.risingwave.proto.PlanCommon;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TableSchema {
+
+    static final Logger LOG = LoggerFactory.getLogger(TableSchema.class);
+
     private final List<String> columnNames;
     private final Map<String, TypeName> columns;
     private final Map<String, Integer> columnIndices;
@@ -80,16 +86,20 @@ public Object getFromRow(String columnName, SinkRow row) {
     }
 
     public static TableSchema fromProto(ConnectorServiceProto.TableSchema tableSchema) {
-        return new TableSchema(
-                tableSchema.getColumnsList().stream()
-                        .map(PlanCommon.ColumnDesc::getName)
-                        .collect(Collectors.toList()),
-                tableSchema.getColumnsList().stream()
-                        .map(PlanCommon.ColumnDesc::getColumnType)
-                        .collect(Collectors.toList()),
-                tableSchema.getPkIndicesList().stream()
-                        .map(i -> tableSchema.getColumns(i).getName())
-                        .collect(Collectors.toList()));
+        // filter out additional columns
+        var instance =
+                new TableSchema(
+                        tableSchema.getColumnsList().stream()
+                                .map(PlanCommon.ColumnDesc::getName)
+                                .collect(Collectors.toList()),
+                        tableSchema.getColumnsList().stream()
+                                .map(PlanCommon.ColumnDesc::getColumnType)
+                                .collect(Collectors.toList()),
+                        tableSchema.getPkIndicesList().stream()
+                                .map(i -> tableSchema.getColumns(i).getName())
+                                .collect(Collectors.toList()));
+        LOG.info("table column names: {}", Arrays.toString(instance.getColumnNames()));
+        return instance;
     }
 
     public List<String> getPrimaryKeys() {
diff --git a/java/connector-node/python-client/.gitignore b/java/connector-node/python-client/.gitignore
index 600d2d33badf4..536c32383d754 100644
--- a/java/connector-node/python-client/.gitignore
+++ b/java/connector-node/python-client/.gitignore
@@ -1 +1,2 @@
-.vscode
\ No newline at end of file
+.vscode
+sink-client-venv/
diff --git a/java/connector-node/python-client/integration_tests.py b/java/connector-node/python-client/integration_tests.py
index b16b5eaf34ad4..909859afc218a 100644
--- a/java/connector-node/python-client/integration_tests.py
+++ b/java/connector-node/python-client/integration_tests.py
@@ -117,7 +117,7 @@ def load_stream_chunk_payload(input_file):
     return payloads
 
 
-def test_sink(prop, format, payload_input, table_schema, is_coordinated=False):
+def test_sink(prop, payload_input, table_schema, is_coordinated=False):
     sink_param = connector_service_pb2.SinkParam(
         sink_id=0,
         properties=prop,
@@ -128,7 +128,6 @@ def test_sink(prop, format, payload_input, table_schema, is_coordinated=False):
     request_list = [
         connector_service_pb2.SinkWriterStreamRequest(
             start=connector_service_pb2.SinkWriterStreamRequest.StartSink(
-                format=format,
                 sink_param=sink_param,
             )
         )
@@ -291,9 +290,6 @@ def test_stream_chunk_data_format(param):
     parser.add_argument(
         "--deltalake_sink", action="store_true", help="run deltalake sink test"
     )
-    parser.add_argument(
-        "--input_file", default="./data/sink_input.json", help="input data to run tests"
-    )
     parser.add_argument(
         "--input_binary_file",
         default="./data/sink_input",
@@ -302,29 +298,18 @@ def test_stream_chunk_data_format(param):
     parser.add_argument(
         "--es_sink", action="store_true", help="run elasticsearch sink test"
     )
-    parser.add_argument(
-        "--data_format_use_json", default=True, help="choose json or streamchunk"
-    )
     args = parser.parse_args()
-    use_json = args.data_format_use_json == True or args.data_format_use_json == "True"
-    if use_json:
-        payload = load_json_payload(args.input_file)
-        format = connector_service_pb2.SinkPayloadFormat.JSON
-    else:
-        payload = load_stream_chunk_payload(args.input_binary_file)
-        format = connector_service_pb2.SinkPayloadFormat.STREAM_CHUNK
+    payload = load_stream_chunk_payload(args.input_binary_file)
 
     # stream chunk format
     if args.stream_chunk_format_test:
         param = {
-            "format": format,
             "payload_input": payload,
             "table_schema": make_mock_schema_stream_chunk(),
         }
         test_stream_chunk_data_format(param)
 
     param = {
-        "format": format,
         "payload_input": payload,
         "table_schema": make_mock_schema(),
     }
@@ -337,7 +322,5 @@ def test_stream_chunk_data_format(param):
         test_deltalake_sink(param)
     if args.es_sink:
         test_elasticsearch_sink(param)
-
-    # json format
     if args.upsert_iceberg_sink:
         test_upsert_iceberg_sink(param)
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JsonDeserializer.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JsonDeserializer.java
deleted file mode 100644
index c941b09efe95c..0000000000000
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JsonDeserializer.java
+++ /dev/null
@@ -1,246 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.connector;
-
-import static io.grpc.Status.INVALID_ARGUMENT;
-
-import com.google.gson.Gson;
-import com.risingwave.connector.api.TableSchema;
-import com.risingwave.connector.api.sink.*;
-import com.risingwave.proto.ConnectorServiceProto;
-import com.risingwave.proto.ConnectorServiceProto.SinkWriterStreamRequest.WriteBatch.JsonPayload;
-import com.risingwave.proto.Data;
-import java.math.BigDecimal;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.LocalTime;
-import java.time.OffsetDateTime;
-import java.util.Base64;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-public class JsonDeserializer implements Deserializer {
-    private final TableSchema tableSchema;
-
-    public JsonDeserializer(TableSchema tableSchema) {
-        this.tableSchema = tableSchema;
-    }
-
-    // Encoding here should be consistent with `datum_to_json_object()` in
-    // src/connector/src/sink/mod.rs
-    @Override
-    public CloseableIterable<SinkRow> deserialize(
-            ConnectorServiceProto.SinkWriterStreamRequest.WriteBatch writeBatch) {
-        if (!writeBatch.hasJsonPayload()) {
-            throw INVALID_ARGUMENT
-                    .withDescription("expected JsonPayload, got " + writeBatch.getPayloadCase())
-                    .asRuntimeException();
-        }
-        JsonPayload jsonPayload = writeBatch.getJsonPayload();
-        return new TrivialCloseIterable<>(
-                jsonPayload.getRowOpsList().stream()
-                        .map(
-                                rowOp -> {
-                                    Map columnValues =
-                                            new Gson().fromJson(rowOp.getLine(), Map.class);
-                                    Object[] values = new Object[columnValues.size()];
-                                    for (String columnName : tableSchema.getColumnNames()) {
-                                        if (!columnValues.containsKey(columnName)) {
-                                            throw INVALID_ARGUMENT
-                                                    .withDescription(
-                                                            "column "
-                                                                    + columnName
-                                                                    + " not found in json")
-                                                    .asRuntimeException();
-                                        }
-                                        Data.DataType.TypeName typeName =
-                                                tableSchema.getColumnType(columnName);
-                                        values[tableSchema.getColumnIndex(columnName)] =
-                                                validateJsonDataTypes(
-                                                        typeName, columnValues.get(columnName));
-                                    }
-                                    return (SinkRow) new ArraySinkRow(rowOp.getOpType(), values);
-                                })
-                        .collect(Collectors.toList()));
-    }
-
-    private static Long castLong(Object value) {
-        if (value instanceof Integer) {
-            return ((Integer) value).longValue();
-        } else if (value instanceof Double) {
-            double d = (Double) value;
-            if (d % 1.0 != 0.0) {
-
-                throw io.grpc.Status.INVALID_ARGUMENT
-                        .withDescription(
-                                "unable to cast into long from non-integer double value: " + d)
-                        .asRuntimeException();
-            }
-            return ((Double) value).longValue();
-        } else if (value instanceof Long) {
-            return (Long) value;
-        } else if (value instanceof Short) {
-            return ((Short) value).longValue();
-        } else if (value instanceof Float) {
-            double f = (Float) value;
-            if (f % 1.0 != 0.0) {
-
-                throw io.grpc.Status.INVALID_ARGUMENT
-                        .withDescription(
-                                "unable to cast into long from non-integer float value: " + f)
-                        .asRuntimeException();
-            }
-            return ((Float) value).longValue();
-        } else {
-            throw io.grpc.Status.INVALID_ARGUMENT
-                    .withDescription("unable to cast into long from " + value.getClass())
-                    .asRuntimeException();
-        }
-    }
-
-    private static Double castDouble(Object value) {
-        if (value instanceof Double) {
-            return (Double) value;
-        } else if (value instanceof Float) {
-            return ((Float) value).doubleValue();
-        } else {
-            throw io.grpc.Status.INVALID_ARGUMENT
-                    .withDescription("unable to cast into double from " + value.getClass())
-                    .asRuntimeException();
-        }
-    }
-
-    private static BigDecimal castDecimal(Object value) {
-        if (value instanceof String) {
-            // FIXME(eric): See `datum_to_json_object()` in src/connector/src/sink/mod.rs
-            return new BigDecimal((String) value);
-        } else if (value instanceof BigDecimal) {
-            return (BigDecimal) value;
-        } else {
-            throw io.grpc.Status.INVALID_ARGUMENT
-                    .withDescription("unable to cast into double from " + value.getClass())
-                    .asRuntimeException();
-        }
-    }
-
-    private static LocalTime castTime(Object value) {
-        try {
-            Long milli = castLong(value);
-            return LocalTime.ofNanoOfDay(milli * 1_000_000L);
-        } catch (RuntimeException e) {
-            throw io.grpc.Status.INVALID_ARGUMENT
-                    .withDescription("unable to cast into time from " + value.getClass())
-                    .asRuntimeException();
-        }
-    }
-
-    private static LocalDate castDate(Object value) {
-        try {
-            Long days = castLong(value);
-            return LocalDate.ofEpochDay(days);
-        } catch (RuntimeException e) {
-            throw io.grpc.Status.INVALID_ARGUMENT
-                    .withDescription("unable to cast into date from " + value.getClass())
-                    .asRuntimeException();
-        }
-    }
-
-    private static Object validateJsonDataTypes(Data.DataType.TypeName typeName, Object value) {
-        // value might be null
-        if (value == null) {
-            return null;
-        }
-        switch (typeName) {
-            case INT16:
-                return castLong(value).shortValue();
-            case INT32:
-                return castLong(value).intValue();
-            case INT64:
-                return castLong(value);
-            case VARCHAR:
-                if (!(value instanceof String)) {
-                    throw io.grpc.Status.INVALID_ARGUMENT
-                            .withDescription("Expected string, got " + value.getClass())
-                            .asRuntimeException();
-                }
-                return value;
-            case DOUBLE:
-                return castDouble(value);
-            case FLOAT:
-                return castDouble(value).floatValue();
-            case DECIMAL:
-                return castDecimal(value);
-            case BOOLEAN:
-                if (!(value instanceof Boolean)) {
-                    throw io.grpc.Status.INVALID_ARGUMENT
-                            .withDescription("Expected boolean, got " + value.getClass())
-                            .asRuntimeException();
-                }
-                return value;
-            case TIMESTAMP:
-                if (!(value instanceof String)) {
-                    throw io.grpc.Status.INVALID_ARGUMENT
-                            .withDescription(
-                                    "Expected timestamp in string, got " + value.getClass())
-                            .asRuntimeException();
-                }
-                return LocalDateTime.parse((String) value);
-            case TIMESTAMPTZ:
-                if (!(value instanceof String)) {
-                    throw io.grpc.Status.INVALID_ARGUMENT
-                            .withDescription(
-                                    "Expected timestamptz in string, got " + value.getClass())
-                            .asRuntimeException();
-                }
-                return OffsetDateTime.parse((String) value);
-            case TIME:
-                return castTime(value);
-            case DATE:
-                return castDate(value);
-            case INTERVAL:
-                if (!(value instanceof String)) {
-                    throw io.grpc.Status.INVALID_ARGUMENT
-                            .withDescription("Expected interval, got " + value.getClass())
-                            .asRuntimeException();
-                }
-                return value;
-            case JSONB:
-                if (!(value instanceof String)) {
-                    throw io.grpc.Status.INVALID_ARGUMENT
-                            .withDescription("Expected jsonb, got " + value.getClass())
-                            .asRuntimeException();
-                }
-                return value;
-            case BYTEA:
-                if (!(value instanceof String)) {
-                    throw io.grpc.Status.INVALID_ARGUMENT
-                            .withDescription("Expected bytea, got " + value.getClass())
-                            .asRuntimeException();
-                }
-                return Base64.getDecoder().decode((String) value);
-            case LIST:
-                if (!(value instanceof java.util.ArrayList<?>)) {
-                    throw io.grpc.Status.INVALID_ARGUMENT
-                            .withDescription("Expected list, got " + value.getClass())
-                            .asRuntimeException();
-                }
-                return ((java.util.ArrayList<?>) value).toArray();
-            default:
-                throw io.grpc.Status.INVALID_ARGUMENT
-                        .withDescription("unsupported type " + typeName)
-                        .asRuntimeException();
-        }
-    }
-}
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkWriterStreamObserver.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkWriterStreamObserver.java
index cd61da38d6cb5..53dfe326fbd9d 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkWriterStreamObserver.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/SinkWriterStreamObserver.java
@@ -206,19 +206,7 @@ private void bindSink(ConnectorServiceProto.SinkWriterStreamRequest.StartSink st
         String connectorName = getConnectorName(sinkParam);
         SinkFactory sinkFactory = SinkUtils.getSinkFactory(connectorName);
         sink = sinkFactory.createWriter(tableSchema, sinkParam.getPropertiesMap());
-        switch (startSink.getFormat()) {
-            case FORMAT_UNSPECIFIED:
-            case UNRECOGNIZED:
-                throw INVALID_ARGUMENT
-                        .withDescription("should specify payload format in request")
-                        .asRuntimeException();
-            case JSON:
-                deserializer = new JsonDeserializer(tableSchema);
-                break;
-            case STREAM_CHUNK:
-                deserializer = new StreamChunkDeserializer(tableSchema);
-                break;
-        }
+        deserializer = new StreamChunkDeserializer(tableSchema);
         this.connectorName = connectorName.toUpperCase();
         ConnectorNodeMetrics.incActiveSinkConnections(connectorName, "node1");
     }
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java
index 33808447caad9..a5804974fb29c 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java
@@ -31,8 +31,8 @@
 public class DbzConnectorConfig {
     private static final Logger LOG = LoggerFactory.getLogger(DbzConnectorConfig.class);
 
-    public static final String WAIT_FOR_STREAMING_START_BEFORE_EXIT_SECS =
-            "cdc.source.wait.streaming.before.exit.seconds";
+    private static final String WAIT_FOR_STREAMING_START_TIMEOUT_SECS =
+            "cdc.source.wait.streaming.start.timeout";
 
     /* Common configs */
     public static final String HOST = "hostname";
@@ -47,6 +47,7 @@ public class DbzConnectorConfig {
 
     /* MySQL configs */
     public static final String MYSQL_SERVER_ID = "server.id";
+    public static final String MYSQL_SSL_MODE = "ssl.mode";
 
     /* Postgres configs */
     public static final String PG_SLOT_NAME = "slot.name";
@@ -88,6 +89,7 @@ private static Map<String, String> extractDebeziumProperties(
     private final SourceTypeE sourceType;
     private final Properties resolvedDbzProps;
     private final boolean isBackfillSource;
+    private final int waitStreamingStartTimeout;
 
     public long getSourceId() {
         return sourceId;
@@ -105,6 +107,10 @@ public boolean isBackfillSource() {
         return isBackfillSource;
     }
 
+    public int getWaitStreamingStartTimeout() {
+        return waitStreamingStartTimeout;
+    }
+
     public DbzConnectorConfig(
             SourceTypeE source,
             long sourceId,
@@ -118,6 +124,9 @@ public DbzConnectorConfig(
         var isCdcBackfill =
                 null != userProps.get(SNAPSHOT_MODE_KEY)
                         && userProps.get(SNAPSHOT_MODE_KEY).equals(SNAPSHOT_MODE_BACKFILL);
+        var waitStreamingStartTimeout =
+                Integer.parseInt(
+                        userProps.getOrDefault(WAIT_FOR_STREAMING_START_TIMEOUT_SECS, "30"));
 
         LOG.info(
                 "DbzConnectorConfig: source={}, sourceId={}, startOffset={}, snapshotDone={}, isCdcBackfill={}, isCdcSourceJob={}",
@@ -231,8 +240,8 @@ public DbzConnectorConfig(
                         ConfigurableOffsetBackingStore.OFFSET_STATE_VALUE, startOffset);
             }
 
-            var mongodbUrl = userProps.get("mongodb.url");
-            var collection = userProps.get("collection.name");
+            var mongodbUrl = userProps.get(MongoDb.MONGO_URL);
+            var collection = userProps.get(MongoDb.MONGO_COLLECTION_NAME);
             var connectionStr = new ConnectionString(mongodbUrl);
             var connectorName =
                     String.format(
@@ -254,6 +263,7 @@ public DbzConnectorConfig(
         this.sourceType = source;
         this.resolvedDbzProps = dbzProps;
         this.isBackfillSource = isCdcBackfill;
+        this.waitStreamingStartTimeout = waitStreamingStartTimeout;
     }
 
     private Properties initiateDbConfig(String fileName, StringSubstitutor substitutor) {
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzSourceUtils.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzSourceUtils.java
index a4a16f010ad63..83c6d59fac921 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzSourceUtils.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzSourceUtils.java
@@ -103,31 +103,30 @@ private static String quotePostgres(String identifier) {
         return "\"" + identifier + "\"";
     }
 
-    public static boolean waitForStreamingRunning(SourceTypeE sourceType, String dbServerName) {
+    public static boolean waitForStreamingRunning(
+            SourceTypeE sourceType, String dbServerName, int waitStreamingStartTimeout) {
         // Wait for streaming source of source that supported backfill
         LOG.info("Waiting for streaming source of {} to start", dbServerName);
         if (sourceType == SourceTypeE.MYSQL) {
-            return waitForStreamingRunningInner("mysql", dbServerName);
+            return waitForStreamingRunningInner("mysql", dbServerName, waitStreamingStartTimeout);
         } else if (sourceType == SourceTypeE.POSTGRES) {
-            return waitForStreamingRunningInner("postgres", dbServerName);
+            return waitForStreamingRunningInner(
+                    "postgres", dbServerName, waitStreamingStartTimeout);
         } else {
             LOG.info("Unsupported backfill source, just return true for {}", dbServerName);
             return true;
         }
     }
 
-    private static boolean waitForStreamingRunningInner(String connector, String dbServerName) {
-        int timeoutSecs =
-                Integer.parseInt(
-                        System.getProperty(
-                                DbzConnectorConfig.WAIT_FOR_STREAMING_START_BEFORE_EXIT_SECS));
+    private static boolean waitForStreamingRunningInner(
+            String connector, String dbServerName, int waitStreamingStartTimeout) {
         int pollCount = 0;
         while (!isStreamingRunning(connector, dbServerName, "streaming")) {
-            if (pollCount > timeoutSecs) {
+            if (pollCount > waitStreamingStartTimeout) {
                 LOG.error(
                         "Debezium streaming source of {} failed to start in timeout {}",
                         dbServerName,
-                        timeoutSecs);
+                        waitStreamingStartTimeout);
                 return false;
             }
             try {
@@ -154,8 +153,9 @@ private static boolean isStreamingRunning(String connector, String server, Strin
                     mbeanServer.getAttribute(
                             getStreamingMetricsObjectName(connector, server, contextName),
                             "Connected");
-        } catch (JMException ex) {
-            LOG.warn("Failed to get streaming metrics", ex);
+        } catch (JMException _ex) {
+            // ignore the exception, as it is expected when the streaming source
+            // (aka. binlog client) is not ready
         }
         return false;
     }
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java
index c92b4dad540c9..a2f63a28bbd7c 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java
@@ -20,10 +20,7 @@
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 public class MySqlValidator extends DatabaseValidator implements AutoCloseable {
     private final Map<String, String> userProps;
@@ -51,9 +48,14 @@ public MySqlValidator(
         var dbName = userProps.get(DbzConnectorConfig.DB_NAME);
         var jdbcUrl = ValidatorUtils.getJdbcUrl(SourceTypeE.MYSQL, dbHost, dbPort, dbName);
 
-        var user = userProps.get(DbzConnectorConfig.USER);
-        var password = userProps.get(DbzConnectorConfig.PASSWORD);
-        this.jdbcConnection = DriverManager.getConnection(jdbcUrl, user, password);
+        var properties = new Properties();
+        properties.setProperty("user", userProps.get(DbzConnectorConfig.USER));
+        properties.setProperty("password", userProps.get(DbzConnectorConfig.PASSWORD));
+        properties.setProperty(
+                "sslMode", userProps.getOrDefault(DbzConnectorConfig.MYSQL_SSL_MODE, "DISABLED"));
+        properties.setProperty("allowPublicKeyRetrieval", "true");
+
+        this.jdbcConnection = DriverManager.getConnection(jdbcUrl, properties);
         this.isCdcSourceJob = isCdcSourceJob;
         this.isBackfillTable = isBackfillTable;
     }
@@ -193,10 +195,9 @@ private void validateTableSchema() throws SQLException {
                 var field = res.getString(1);
                 var dataType = res.getString(2);
                 var key = res.getString(3);
-                schema.put(field.toLowerCase(), dataType);
+                schema.put(field, dataType);
                 if (key.equalsIgnoreCase("PRI")) {
-                    // RisingWave always use lower case for column name
-                    pkFields.add(field.toLowerCase());
+                    pkFields.add(field);
                 }
             }
 
@@ -206,7 +207,7 @@ private void validateTableSchema() throws SQLException {
                 if (e.getKey().startsWith(ValidatorUtils.INTERNAL_COLUMN_PREFIX)) {
                     continue;
                 }
-                var dataType = schema.get(e.getKey().toLowerCase());
+                var dataType = schema.get(e.getKey());
                 if (dataType == null) {
                     throw ValidatorUtils.invalidArgument(
                             "Column '" + e.getKey() + "' not found in the upstream database");
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java
index d91e5f885d609..31b016146e000 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java
@@ -79,14 +79,19 @@ public PostgresValidator(
 
     @Override
     public void validateDbConfig() {
-        // TODO: check database server version
-        try (var stmt = jdbcConnection.createStatement()) {
-            // check whether wal has been enabled
-            var res = stmt.executeQuery(ValidatorUtils.getSql("postgres.wal"));
-            while (res.next()) {
-                if (!res.getString(1).equals("logical")) {
-                    throw ValidatorUtils.invalidArgument(
-                            "Postgres wal_level should be 'logical'.\nPlease modify the config and restart your Postgres server.");
+        try {
+            if (jdbcConnection.getMetaData().getDatabaseMajorVersion() > 16) {
+                throw ValidatorUtils.failedPrecondition("Postgres version should be less than 16.");
+            }
+
+            try (var stmt = jdbcConnection.createStatement()) {
+                // check whether wal has been enabled
+                var res = stmt.executeQuery(ValidatorUtils.getSql("postgres.wal"));
+                while (res.next()) {
+                    if (!res.getString(1).equals("logical")) {
+                        throw ValidatorUtils.invalidArgument(
+                                "Postgres wal_level should be 'logical'.\nPlease modify the config and restart your Postgres server.");
+                    }
                 }
             }
         } catch (SQLException e) {
@@ -180,8 +185,7 @@ private void validateTableSchema() throws SQLException {
             var pkFields = new HashSet<String>();
             while (res.next()) {
                 var name = res.getString(1);
-                // RisingWave always use lower case for column name
-                pkFields.add(name.toLowerCase());
+                pkFields.add(name);
             }
 
             if (!ValidatorUtils.isPrimaryKeyMatch(tableSchema, pkFields)) {
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngine.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngine.java
index b515ce8bd79b6..ed22fe36416ec 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngine.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngine.java
@@ -17,6 +17,7 @@
 import static io.debezium.config.CommonConnectorConfig.TOPIC_PREFIX;
 import static io.debezium.schema.AbstractTopicNamingStrategy.*;
 
+import com.risingwave.connector.api.source.SourceTypeE;
 import com.risingwave.proto.ConnectorServiceProto;
 import io.debezium.embedded.Connect;
 import io.debezium.engine.DebeziumEngine;
@@ -33,6 +34,7 @@ public class DbzCdcEngine implements Runnable {
 
     /** If config is not valid will throw exceptions */
     public DbzCdcEngine(
+            SourceTypeE connector,
             long sourceId,
             Properties config,
             DebeziumEngine.CompletionCallback completionCallback) {
@@ -41,6 +43,7 @@ public DbzCdcEngine(
         var transactionTopic = String.format("%s.%s", topicPrefix, DEFAULT_TRANSACTION_TOPIC);
         var consumer =
                 new DbzChangeEventConsumer(
+                        connector,
                         sourceId,
                         heartbeatTopicPrefix,
                         transactionTopic,
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngineRunner.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngineRunner.java
index b223d0dfba142..a64c11745cf91 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngineRunner.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngineRunner.java
@@ -14,7 +14,6 @@
 
 package com.risingwave.connector.source.core;
 
-import com.risingwave.connector.api.source.*;
 import com.risingwave.connector.source.common.DbzConnectorConfig;
 import com.risingwave.connector.source.common.DbzSourceUtils;
 import com.risingwave.java.binding.Binding;
@@ -43,6 +42,7 @@ public static DbzCdcEngineRunner newCdcEngineRunner(
             var sourceId = config.getSourceId();
             var engine =
                     new DbzCdcEngine(
+                            config.getSourceType(),
                             config.getSourceId(),
                             config.getResolvedDebeziumProps(),
                             (success, message, error) -> {
@@ -76,6 +76,7 @@ public static DbzCdcEngineRunner create(DbzConnectorConfig config, long channelP
             final DbzCdcEngineRunner finalRunner = runner;
             var engine =
                     new DbzCdcEngine(
+                            config.getSourceType(),
                             config.getSourceId(),
                             config.getResolvedDebeziumProps(),
                             (success, message, error) -> {
@@ -144,7 +145,9 @@ public boolean start() throws InterruptedException {
                             .getProperty(CommonConnectorConfig.TOPIC_PREFIX.name());
             startOk =
                     DbzSourceUtils.waitForStreamingRunning(
-                            config.getSourceType(), databaseServerName);
+                            config.getSourceType(),
+                            databaseServerName,
+                            config.getWaitStreamingStartTimeout());
         }
 
         running.set(true);
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzChangeEventConsumer.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzChangeEventConsumer.java
index b6d030537c105..cabf9e29a68aa 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzChangeEventConsumer.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzChangeEventConsumer.java
@@ -14,6 +14,7 @@
 
 package com.risingwave.connector.source.core;
 
+import com.risingwave.connector.api.source.SourceTypeE;
 import com.risingwave.connector.cdc.debezium.internal.DebeziumOffset;
 import com.risingwave.connector.cdc.debezium.internal.DebeziumOffsetSerializer;
 import com.risingwave.proto.ConnectorServiceProto.CdcMessage;
@@ -49,6 +50,8 @@ public class DbzChangeEventConsumer
     static final Logger LOG = LoggerFactory.getLogger(DbzChangeEventConsumer.class);
 
     private final BlockingQueue<GetEventStreamResponse> outputChannel;
+
+    private final SourceTypeE connector;
     private final long sourceId;
     private final JsonConverter payloadConverter;
     private final JsonConverter keyConverter;
@@ -59,10 +62,12 @@ public class DbzChangeEventConsumer
             currentRecordCommitter;
 
     DbzChangeEventConsumer(
+            SourceTypeE connector,
             long sourceId,
             String heartbeatTopicPrefix,
             String transactionTopic,
             BlockingQueue<GetEventStreamResponse> queue) {
+        this.connector = connector;
         this.sourceId = sourceId;
         this.outputChannel = queue;
         this.heartbeatTopicPrefix = heartbeatTopicPrefix;
@@ -87,6 +92,14 @@ public class DbzChangeEventConsumer
         this.keyConverter = keyConverter;
     }
 
+    /**
+     * Postgres and Oracle connectors need to commit the offset to the upstream database, so that we
+     * need to wait for the epoch commit before committing the record offset.
+     */
+    private boolean noNeedCommitOffset() {
+        return connector != SourceTypeE.POSTGRES;
+    }
+
     private EventType getEventType(SourceRecord record) {
         if (isHeartbeatEvent(record)) {
             return EventType.HEARTBEAT;
@@ -207,6 +220,9 @@ var record = event.value();
                 default:
                     break;
             }
+            if (noNeedCommitOffset()) {
+                committer.markProcessed(event);
+            }
         }
 
         LOG.debug("recv {} events", respBuilder.getEventsCount());
@@ -216,6 +232,9 @@ var record = event.value();
             var response = respBuilder.build();
             outputChannel.put(response);
         }
+        if (noNeedCommitOffset()) {
+            committer.markBatchFinished();
+        }
     }
 
     public BlockingQueue<GetEventStreamResponse> getOutputChannel() {
diff --git a/java/connector-node/risingwave-connector-service/src/main/resources/debezium.properties b/java/connector-node/risingwave-connector-service/src/main/resources/debezium.properties
index 4d85e1b98c78f..8b887cadbab02 100644
--- a/java/connector-node/risingwave-connector-service/src/main/resources/debezium.properties
+++ b/java/connector-node/risingwave-connector-service/src/main/resources/debezium.properties
@@ -11,7 +11,7 @@ decimal.handling.mode=${debezium.decimal.handling.mode:-string}
 interval.handling.mode=string
 max.batch.size=${debezium.max.batch.size:-1024}
 max.queue.size=${debezium.max.queue.size:-8192}
-time.precision.mode=connect
+time.precision.mode=adaptive_time_microseconds
 # Quoted from the debezium document:
 # > Your application should always properly stop the engine to ensure graceful and complete
 # > shutdown and that each source record is sent to the application exactly one time.
@@ -19,3 +19,4 @@ time.precision.mode=connect
 # handle conflicts in the mview operator, thus we don't need to obey the above
 # instructions. So we decrease the wait time here to reclaim jvm thread faster.
 debezium.embedded.shutdown.pause.before.interrupt.ms=1
+offset.flush.interval.ms=60000
diff --git a/java/connector-node/risingwave-connector-service/src/main/resources/mysql.properties b/java/connector-node/risingwave-connector-service/src/main/resources/mysql.properties
index bcf99d0c29580..0c62a51986b1c 100644
--- a/java/connector-node/risingwave-connector-service/src/main/resources/mysql.properties
+++ b/java/connector-node/risingwave-connector-service/src/main/resources/mysql.properties
@@ -15,6 +15,8 @@ schema.history.internal.store.only.captured.databases.ddl=true
 # default to disable schema change events
 include.schema.changes=${debezium.include.schema.changes:-false}
 database.server.id=${server.id}
+# default to use unencrypted connection
+database.ssl.mode=${ssl.mode:-disabled}
 # default heartbeat interval 60 seconds
 heartbeat.interval.ms=${debezium.heartbeat.interval.ms:-60000}
 # In sharing cdc mode, we will subscribe to multiple tables in the given database,
@@ -23,5 +25,6 @@ name=${hostname}:${port}:${database.name}.${table.name:-RW_CDC_Sharing}
 # In sharing cdc mode, transaction metadata will be enabled in frontend
 provide.transaction.metadata=${transactional:-false}
 ## Pass-through driver properties
-# set connector timezone to UTC(+00:00)
+# force connection session timezone to UTC(+00:00)
 driver.connectionTimeZone=+00:00
+driver.forceConnectionTimeZoneToSession=true
diff --git a/java/connector-node/risingwave-connector-service/src/main/resources/postgres.properties b/java/connector-node/risingwave-connector-service/src/main/resources/postgres.properties
index 326138403d3b2..06c4210fcf468 100644
--- a/java/connector-node/risingwave-connector-service/src/main/resources/postgres.properties
+++ b/java/connector-node/risingwave-connector-service/src/main/resources/postgres.properties
@@ -17,6 +17,9 @@ publication.autocreate.mode=disabled
 publication.name=${publication.name:-rw_publication}
 # default heartbeat interval 5 mins
 heartbeat.interval.ms=${debezium.heartbeat.interval.ms:-300000}
+# emit a WAL message to the replication stream
+# see https://github.com/risingwavelabs/risingwave/issues/16697 for more details
+heartbeat.action.query=SELECT pg_logical_emit_message(false, 'heartbeat', now()::varchar)
 # In sharing cdc source mode, we will subscribe to multiple tables in the given database,
 # so here we set ${table.name} to a default value `RW_CDC_Sharing` just for display.
 name=${hostname}:${port}:${database.name}.${schema.name}.${table.name:-RW_CDC_Sharing}
diff --git a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/DeserializerTest.java b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/DeserializerTest.java
deleted file mode 100644
index 9284a2ef8fd20..0000000000000
--- a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/DeserializerTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.connector.sink;
-
-import com.risingwave.connector.JsonDeserializer;
-import com.risingwave.connector.TestUtils;
-import com.risingwave.connector.api.sink.SinkRow;
-import com.risingwave.proto.ConnectorServiceProto;
-import com.risingwave.proto.ConnectorServiceProto.SinkWriterStreamRequest.WriteBatch.JsonPayload;
-import com.risingwave.proto.Data;
-import junit.framework.TestCase;
-
-public class DeserializerTest extends TestCase {
-    public void testJsonDeserializer() {
-        JsonDeserializer deserializer = new JsonDeserializer(TestUtils.getMockTableSchema());
-        JsonPayload jsonPayload =
-                JsonPayload.newBuilder()
-                        .addRowOps(
-                                JsonPayload.RowOp.newBuilder()
-                                        .setOpType(Data.Op.INSERT)
-                                        .setLine("{\"id\": 1, \"name\": \"John\"}")
-                                        .build())
-                        .build();
-        ConnectorServiceProto.SinkWriterStreamRequest.WriteBatch writeBatch =
-                ConnectorServiceProto.SinkWriterStreamRequest.WriteBatch.newBuilder()
-                        .setJsonPayload(jsonPayload)
-                        .build();
-        SinkRow outcome = deserializer.deserialize(writeBatch).iterator().next();
-        assertEquals(outcome.get(0), 1);
-        assertEquals(outcome.get(1), "John");
-    }
-}
diff --git a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/SinkStreamObserverTest.java b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/SinkStreamObserverTest.java
index f0dcc4c1c4930..885fc7eb927a3 100644
--- a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/SinkStreamObserverTest.java
+++ b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/SinkStreamObserverTest.java
@@ -14,10 +14,10 @@
 
 package com.risingwave.connector.sink;
 
+import com.google.protobuf.ByteString;
 import com.risingwave.connector.SinkWriterStreamObserver;
 import com.risingwave.connector.TestUtils;
 import com.risingwave.proto.ConnectorServiceProto;
-import com.risingwave.proto.Data.Op;
 import io.grpc.stub.StreamObserver;
 import java.util.Map;
 import org.junit.Assert;
@@ -94,7 +94,6 @@ public void testOnNext_syncValidation() {
                         .setStart(
                                 ConnectorServiceProto.SinkWriterStreamRequest.StartSink.newBuilder()
                                         .setSinkParam(fileSinkParam)
-                                        .setFormat(ConnectorServiceProto.SinkPayloadFormat.JSON)
                                         .build())
                         .build();
         ConnectorServiceProto.SinkWriterStreamRequest firstSync =
@@ -138,7 +137,6 @@ public void testOnNext_startEpochValidation() {
                         .setStart(
                                 ConnectorServiceProto.SinkWriterStreamRequest.StartSink.newBuilder()
                                         .setSinkParam(fileSinkParam)
-                                        .setFormat(ConnectorServiceProto.SinkPayloadFormat.JSON)
                                         .build())
                         .build();
         ConnectorServiceProto.SinkWriterStreamRequest firstSync =
@@ -156,6 +154,8 @@ public void testOnNext_startEpochValidation() {
         sinkWriterStreamObserver.onNext(firstSync);
     }
 
+    // WARN! This test is skipped in CI pipeline see
+    // `.github/workflows/connector-node-integration.yml`
     @Test
     public void testOnNext_writeValidation() {
         SinkWriterStreamObserver sinkWriterStreamObserver;
@@ -164,10 +164,16 @@ public void testOnNext_writeValidation() {
                 ConnectorServiceProto.SinkWriterStreamRequest.newBuilder()
                         .setStart(
                                 ConnectorServiceProto.SinkWriterStreamRequest.StartSink.newBuilder()
-                                        .setFormat(ConnectorServiceProto.SinkPayloadFormat.JSON)
                                         .setSinkParam(fileSinkParam))
                         .build();
 
+        // Encoded StreamChunk: 1 'test'
+        byte[] data1 =
+                new byte[] {
+                    8, 1, 18, 1, 1, 26, 20, 8, 2, 18, 6, 8, 1, 18, 2, 1, 1, 26, 8, 8, 1, 18, 4, 0,
+                    0, 0, 1, 26, 42, 8, 6, 18, 6, 8, 1, 18, 2, 1, 1, 26, 20, 8, 1, 18, 16, 0, 0, 0,
+                    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 26, 8, 8, 1, 18, 4, 116, 101, 115, 116
+                };
         ConnectorServiceProto.SinkWriterStreamRequest firstWrite =
                 ConnectorServiceProto.SinkWriterStreamRequest.newBuilder()
                         .setWriteBatch(
@@ -175,19 +181,11 @@ public void testOnNext_writeValidation() {
                                         .newBuilder()
                                         .setEpoch(0)
                                         .setBatchId(1)
-                                        .setJsonPayload(
+                                        .setStreamChunkPayload(
                                                 ConnectorServiceProto.SinkWriterStreamRequest
-                                                        .WriteBatch.JsonPayload.newBuilder()
-                                                        .addRowOps(
-                                                                ConnectorServiceProto
-                                                                        .SinkWriterStreamRequest
-                                                                        .WriteBatch.JsonPayload
-                                                                        .RowOp.newBuilder()
-                                                                        .setOpType(Op.INSERT)
-                                                                        .setLine(
-                                                                                "{\"id\": 1, \"name\": \"test\"}")
-                                                                        .build()))
-                                        .build())
+                                                        .WriteBatch.StreamChunkPayload.newBuilder()
+                                                        .setBinaryData(ByteString.copyFrom(data1))
+                                                        .build()))
                         .build();
 
         ConnectorServiceProto.SinkWriterStreamRequest firstSync =
@@ -199,6 +197,13 @@ public void testOnNext_writeValidation() {
                                         .build())
                         .build();
 
+        // Encoded StreamChunk: 2 'test'
+        byte[] data2 =
+                new byte[] {
+                    8, 1, 18, 1, 1, 26, 20, 8, 2, 18, 6, 8, 1, 18, 2, 1, 1, 26, 8, 8, 1, 18, 4, 0,
+                    0, 0, 2, 26, 42, 8, 6, 18, 6, 8, 1, 18, 2, 1, 1, 26, 20, 8, 1, 18, 16, 0, 0, 0,
+                    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 26, 8, 8, 1, 18, 4, 116, 101, 115, 116
+                };
         ConnectorServiceProto.SinkWriterStreamRequest secondWrite =
                 ConnectorServiceProto.SinkWriterStreamRequest.newBuilder()
                         .setWriteBatch(
@@ -206,19 +211,11 @@ public void testOnNext_writeValidation() {
                                         .newBuilder()
                                         .setEpoch(1)
                                         .setBatchId(2)
-                                        .setJsonPayload(
+                                        .setStreamChunkPayload(
                                                 ConnectorServiceProto.SinkWriterStreamRequest
-                                                        .WriteBatch.JsonPayload.newBuilder()
-                                                        .addRowOps(
-                                                                ConnectorServiceProto
-                                                                        .SinkWriterStreamRequest
-                                                                        .WriteBatch.JsonPayload
-                                                                        .RowOp.newBuilder()
-                                                                        .setOpType(Op.INSERT)
-                                                                        .setLine(
-                                                                                "{\"id\": 2, \"name\": \"test\"}")
-                                                                        .build()))
-                                        .build())
+                                                        .WriteBatch.StreamChunkPayload.newBuilder()
+                                                        .setBinaryData(ByteString.copyFrom(data2))
+                                                        .build()))
                         .build();
 
         ConnectorServiceProto.SinkWriterStreamRequest secondWriteWrongEpoch =
@@ -228,19 +225,11 @@ public void testOnNext_writeValidation() {
                                         .newBuilder()
                                         .setEpoch(2)
                                         .setBatchId(3)
-                                        .setJsonPayload(
+                                        .setStreamChunkPayload(
                                                 ConnectorServiceProto.SinkWriterStreamRequest
-                                                        .WriteBatch.JsonPayload.newBuilder()
-                                                        .addRowOps(
-                                                                ConnectorServiceProto
-                                                                        .SinkWriterStreamRequest
-                                                                        .WriteBatch.JsonPayload
-                                                                        .RowOp.newBuilder()
-                                                                        .setOpType(Op.INSERT)
-                                                                        .setLine(
-                                                                                "{\"id\": 2, \"name\": \"test\"}")
-                                                                        .build()))
-                                        .build())
+                                                        .WriteBatch.StreamChunkPayload.newBuilder()
+                                                        .setBinaryData(ByteString.copyFrom(data2))
+                                                        .build()))
                         .build();
 
         boolean exceptionThrown = false;
@@ -251,7 +240,10 @@ public void testOnNext_writeValidation() {
             sinkWriterStreamObserver.onNext(firstWrite);
         } catch (RuntimeException e) {
             exceptionThrown = true;
-            Assert.assertTrue(e.getMessage().toLowerCase().contains("batch id"));
+            if (!e.getMessage().toLowerCase().contains("batch id")) {
+                e.printStackTrace();
+                Assert.fail("Expected `batch id`, but got " + e.getMessage());
+            }
         }
         if (!exceptionThrown) {
             Assert.fail("Expected exception not thrown: `invalid batch id`");
@@ -267,7 +259,10 @@ public void testOnNext_writeValidation() {
             sinkWriterStreamObserver.onNext(secondWriteWrongEpoch);
         } catch (RuntimeException e) {
             exceptionThrown = true;
-            Assert.assertTrue(e.getMessage().toLowerCase().contains("invalid epoch"));
+            if (!e.getMessage().toLowerCase().contains("invalid epoch")) {
+                e.printStackTrace();
+                Assert.fail("Expected `invalid epoch`, but got " + e.getMessage());
+            }
         }
         if (!exceptionThrown) {
             Assert.fail(
diff --git a/java/connector-node/risingwave-sink-iceberg/pom.xml b/java/connector-node/risingwave-sink-iceberg/pom.xml
index 9f733d830a475..a491823bb07f8 100644
--- a/java/connector-node/risingwave-sink-iceberg/pom.xml
+++ b/java/connector-node/risingwave-sink-iceberg/pom.xml
@@ -113,8 +113,8 @@
             <artifactId>postgresql</artifactId>
         </dependency>
         <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
         </dependency>
         <dependency>
             <groupId>org.xerial</groupId>
diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/catalog/JniCatalogWrapper.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/catalog/JniCatalogWrapper.java
index e8c900b37e88d..583747f3b2f3f 100644
--- a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/catalog/JniCatalogWrapper.java
+++ b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/catalog/JniCatalogWrapper.java
@@ -74,15 +74,6 @@ public static JniCatalogWrapper create(String name, String klassName, String[] p
         checkArgument(
                 props.length % 2 == 0,
                 "props should be key-value pairs, but length is: " + props.length);
-
-        //      Thread.currentThread().setContextClassLoader(ClassLoader.getSystemClassLoader());
-        System.out.println("Current thread name is: " + Thread.currentThread().getName());
-
-        //        try {
-        //            Thread.currentThread().getContextClassLoader().loadClass(klassName);
-        //        } catch (ClassNotFoundException e) {
-        //            throw new RuntimeException(e);
-        //        }
         try {
             HashMap<String, String> config = new HashMap<>(props.length / 2);
             for (int i = 0; i < props.length; i += 2) {
diff --git a/java/connector-node/risingwave-sink-jdbc/pom.xml b/java/connector-node/risingwave-sink-jdbc/pom.xml
index 0390d57a66870..52947595c7413 100644
--- a/java/connector-node/risingwave-sink-jdbc/pom.xml
+++ b/java/connector-node/risingwave-sink-jdbc/pom.xml
@@ -38,8 +38,8 @@
             <artifactId>postgresql</artifactId>
         </dependency>
         <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
         </dependency>
 
     </dependencies>
diff --git a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JDBCSink.java b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JDBCSink.java
index dfdc01f4a36ed..399a312758e3b 100644
--- a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JDBCSink.java
+++ b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JDBCSink.java
@@ -23,6 +23,7 @@
 import io.grpc.Status;
 import java.sql.*;
 import java.util.*;
+import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -52,26 +53,33 @@ public JDBCSink(JDBCSinkConfig config, TableSchema tableSchema) {
         this.config = config;
         try {
             conn = JdbcUtils.getConnection(config.getJdbcUrl());
-            // Retrieve primary keys and column type mappings from the database
-            this.pkColumnNames =
-                    getPkColumnNames(conn, config.getTableName(), config.getSchemaName());
+            // Table schema has been validated before, so we get the PK from it directly
+            this.pkColumnNames = tableSchema.getPrimaryKeys();
             // column name -> java.sql.Types
             Map<String, Integer> columnTypeMapping =
                     getColumnTypeMapping(conn, config.getTableName(), config.getSchemaName());
-            // create an array that each slot corresponding to each column in TableSchema
-            var columnSqlTypes = new int[tableSchema.getNumColumns()];
-            for (int columnIdx = 0; columnIdx < tableSchema.getNumColumns(); columnIdx++) {
-                var columnName = tableSchema.getColumnNames()[columnIdx];
-                columnSqlTypes[columnIdx] = columnTypeMapping.get(columnName);
-            }
+
+            // A vector of upstream column types
+            List<Integer> columnSqlTypes =
+                    Arrays.stream(tableSchema.getColumnNames())
+                            .map(columnTypeMapping::get)
+                            .collect(Collectors.toList());
+
+            List<Integer> pkIndices =
+                    tableSchema.getPrimaryKeys().stream()
+                            .map(tableSchema::getColumnIndex)
+                            .collect(Collectors.toList());
+
             LOG.info(
-                    "schema = {}, table = {}: columnSqlTypes = {}",
+                    "schema = {}, table = {}, tableSchema = {}, columnSqlTypes = {}, pkIndices = {}",
                     config.getSchemaName(),
                     config.getTableName(),
-                    Arrays.toString(columnSqlTypes));
+                    tableSchema,
+                    columnSqlTypes,
+                    pkIndices);
 
             if (factory.isPresent()) {
-                this.jdbcDialect = factory.get().create(columnSqlTypes);
+                this.jdbcDialect = factory.get().create(columnSqlTypes, pkIndices);
             } else {
                 throw Status.INVALID_ARGUMENT
                         .withDescription("Unsupported jdbc url: " + jdbcUrl)
@@ -81,6 +89,8 @@ public JDBCSink(JDBCSinkConfig config, TableSchema tableSchema) {
                     "JDBC connection: autoCommit = {}, trxn = {}",
                     conn.getAutoCommit(),
                     conn.getTransactionIsolation());
+            // Commit the `getTransactionIsolation`
+            conn.commit();
 
             jdbcStatements = new JdbcStatements(conn);
         } catch (SQLException e) {
@@ -117,28 +127,6 @@ private static Map<String, Integer> getColumnTypeMapping(
         return columnTypeMap;
     }
 
-    private static List<String> getPkColumnNames(
-            Connection conn, String tableName, String schemaName) {
-        List<String> pkColumnNames = new ArrayList<>();
-        try {
-            var pks = conn.getMetaData().getPrimaryKeys(null, schemaName, tableName);
-            while (pks.next()) {
-                pkColumnNames.add(pks.getString(JDBC_COLUMN_NAME_KEY));
-            }
-        } catch (SQLException e) {
-            throw Status.INTERNAL
-                    .withDescription(
-                            String.format(ERROR_REPORT_TEMPLATE, e.getSQLState(), e.getMessage()))
-                    .asRuntimeException();
-        }
-        LOG.info(
-                "schema = {}, table = {}: detected pk column = {}",
-                schemaName,
-                tableName,
-                pkColumnNames);
-        return pkColumnNames;
-    }
-
     @Override
     public boolean write(Iterable<SinkRow> rows) {
         final int maxRetryCount = 4;
@@ -303,11 +291,7 @@ public void prepareDelete(SinkRow row) {
                         .asRuntimeException();
             }
             try {
-                int placeholderIdx = 1;
-                for (String primaryKey : pkColumnNames) {
-                    Object fromRow = tableSchema.getFromRow(primaryKey, row);
-                    deleteStatement.setObject(placeholderIdx++, fromRow);
-                }
+                jdbcDialect.bindDeleteStatement(deleteStatement, tableSchema, row);
                 deleteStatement.addBatch();
             } catch (SQLException e) {
                 throw Status.INTERNAL
diff --git a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JDBCSinkFactory.java b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JDBCSinkFactory.java
index 835c7b9693668..875bda89681de 100644
--- a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JDBCSinkFactory.java
+++ b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JDBCSinkFactory.java
@@ -75,7 +75,11 @@ public void validate(
         } catch (SQLException e) {
             LOG.error("failed to connect to target database. jdbcUrl: {}", jdbcUrl, e);
             throw Status.INVALID_ARGUMENT
-                    .withDescription("failed to connect to target database: " + e.getSQLState())
+                    .withDescription(
+                            "failed to connect to target database: "
+                                    + e.getSQLState()
+                                    + ": "
+                                    + e.getMessage())
                     .asRuntimeException();
         }
 
diff --git a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JdbcUtils.java b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JdbcUtils.java
index 7d28e4553fe3e..2360ac96585dd 100644
--- a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JdbcUtils.java
+++ b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JdbcUtils.java
@@ -25,6 +25,9 @@
 
 public abstract class JdbcUtils {
 
+    static final int CONNECTION_TIMEOUT = 30;
+    static final int SOCKET_TIMEOUT = 300;
+
     public static Optional<JdbcDialectFactory> getDialectFactory(String jdbcUrl) {
         if (jdbcUrl.startsWith("jdbc:mysql")) {
             return Optional.of(new MySqlDialectFactory());
@@ -43,6 +46,16 @@ public static Connection getConnection(String jdbcUrl) throws SQLException {
         // https://jdbc.postgresql.org/documentation/use/
         // https://dev.mysql.com/doc/connectors/en/connector-j-connp-props-networking.html#cj-conn-prop_tcpKeepAlive
         props.setProperty("tcpKeepAlive", "true");
+
+        // default timeout in seconds
+        boolean isPg = jdbcUrl.startsWith("jdbc:postgresql");
+
+        // postgres use seconds and mysql use milliseconds
+        int connectTimeout = isPg ? CONNECTION_TIMEOUT : CONNECTION_TIMEOUT * 1000;
+        int socketTimeout = isPg ? SOCKET_TIMEOUT : SOCKET_TIMEOUT * 1000;
+        props.setProperty("connectTimeout", String.valueOf(connectTimeout));
+        props.setProperty("socketTimeout", String.valueOf(socketTimeout));
+
         var conn = DriverManager.getConnection(jdbcUrl, props);
         // disable auto commit can improve performance
         conn.setAutoCommit(false);
diff --git a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/JdbcDialect.java b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/JdbcDialect.java
index 1be3e949d474d..308f9927457a2 100644
--- a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/JdbcDialect.java
+++ b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/JdbcDialect.java
@@ -110,4 +110,8 @@ void bindUpsertStatement(
     void bindInsertIntoStatement(
             PreparedStatement stmt, Connection conn, TableSchema tableSchema, SinkRow row)
             throws SQLException;
+
+    /** Bind the values of primary key fields to the {@code DELETE} statement. */
+    void bindDeleteStatement(PreparedStatement stmt, TableSchema tableSchema, SinkRow row)
+            throws SQLException;
 }
diff --git a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/JdbcDialectFactory.java b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/JdbcDialectFactory.java
index 1bfbf5f9f24c2..75e349489b461 100644
--- a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/JdbcDialectFactory.java
+++ b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/JdbcDialectFactory.java
@@ -14,7 +14,9 @@
 
 package com.risingwave.connector.jdbc;
 
+import java.util.List;
+
 public interface JdbcDialectFactory {
 
-    JdbcDialect create(int[] columnSqlTypes);
+    JdbcDialect create(List<Integer> columnSqlTypes, List<Integer> pkIndices);
 }
diff --git a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/MySqlDialect.java b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/MySqlDialect.java
index d2cdbd6f1e1d0..2c4ea73448a60 100644
--- a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/MySqlDialect.java
+++ b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/MySqlDialect.java
@@ -26,6 +26,21 @@
 
 public class MySqlDialect implements JdbcDialect {
 
+    private final int[] pkIndices;
+    private final int[] pkColumnSqlTypes;
+
+    public MySqlDialect(List<Integer> columnSqlTypes, List<Integer> pkIndices) {
+        var columnSqlTypesArr = columnSqlTypes.stream().mapToInt(i -> i).toArray();
+        this.pkIndices = pkIndices.stream().mapToInt(i -> i).toArray();
+
+        // derive sql types for pk columns
+        var pkColumnSqlTypes = new int[pkIndices.size()];
+        for (int i = 0; i < pkIndices.size(); i++) {
+            pkColumnSqlTypes[i] = columnSqlTypesArr[this.pkIndices[i]];
+        }
+        this.pkColumnSqlTypes = pkColumnSqlTypes;
+    }
+
     @Override
     public SchemaTableName createSchemaTableName(String schemaName, String tableName) {
         return new SchemaTableName(schemaName, tableName);
@@ -100,4 +115,15 @@ public void bindInsertIntoStatement(
             }
         }
     }
+
+    @Override
+    public void bindDeleteStatement(PreparedStatement stmt, TableSchema tableSchema, SinkRow row)
+            throws SQLException {
+        // set the values of primary key fields
+        int placeholderIdx = 1;
+        for (int i = 0; i < pkIndices.length; ++i) {
+            Object pkField = row.get(pkIndices[i]);
+            stmt.setObject(placeholderIdx++, pkField, pkColumnSqlTypes[i]);
+        }
+    }
 }
diff --git a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/MySqlDialectFactory.java b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/MySqlDialectFactory.java
index b6a8948bfe84e..ebc46d86967f3 100644
--- a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/MySqlDialectFactory.java
+++ b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/MySqlDialectFactory.java
@@ -14,10 +14,12 @@
 
 package com.risingwave.connector.jdbc;
 
+import java.util.List;
+
 public class MySqlDialectFactory implements JdbcDialectFactory {
 
     @Override
-    public JdbcDialect create(int[] columnSqlTypes) {
-        return new MySqlDialect();
+    public JdbcDialect create(List<Integer> columnSqlTypes, List<Integer> pkIndices) {
+        return new MySqlDialect(columnSqlTypes, pkIndices);
     }
 }
diff --git a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/PostgresDialect.java b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/PostgresDialect.java
index b7b13be2d154a..6264d9c5eac18 100644
--- a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/PostgresDialect.java
+++ b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/PostgresDialect.java
@@ -30,9 +30,11 @@
 public class PostgresDialect implements JdbcDialect {
 
     private final int[] columnSqlTypes;
+    private final int[] pkIndices;
 
-    public PostgresDialect(int[] columnSqlTypes) {
-        this.columnSqlTypes = columnSqlTypes;
+    public PostgresDialect(List<Integer> columnSqlTypes, List<Integer> pkIndices) {
+        this.columnSqlTypes = columnSqlTypes.stream().mapToInt(i -> i).toArray();
+        this.pkIndices = pkIndices.stream().mapToInt(i -> i).toArray();
     }
 
     private static final HashMap<TypeName, String> RW_TYPE_TO_JDBC_TYPE_NAME;
@@ -154,4 +156,15 @@ public void bindInsertIntoStatement(
             }
         }
     }
+
+    @Override
+    public void bindDeleteStatement(PreparedStatement stmt, TableSchema tableSchema, SinkRow row)
+            throws SQLException {
+        // set the values of primary key fields
+        int placeholderIdx = 1;
+        for (int pkIdx : pkIndices) {
+            Object pkField = row.get(pkIdx);
+            stmt.setObject(placeholderIdx++, pkField, columnSqlTypes[pkIdx]);
+        }
+    }
 }
diff --git a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/PostgresDialectFactory.java b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/PostgresDialectFactory.java
index 651875e27dd10..3b6ad2b2bcf5b 100644
--- a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/PostgresDialectFactory.java
+++ b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/jdbc/PostgresDialectFactory.java
@@ -14,10 +14,12 @@
 
 package com.risingwave.connector.jdbc;
 
+import java.util.List;
+
 public class PostgresDialectFactory implements JdbcDialectFactory {
 
     @Override
-    public JdbcDialect create(int[] columnSqlTypes) {
-        return new PostgresDialect(columnSqlTypes);
+    public JdbcDialect create(List<Integer> columnSqlTypes, List<Integer> pkIndices) {
+        return new PostgresDialect(columnSqlTypes, pkIndices);
     }
 }
diff --git a/java/connector-node/risingwave-source-cdc/pom.xml b/java/connector-node/risingwave-source-cdc/pom.xml
index fd1dccd3f092f..5ee531ef805e9 100644
--- a/java/connector-node/risingwave-source-cdc/pom.xml
+++ b/java/connector-node/risingwave-source-cdc/pom.xml
@@ -50,8 +50,8 @@
 
         <!-- database dependencies  -->
         <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
         </dependency>
         <dependency>
             <groupId>com.zendesk</groupId>
diff --git a/java/connector-node/risingwave-source-test/src/test/java/com/risingwave/connector/source/PostgresSourceTest.java b/java/connector-node/risingwave-source-test/src/test/java/com/risingwave/connector/source/PostgresSourceTest.java
index 477ae42899f16..5ccb24cef3ffc 100644
--- a/java/connector-node/risingwave-source-test/src/test/java/com/risingwave/connector/source/PostgresSourceTest.java
+++ b/java/connector-node/risingwave-source-test/src/test/java/com/risingwave/connector/source/PostgresSourceTest.java
@@ -145,7 +145,8 @@ public void testLines() throws Exception {
         int count = countResult.get();
         LOG.info("number of cdc messages received: {}", count);
         try {
-            assertEquals(10000, count);
+            // 10000 rows plus one heartbeat message
+            assertTrue(count >= 10000);
         } catch (Exception e) {
             Assert.fail("validate rpc fail: " + e.getMessage());
         } finally {
diff --git a/java/dev.md b/java/dev.md
index ac20c30fe69fa..148fde173baad 100644
--- a/java/dev.md
+++ b/java/dev.md
@@ -56,9 +56,3 @@ Config with the following. It may work.
   "java.format.settings.profile": "Android"
 }
 ```
-
-## Deploy UDF Library to Maven
-
-```sh
-mvn clean deploy --pl udf --am
-```
\ No newline at end of file
diff --git a/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java b/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java
index a16acda73e7fd..db832566fdfa7 100644
--- a/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java
+++ b/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java
@@ -26,6 +26,8 @@ public class Binding {
         }
     }
 
+    static void ensureInitialized() {}
+
     public static native void tracingSlf4jEvent(
             String threadName, String name, int level, String message);
 
@@ -33,10 +35,6 @@ public static native void tracingSlf4jEvent(
 
     public static native int vnodeCount();
 
-    // hummock iterator method
-    // Return a pointer to the iterator
-    static native long iteratorNewHummock(byte[] readPlan);
-
     static native long iteratorNewStreamChunk(long pointer);
 
     static native boolean iteratorNext(long pointer);
diff --git a/java/java-binding/src/main/java/com/risingwave/java/binding/HummockIterator.java b/java/java-binding/src/main/java/com/risingwave/java/binding/HummockIterator.java
index 03282a2dce528..a30391edbd380 100644
--- a/java/java-binding/src/main/java/com/risingwave/java/binding/HummockIterator.java
+++ b/java/java-binding/src/main/java/com/risingwave/java/binding/HummockIterator.java
@@ -20,8 +20,16 @@ public class HummockIterator implements AutoCloseable {
     private final long pointer;
     private boolean isClosed;
 
+    static {
+        Binding.ensureInitialized();
+    }
+
+    // hummock iterator method
+    // Return a pointer to the iterator
+    private static native long iteratorNewHummock(byte[] readPlan);
+
     public HummockIterator(ReadPlan readPlan) {
-        this.pointer = Binding.iteratorNewHummock(readPlan.toByteArray());
+        this.pointer = iteratorNewHummock(readPlan.toByteArray());
         this.isClosed = false;
     }
 
diff --git a/java/pom.xml b/java/pom.xml
index 89df5b870077b..5f0327bf8ffc9 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -37,8 +37,6 @@
 
     <modules>
         <module>proto</module>
-        <module>udf</module>
-        <module>udf-example</module>
         <module>java-binding</module>
         <module>common-utils</module>
         <module>java-binding-integration-test</module>
@@ -85,7 +83,7 @@
         <flink.version>1.18.0</flink.version>
         <testcontainers.version>1.17.6</testcontainers.version>
         <postgresql.version>42.5.5</postgresql.version>
-        <mysql.connector.java.version>8.0.28</mysql.connector.java.version>
+        <mysql.connector.java.version>8.3.0</mysql.connector.java.version>
         <mongodb.driver.sync.version>4.11.1</mongodb.driver.sync.version>
         <sqlite.version>3.45.0.0</sqlite.version>
         <aws.version>2.21.42</aws.version>
@@ -178,8 +176,8 @@
                 <version>${postgresql.version}</version>
             </dependency>
             <dependency>
-                <groupId>mysql</groupId>
-                <artifactId>mysql-connector-java</artifactId>
+                <groupId>com.mysql</groupId>
+                <artifactId>mysql-connector-j</artifactId>
                 <version>${mysql.connector.java.version}</version>
             </dependency>
             <dependency>
@@ -360,11 +358,6 @@
                 <artifactId>apache-client</artifactId>
                 <version>${aws.version}</version>
             </dependency>
-            <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-common</artifactId>
-                <version>${hadoop.version}</version>
-            </dependency>
             <dependency>
                 <groupId>org.apache.hive</groupId>
                 <artifactId>hive-metastore</artifactId>
diff --git a/java/udf/CHANGELOG.md b/java/udf/CHANGELOG.md
deleted file mode 100644
index fb1f055783225..0000000000000
--- a/java/udf/CHANGELOG.md
+++ /dev/null
@@ -1,39 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [Unreleased]
-
-## [0.1.3] - 2023-12-06
-
-### Fixed
-
-- Fix decimal type output.
-
-## [0.1.2] - 2023-12-04
-
-### Fixed
-
-- Fix index-out-of-bound error when string or string list is large.
-- Fix memory leak.
-
-## [0.1.1] - 2023-12-03
-
-### Added
-
-- Support struct in struct and struct[] in struct.
-
-### Changed
-
-- Bump Arrow version to 14.
-
-### Fixed
-
-- Fix unconstrained decimal type.
-
-## [0.1.0] - 2023-09-01
-
-- Initial release.
\ No newline at end of file
diff --git a/java/udf/README.md b/java/udf/README.md
deleted file mode 100644
index 200b897b8b890..0000000000000
--- a/java/udf/README.md
+++ /dev/null
@@ -1,274 +0,0 @@
-# RisingWave Java UDF SDK
-
-This library provides a Java SDK for creating user-defined functions (UDF) in RisingWave.
-
-## Introduction
-
-RisingWave supports user-defined functions implemented as external functions.
-With the RisingWave Java UDF SDK, users can define custom UDFs using Java and start a Java process as a UDF server.
-RisingWave can then remotely access the UDF server to execute the defined functions.
-
-## Installation
-
-To install the RisingWave Java UDF SDK:
-
-```sh
-git clone https://github.com/risingwavelabs/risingwave.git
-cd risingwave/java/udf
-mvn install
-```
-
-Or you can add the following dependency to your `pom.xml` file:
-
-```xml
-<dependencies>
-    <dependency>
-        <groupId>com.risingwave</groupId>
-        <artifactId>risingwave-udf</artifactId>
-        <version>0.1.0</version>
-    </dependency>
-</dependencies>
-```
-
-
-## Creating a New Project
-
-> NOTE: You can also start from the [udf-example](../udf-example) project without creating the project from scratch.
-
-To create a new project using the RisingWave Java UDF SDK, follow these steps:
-
-```sh
-mvn archetype:generate -DgroupId=com.example -DartifactId=udf-example -DarchetypeArtifactId=maven-archetype-quickstart -DarchetypeVersion=1.4 -DinteractiveMode=false
-```
-
-Configure your `pom.xml` file as follows:
-
-```xml
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <groupId>com.example</groupId>
-    <artifactId>udf-example</artifactId>
-    <version>1.0-SNAPSHOT</version>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.risingwave</groupId>
-            <artifactId>risingwave-udf</artifactId>
-            <version>0.1.0</version>
-        </dependency>
-    </dependencies>
-</project>
-```
-
-The `--add-opens` flag must be added when running unit tests through Maven:
-
-```xml
-<build>
-    <plugins>
-        <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-surefire-plugin</artifactId>
-            <version>3.0.0</version>
-            <configuration>
-                <argLine>--add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
-            </configuration>
-        </plugin>
-    </plugins>
-</build>
-```
-
-## Scalar Functions
-
-A user-defined scalar function maps zero, one, or multiple scalar values to a new scalar value.
-
-In order to define a scalar function, one has to create a new class that implements the `ScalarFunction`
-interface in `com.risingwave.functions` and implement exactly one evaluation method named `eval(...)`.
-This method must be declared public and non-static.
-
-Any [data type](#data-types) listed in the data types section can be used as a parameter or return type of an evaluation method.
-
-Here's an example of a scalar function that calculates the greatest common divisor (GCD) of two integers:
-
-```java
-import com.risingwave.functions.ScalarFunction;
-
-public class Gcd implements ScalarFunction {
-    public int eval(int a, int b) {
-        while (b != 0) {
-            int temp = b;
-            b = a % b;
-            a = temp;
-        }
-        return a;
-    }
-}
-```
-
-> **NOTE:** Differences with Flink
-> 1. The `ScalarFunction` is an interface instead of an abstract class.
-> 2. Multiple overloaded `eval` methods are not supported.
-> 3. Variable arguments such as `eval(Integer...)` are not supported.
-
-## Table Functions
-
-A user-defined table function maps zero, one, or multiple scalar values to one or multiple
-rows (structured types).
-
-In order to define a table function, one has to create a new class that implements the `TableFunction`
-interface in `com.risingwave.functions` and implement exactly one evaluation method named `eval(...)`.
-This method must be declared public and non-static.
-
-The return type must be an `Iterator` of any [data type](#data-types) listed in the data types section.
-Similar to scalar functions, input and output data types are automatically extracted using reflection.
-This includes the generic argument T of the return value for determining an output data type.
-
-Here's an example of a table function that generates a series of integers:
-
-```java
-import com.risingwave.functions.TableFunction;
-
-public class Series implements TableFunction {
-    public Iterator<Integer> eval(int n) {
-        return java.util.stream.IntStream.range(0, n).iterator();
-    }
-}
-```
-
-> **NOTE:** Differences with Flink
-> 1. The `TableFunction` is an interface instead of an abstract class. It has no generic arguments.
-> 2. Instead of calling `collect` to emit a row, the `eval` method returns an `Iterator` of the output rows.
-> 3. Multiple overloaded `eval` methods are not supported.
-> 4. Variable arguments such as `eval(Integer...)` are not supported.
-> 5. In SQL, table functions can be used in the `FROM` clause directly. `JOIN LATERAL TABLE` is not supported.
-
-## UDF Server
-
-To create a UDF server and register functions:
-
-```java
-import com.risingwave.functions.UdfServer;
-
-public class App {
-    public static void main(String[] args) {
-        try (var server = new UdfServer("0.0.0.0", 8815)) {
-            // register functions
-            server.addFunction("gcd", new Gcd());
-            server.addFunction("series", new Series());
-            // start the server
-            server.start();
-            server.awaitTermination();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-}
-```
-
-To run the UDF server, execute the following command:
-
-```sh
-_JAVA_OPTIONS="--add-opens=java.base/java.nio=ALL-UNNAMED" mvn exec:java -Dexec.mainClass="com.example.App"
-```
-
-## Creating Functions in RisingWave
-
-```sql
-create function gcd(int, int) returns int
-as gcd using link 'http://localhost:8815';
-
-create function series(int) returns table (x int)
-as series using link 'http://localhost:8815';
-```
-
-For more detailed information and examples, please refer to the official RisingWave [documentation](https://www.risingwave.dev/docs/current/user-defined-functions/#4-declare-your-functions-in-risingwave).
-
-## Using Functions in RisingWave
-
-Once the user-defined functions are created in RisingWave, you can use them in SQL queries just like any built-in functions. Here are a few examples:
-
-```sql
-select gcd(25, 15);
-
-select * from series(10);
-```
-
-## Data Types
-
-The RisingWave Java UDF SDK supports the following data types:
-
-| SQL Type         | Java Type                               | Notes              |
-| ---------------- | --------------------------------------- | ------------------ |
-| BOOLEAN          | boolean, Boolean                        |                    |
-| SMALLINT         | short, Short                            |                    |
-| INT              | int, Integer                            |                    |
-| BIGINT           | long, Long                              |                    |
-| REAL             | float, Float                            |                    |
-| DOUBLE PRECISION | double, Double                          |                    |
-| DECIMAL          | BigDecimal                              |                    |
-| DATE             | java.time.LocalDate                     |                    |
-| TIME             | java.time.LocalTime                     |                    |
-| TIMESTAMP        | java.time.LocalDateTime                 |                    |
-| INTERVAL         | com.risingwave.functions.PeriodDuration |                    |
-| VARCHAR          | String                                  |                    |
-| BYTEA            | byte[]                                  |                    |
-| JSONB            | String                                  | Use `@DataTypeHint("JSONB") String` as the type. See [example](#jsonb).           |
-| T[]              | T'[]                                    | `T` can be any of the above SQL types. `T'` should be the corresponding Java type.|
-| STRUCT<>         | user-defined class                      | Define a data class as the type. See [example](#struct-type).                     |
-| ...others        |                                         | Not supported yet. |
-
-### JSONB
-
-```java
-import com.google.gson.Gson;
-
-// Returns the i-th element of a JSON array.
-public class JsonbAccess implements ScalarFunction {
-    static Gson gson = new Gson();
-
-    public @DataTypeHint("JSONB") String eval(@DataTypeHint("JSONB") String json, int index) {
-        if (json == null)
-            return null;
-        var array = gson.fromJson(json, Object[].class);
-        if (index >= array.length || index < 0)
-            return null;
-        var obj = array[index];
-        return gson.toJson(obj);
-    }
-}
-```
-
-```sql
-create function jsonb_access(jsonb, int) returns jsonb
-as jsonb_access using link 'http://localhost:8815';
-```
-
-### Struct Type
-
-```java
-// Split a socket address into host and port.
-public static class IpPort implements ScalarFunction {
-    public static class SocketAddr {
-        public String host;
-        public short port;
-    }
-
-    public SocketAddr eval(String addr) {
-        var socketAddr = new SocketAddr();
-        var parts = addr.split(":");
-        socketAddr.host = parts[0];
-        socketAddr.port = Short.parseShort(parts[1]);
-        return socketAddr;
-    }
-}
-```
-
-```sql
-create function ip_port(varchar) returns struct<host varchar, port smallint>
-as ip_port using link 'http://localhost:8815';
-```
-
-## Full Example
-
-You can checkout [udf-example](../udf-example) and use it as a template to create your own UDFs.
diff --git a/java/udf/pom.xml b/java/udf/pom.xml
deleted file mode 100644
index f747603ca8429..0000000000000
--- a/java/udf/pom.xml
+++ /dev/null
@@ -1,58 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-
-  <groupId>com.risingwave</groupId>
-  <artifactId>risingwave-udf</artifactId>
-  <packaging>jar</packaging>
-  <version>0.1.3-SNAPSHOT</version>
-
-  <parent>
-    <artifactId>risingwave-java-root</artifactId>
-    <groupId>com.risingwave</groupId>
-    <version>0.1.0-SNAPSHOT</version>
-    <relativePath>../pom.xml</relativePath>
-  </parent>
-
-  <name>RisingWave Java UDF SDK</name>
-  <url>https://docs.risingwave.com/docs/current/udf-java</url>
-
-  <dependencies>
-    <dependency>
-      <groupId>org.junit.jupiter</groupId>
-      <artifactId>junit-jupiter-engine</artifactId>
-      <version>5.9.1</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.arrow</groupId>
-      <artifactId>arrow-vector</artifactId>
-      <version>14.0.0</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.arrow</groupId>
-      <artifactId>flight-core</artifactId>
-      <version>14.0.0</version>
-    </dependency>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
-      <version>2.0.7</version>
-    </dependency>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-simple</artifactId>
-      <version>2.0.7</version>
-    </dependency>
-  </dependencies>
-  <build>
-    <extensions>
-      <extension>
-        <groupId>kr.motd.maven</groupId>
-        <artifactId>os-maven-plugin</artifactId>
-        <version>1.7.0</version>
-      </extension>
-    </extensions>
-  </build>
-</project>
\ No newline at end of file
diff --git a/java/udf/src/main/java/com/risingwave/functions/ScalarFunction.java b/java/udf/src/main/java/com/risingwave/functions/ScalarFunction.java
deleted file mode 100644
index 5f3fcaf287330..0000000000000
--- a/java/udf/src/main/java/com/risingwave/functions/ScalarFunction.java
+++ /dev/null
@@ -1,53 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.functions;
-
-/**
- * Base interface for a user-defined scalar function. A user-defined scalar function maps zero, one,
- * or multiple scalar values to a new scalar value.
- *
- * <p>The behavior of a {@link ScalarFunction} can be defined by implementing a custom evaluation
- * method. An evaluation method must be declared publicly, not static, and named <code>eval</code>.
- * Multiple overloaded methods named <code>eval</code> are not supported yet.
- *
- * <p>By default, input and output data types are automatically extracted using reflection.
- *
- * <p>The following examples show how to specify a scalar function:
- *
- * <pre>{@code
- * // a function that accepts two INT arguments and computes a sum
- * class SumFunction implements ScalarFunction {
- *     public Integer eval(Integer a, Integer b) {
- *         return a + b;
- *     }
- * }
- *
- * // a function that returns a struct type
- * class StructFunction implements ScalarFunction {
- *     public static class KeyValue {
- *         public String key;
- *         public int value;
- *     }
- *
- *     public KeyValue eval(int a) {
- *         KeyValue kv = new KeyValue();
- *         kv.key = a.toString();
- *         kv.value = a;
- *         return kv;
- *     }
- * }
- * }</pre>
- */
-public interface ScalarFunction extends UserDefinedFunction {}
diff --git a/java/udf/src/main/java/com/risingwave/functions/ScalarFunctionBatch.java b/java/udf/src/main/java/com/risingwave/functions/ScalarFunctionBatch.java
deleted file mode 100644
index 5d837d3b370f9..0000000000000
--- a/java/udf/src/main/java/com/risingwave/functions/ScalarFunctionBatch.java
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.functions;
-
-import java.lang.invoke.MethodHandle;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.function.Function;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-
-/** Batch-processing wrapper over a user-defined scalar function. */
-class ScalarFunctionBatch extends UserDefinedFunctionBatch {
-    ScalarFunction function;
-    MethodHandle methodHandle;
-    Function<Object, Object>[] processInputs;
-
-    ScalarFunctionBatch(ScalarFunction function) {
-        this.function = function;
-        var method = Reflection.getEvalMethod(function);
-        this.methodHandle = Reflection.getMethodHandle(method);
-        this.inputSchema = TypeUtils.methodToInputSchema(method);
-        this.outputSchema = TypeUtils.methodToOutputSchema(method);
-        this.processInputs = TypeUtils.methodToProcessInputs(method);
-    }
-
-    @Override
-    Iterator<VectorSchemaRoot> evalBatch(VectorSchemaRoot batch, BufferAllocator allocator) {
-        var row = new Object[batch.getSchema().getFields().size() + 1];
-        row[0] = this.function;
-        var outputValues = new Object[batch.getRowCount()];
-        for (int i = 0; i < batch.getRowCount(); i++) {
-            for (int j = 0; j < row.length - 1; j++) {
-                var val = batch.getVector(j).getObject(i);
-                row[j + 1] = this.processInputs[j].apply(val);
-            }
-            try {
-                outputValues[i] = this.methodHandle.invokeWithArguments(row);
-            } catch (Throwable e) {
-                throw new RuntimeException(e);
-            }
-        }
-        var outputVector =
-                TypeUtils.createVector(
-                        this.outputSchema.getFields().get(0), allocator, outputValues);
-        var outputBatch = VectorSchemaRoot.of(outputVector);
-        return Collections.singleton(outputBatch).iterator();
-    }
-}
diff --git a/java/udf/src/main/java/com/risingwave/functions/TableFunction.java b/java/udf/src/main/java/com/risingwave/functions/TableFunction.java
deleted file mode 100644
index ec5b9d214553f..0000000000000
--- a/java/udf/src/main/java/com/risingwave/functions/TableFunction.java
+++ /dev/null
@@ -1,60 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.functions;
-
-/**
- * Base interface for a user-defined table function. A user-defined table function maps zero, one,
- * or multiple scalar values to zero, one, or multiple rows (or structured types). If an output
- * record consists of only one field, the structured record can be omitted, and a scalar value can
- * be emitted that will be implicitly wrapped into a row by the runtime.
- *
- * <p>The behavior of a {@link TableFunction} can be defined by implementing a custom evaluation
- * method. An evaluation method must be declared publicly, not static, and named <code>eval</code>.
- * The return type must be an Iterator. Multiple overloaded methods named <code>eval</code> are not
- * supported yet.
- *
- * <p>By default, input and output data types are automatically extracted using reflection.
- *
- * <p>The following examples show how to specify a table function:
- *
- * <pre>{@code
- * // a function that accepts an INT arguments and emits the range from 0 to the
- * // given number.
- * class Series implements TableFunction {
- *     public Iterator<Integer> eval(int n) {
- *         return java.util.stream.IntStream.range(0, n).iterator();
- *     }
- * }
- *
- * // a function that accepts an String arguments and emits the words of the
- * // given string.
- * class Split implements TableFunction {
- *     public static class Row {
- *         public String word;
- *         public int length;
- *     }
- *
- *     public Iterator<Row> eval(String str) {
- *         return Stream.of(str.split(" ")).map(s -> {
- *             Row row = new Row();
- *             row.word = s;
- *             row.length = s.length();
- *             return row;
- *         }).iterator();
- *     }
- * }
- * }</pre>
- */
-public interface TableFunction extends UserDefinedFunction {}
diff --git a/java/udf/src/main/java/com/risingwave/functions/TableFunctionBatch.java b/java/udf/src/main/java/com/risingwave/functions/TableFunctionBatch.java
deleted file mode 100644
index a0e0608e60210..0000000000000
--- a/java/udf/src/main/java/com/risingwave/functions/TableFunctionBatch.java
+++ /dev/null
@@ -1,87 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.functions;
-
-import java.lang.invoke.MethodHandle;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.function.Function;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-
-/** Batch-processing wrapper over a user-defined table function. */
-class TableFunctionBatch extends UserDefinedFunctionBatch {
-    TableFunction function;
-    MethodHandle methodHandle;
-    Function<Object, Object>[] processInputs;
-    int chunkSize = 1024;
-
-    TableFunctionBatch(TableFunction function) {
-        this.function = function;
-        var method = Reflection.getEvalMethod(function);
-        this.methodHandle = Reflection.getMethodHandle(method);
-        this.inputSchema = TypeUtils.methodToInputSchema(method);
-        this.outputSchema = TypeUtils.tableFunctionToOutputSchema(method);
-        this.processInputs = TypeUtils.methodToProcessInputs(method);
-    }
-
-    @Override
-    Iterator<VectorSchemaRoot> evalBatch(VectorSchemaRoot batch, BufferAllocator allocator) {
-        var outputs = new ArrayList<VectorSchemaRoot>();
-        var row = new Object[batch.getSchema().getFields().size() + 1];
-        row[0] = this.function;
-        var indexes = new ArrayList<Integer>();
-        var values = new ArrayList<Object>();
-        Runnable buildChunk =
-                () -> {
-                    var fields = this.outputSchema.getFields();
-                    var indexVector =
-                            TypeUtils.createVector(fields.get(0), allocator, indexes.toArray());
-                    var valueVector =
-                            TypeUtils.createVector(fields.get(1), allocator, values.toArray());
-                    indexes.clear();
-                    values.clear();
-                    var outputBatch = VectorSchemaRoot.of(indexVector, valueVector);
-                    outputs.add(outputBatch);
-                };
-        for (int i = 0; i < batch.getRowCount(); i++) {
-            // prepare input row
-            for (int j = 0; j < row.length - 1; j++) {
-                var val = batch.getVector(j).getObject(i);
-                row[j + 1] = this.processInputs[j].apply(val);
-            }
-            // call function
-            Iterator<?> iterator;
-            try {
-                iterator = (Iterator<?>) this.methodHandle.invokeWithArguments(row);
-            } catch (Throwable e) {
-                throw new RuntimeException(e);
-            }
-            // push values
-            while (iterator.hasNext()) {
-                indexes.add(i);
-                values.add(iterator.next());
-                // check if we need to flush
-                if (indexes.size() >= this.chunkSize) {
-                    buildChunk.run();
-                }
-            }
-        }
-        if (indexes.size() > 0) {
-            buildChunk.run();
-        }
-        return outputs.iterator();
-    }
-}
diff --git a/java/udf/src/main/java/com/risingwave/functions/TypeUtils.java b/java/udf/src/main/java/com/risingwave/functions/TypeUtils.java
deleted file mode 100644
index 06c2f79858c40..0000000000000
--- a/java/udf/src/main/java/com/risingwave/functions/TypeUtils.java
+++ /dev/null
@@ -1,505 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.functions;
-
-import java.lang.invoke.MethodHandles;
-import java.lang.reflect.Array;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.ParameterizedType;
-import java.math.BigDecimal;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.LocalTime;
-import java.util.AbstractMap;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.ListVector;
-import org.apache.arrow.vector.complex.StructVector;
-import org.apache.arrow.vector.types.*;
-import org.apache.arrow.vector.types.pojo.*;
-
-class TypeUtils {
-    /** Convert a string to an Arrow type. */
-    static Field stringToField(String typeStr, String name) {
-        typeStr = typeStr.toUpperCase();
-        if (typeStr.equals("BOOLEAN") || typeStr.equals("BOOL")) {
-            return Field.nullable(name, new ArrowType.Bool());
-        } else if (typeStr.equals("SMALLINT") || typeStr.equals("INT2")) {
-            return Field.nullable(name, new ArrowType.Int(16, true));
-        } else if (typeStr.equals("INT") || typeStr.equals("INTEGER") || typeStr.equals("INT4")) {
-            return Field.nullable(name, new ArrowType.Int(32, true));
-        } else if (typeStr.equals("BIGINT") || typeStr.equals("INT8")) {
-            return Field.nullable(name, new ArrowType.Int(64, true));
-        } else if (typeStr.equals("FLOAT4") || typeStr.equals("REAL")) {
-            return Field.nullable(name, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE));
-        } else if (typeStr.equals("FLOAT8") || typeStr.equals("DOUBLE PRECISION")) {
-            return Field.nullable(name, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE));
-        } else if (typeStr.equals("DECIMAL") || typeStr.equals("NUMERIC")) {
-            return Field.nullable(name, new ArrowType.LargeBinary());
-        } else if (typeStr.equals("DATE")) {
-            return Field.nullable(name, new ArrowType.Date(DateUnit.DAY));
-        } else if (typeStr.equals("TIME") || typeStr.equals("TIME WITHOUT TIME ZONE")) {
-            return Field.nullable(name, new ArrowType.Time(TimeUnit.MICROSECOND, 64));
-        } else if (typeStr.equals("TIMESTAMP") || typeStr.equals("TIMESTAMP WITHOUT TIME ZONE")) {
-            return Field.nullable(name, new ArrowType.Timestamp(TimeUnit.MICROSECOND, null));
-        } else if (typeStr.startsWith("INTERVAL")) {
-            return Field.nullable(name, new ArrowType.Interval(IntervalUnit.MONTH_DAY_NANO));
-        } else if (typeStr.equals("VARCHAR")) {
-            return Field.nullable(name, new ArrowType.Utf8());
-        } else if (typeStr.equals("JSONB")) {
-            return Field.nullable(name, new ArrowType.LargeUtf8());
-        } else if (typeStr.equals("BYTEA")) {
-            return Field.nullable(name, new ArrowType.Binary());
-        } else if (typeStr.endsWith("[]")) {
-            Field innerField = stringToField(typeStr.substring(0, typeStr.length() - 2), "");
-            return new Field(
-                    name, FieldType.nullable(new ArrowType.List()), Arrays.asList(innerField));
-        } else if (typeStr.startsWith("STRUCT")) {
-            // extract "STRUCT<INT, VARCHAR, ...>"
-            var typeList = typeStr.substring(7, typeStr.length() - 1);
-            var fields =
-                    Arrays.stream(typeList.split(","))
-                            .map(s -> stringToField(s.trim(), ""))
-                            .collect(Collectors.toList());
-            return new Field(name, FieldType.nullable(new ArrowType.Struct()), fields);
-        } else {
-            throw new IllegalArgumentException("Unsupported type: " + typeStr);
-        }
-    }
-
-    /**
-     * Convert a Java class to an Arrow type.
-     *
-     * @param param The Java class.
-     * @param hint An optional DataTypeHint annotation.
-     * @param name The name of the field.
-     * @return The Arrow type.
-     */
-    static Field classToField(Class<?> param, DataTypeHint hint, String name) {
-        if (hint != null) {
-            return stringToField(hint.value(), name);
-        } else if (param == Boolean.class || param == boolean.class) {
-            return Field.nullable(name, new ArrowType.Bool());
-        } else if (param == Short.class || param == short.class) {
-            return Field.nullable(name, new ArrowType.Int(16, true));
-        } else if (param == Integer.class || param == int.class) {
-            return Field.nullable(name, new ArrowType.Int(32, true));
-        } else if (param == Long.class || param == long.class) {
-            return Field.nullable(name, new ArrowType.Int(64, true));
-        } else if (param == Float.class || param == float.class) {
-            return Field.nullable(name, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE));
-        } else if (param == Double.class || param == double.class) {
-            return Field.nullable(name, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE));
-        } else if (param == BigDecimal.class) {
-            return Field.nullable(name, new ArrowType.LargeBinary());
-        } else if (param == LocalDate.class) {
-            return Field.nullable(name, new ArrowType.Date(DateUnit.DAY));
-        } else if (param == LocalTime.class) {
-            return Field.nullable(name, new ArrowType.Time(TimeUnit.MICROSECOND, 64));
-        } else if (param == LocalDateTime.class) {
-            return Field.nullable(name, new ArrowType.Timestamp(TimeUnit.MICROSECOND, null));
-        } else if (param == PeriodDuration.class) {
-            return Field.nullable(name, new ArrowType.Interval(IntervalUnit.MONTH_DAY_NANO));
-        } else if (param == String.class) {
-            return Field.nullable(name, new ArrowType.Utf8());
-        } else if (param == byte[].class) {
-            return Field.nullable(name, new ArrowType.Binary());
-        } else if (param.isArray()) {
-            var innerField = classToField(param.getComponentType(), null, "");
-            return new Field(
-                    name, FieldType.nullable(new ArrowType.List()), Arrays.asList(innerField));
-        } else {
-            // struct type
-            var fields = new ArrayList<Field>();
-            for (var field : param.getDeclaredFields()) {
-                var subhint = field.getAnnotation(DataTypeHint.class);
-                fields.add(classToField(field.getType(), subhint, field.getName()));
-            }
-            return new Field(name, FieldType.nullable(new ArrowType.Struct()), fields);
-            // TODO: more types
-            // throw new IllegalArgumentException("Unsupported type: " + param);
-        }
-    }
-
-    /** Get the input schema from a Java method. */
-    static Schema methodToInputSchema(Method method) {
-        var fields = new ArrayList<Field>();
-        for (var param : method.getParameters()) {
-            var hint = param.getAnnotation(DataTypeHint.class);
-            fields.add(classToField(param.getType(), hint, param.getName()));
-        }
-        return new Schema(fields);
-    }
-
-    /** Get the output schema of a scalar function from a Java method. */
-    static Schema methodToOutputSchema(Method method) {
-        var type = method.getReturnType();
-        var hint = method.getAnnotation(DataTypeHint.class);
-        return new Schema(Arrays.asList(classToField(type, hint, "")));
-    }
-
-    /** Get the output schema of a table function from a Java class. */
-    static Schema tableFunctionToOutputSchema(Method method) {
-        var hint = method.getAnnotation(DataTypeHint.class);
-        var type = method.getReturnType();
-        if (!Iterator.class.isAssignableFrom(type)) {
-            throw new IllegalArgumentException("Table function must return Iterator");
-        }
-        var typeArguments =
-                ((ParameterizedType) method.getGenericReturnType()).getActualTypeArguments();
-        type = (Class<?>) typeArguments[0];
-        var rowIndex = Field.nullable("row_index", new ArrowType.Int(32, true));
-        return new Schema(Arrays.asList(rowIndex, classToField(type, hint, "")));
-    }
-
-    /** Return functions to process input values from a Java method. */
-    static Function<Object, Object>[] methodToProcessInputs(Method method) {
-        var schema = methodToInputSchema(method);
-        var params = method.getParameters();
-        @SuppressWarnings("unchecked")
-        Function<Object, Object>[] funcs = new Function[schema.getFields().size()];
-        for (int i = 0; i < schema.getFields().size(); i++) {
-            funcs[i] = processFunc(schema.getFields().get(i), params[i].getType());
-        }
-        return funcs;
-    }
-
-    /** Create an Arrow vector from an array of values. */
-    static FieldVector createVector(Field field, BufferAllocator allocator, Object[] values) {
-        var vector = field.createVector(allocator);
-        fillVector(vector, values);
-        return vector;
-    }
-
-    /** Fill an Arrow vector with an array of values. */
-    static void fillVector(FieldVector fieldVector, Object[] values) {
-        if (fieldVector instanceof BitVector) {
-            var vector = (BitVector) fieldVector;
-            vector.allocateNew(values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.set(i, (boolean) values[i] ? 1 : 0);
-                }
-            }
-        } else if (fieldVector instanceof SmallIntVector) {
-            var vector = (SmallIntVector) fieldVector;
-            vector.allocateNew(values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.set(i, (short) values[i]);
-                }
-            }
-        } else if (fieldVector instanceof IntVector) {
-            var vector = (IntVector) fieldVector;
-            vector.allocateNew(values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.set(i, (int) values[i]);
-                }
-            }
-        } else if (fieldVector instanceof BigIntVector) {
-            var vector = (BigIntVector) fieldVector;
-            vector.allocateNew(values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.set(i, (long) values[i]);
-                }
-            }
-        } else if (fieldVector instanceof Float4Vector) {
-            var vector = (Float4Vector) fieldVector;
-            vector.allocateNew(values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.set(i, (float) values[i]);
-                }
-            }
-        } else if (fieldVector instanceof Float8Vector) {
-            var vector = (Float8Vector) fieldVector;
-            vector.allocateNew(values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.set(i, (double) values[i]);
-                }
-            }
-        } else if (fieldVector instanceof LargeVarBinaryVector) {
-            var vector = (LargeVarBinaryVector) fieldVector;
-            vector.allocateNew(values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    // use `toPlainString` to avoid scientific notation
-                    vector.set(i, ((BigDecimal) values[i]).toPlainString().getBytes());
-                }
-            }
-        } else if (fieldVector instanceof DateDayVector) {
-            var vector = (DateDayVector) fieldVector;
-            vector.allocateNew(values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.set(i, (int) ((LocalDate) values[i]).toEpochDay());
-                }
-            }
-        } else if (fieldVector instanceof TimeMicroVector) {
-            var vector = (TimeMicroVector) fieldVector;
-            vector.allocateNew(values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.set(i, ((LocalTime) values[i]).toNanoOfDay() / 1000);
-                }
-            }
-        } else if (fieldVector instanceof TimeStampMicroVector) {
-            var vector = (TimeStampMicroVector) fieldVector;
-            vector.allocateNew(values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.set(i, timestampToMicros((LocalDateTime) values[i]));
-                }
-            }
-        } else if (fieldVector instanceof IntervalMonthDayNanoVector) {
-            var vector = (IntervalMonthDayNanoVector) fieldVector;
-            vector.allocateNew(values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    var pd = (PeriodDuration) values[i];
-                    var months = (int) pd.getPeriod().toTotalMonths();
-                    var days = pd.getPeriod().getDays();
-                    var nanos = pd.getDuration().toNanos();
-                    vector.set(i, months, days, nanos);
-                }
-            }
-        } else if (fieldVector instanceof VarCharVector) {
-            var vector = (VarCharVector) fieldVector;
-            int totalBytes = 0;
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    totalBytes += ((String) values[i]).length();
-                }
-            }
-            vector.allocateNew(totalBytes, values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.set(i, ((String) values[i]).getBytes());
-                }
-            }
-        } else if (fieldVector instanceof LargeVarCharVector) {
-            var vector = (LargeVarCharVector) fieldVector;
-            int totalBytes = 0;
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    totalBytes += ((String) values[i]).length();
-                }
-            }
-            vector.allocateNew(totalBytes, values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.set(i, ((String) values[i]).getBytes());
-                }
-            }
-        } else if (fieldVector instanceof VarBinaryVector) {
-            var vector = (VarBinaryVector) fieldVector;
-            int totalBytes = 0;
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    totalBytes += ((byte[]) values[i]).length;
-                }
-            }
-            vector.allocateNew(totalBytes, values.length);
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.set(i, (byte[]) values[i]);
-                }
-            }
-        } else if (fieldVector instanceof ListVector) {
-            var vector = (ListVector) fieldVector;
-            vector.allocateNew();
-            // flatten the `values`
-            var flattenLength = 0;
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] == null) {
-                    continue;
-                }
-                var len = Array.getLength(values[i]);
-                vector.startNewValue(i);
-                vector.endValue(i, len);
-                flattenLength += len;
-            }
-            var flattenValues = new Object[flattenLength];
-            var ii = 0;
-            for (var list : values) {
-                if (list == null) {
-                    continue;
-                }
-                var length = Array.getLength(list);
-                for (int i = 0; i < length; i++) {
-                    flattenValues[ii++] = Array.get(list, i);
-                }
-            }
-            // fill the inner vector
-            fillVector(vector.getDataVector(), flattenValues);
-        } else if (fieldVector instanceof StructVector) {
-            var vector = (StructVector) fieldVector;
-            vector.allocateNew();
-            var lookup = MethodHandles.lookup();
-            // get class of the first non-null value
-            Class<?> valueClass = null;
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    valueClass = values[i].getClass();
-                    break;
-                }
-            }
-            for (var field : vector.getField().getChildren()) {
-                // extract field from values
-                var subvalues = new Object[values.length];
-                if (valueClass != null) {
-                    try {
-                        var javaField = valueClass.getDeclaredField(field.getName());
-                        var varHandle = lookup.unreflectVarHandle(javaField);
-                        for (int i = 0; i < values.length; i++) {
-                            if (values[i] != null) {
-                                subvalues[i] = varHandle.get(values[i]);
-                            }
-                        }
-                    } catch (NoSuchFieldException | IllegalAccessException e) {
-                        throw new RuntimeException(e);
-                    }
-                }
-                var subvector = vector.getChild(field.getName());
-                fillVector(subvector, subvalues);
-            }
-            for (int i = 0; i < values.length; i++) {
-                if (values[i] != null) {
-                    vector.setIndexDefined(i);
-                }
-            }
-        } else {
-            throw new IllegalArgumentException("Unsupported type: " + fieldVector.getClass());
-        }
-        fieldVector.setValueCount(values.length);
-    }
-
-    static long timestampToMicros(LocalDateTime timestamp) {
-        var date = timestamp.toLocalDate().toEpochDay();
-        var time = timestamp.toLocalTime().toNanoOfDay();
-        return date * 24 * 3600 * 1000 * 1000 + time / 1000;
-    }
-
-    /** Return a function that converts the object get from input array to the correct type. */
-    static Function<Object, Object> processFunc(Field field, Class<?> targetClass) {
-        var inner = processFunc0(field, targetClass);
-        return obj -> obj == null ? null : inner.apply(obj);
-    }
-
-    static Function<Object, Object> processFunc0(Field field, Class<?> targetClass) {
-        if (field.getType() instanceof ArrowType.Utf8 && targetClass == String.class) {
-            // object is org.apache.arrow.vector.util.Text
-            return obj -> obj.toString();
-        } else if (field.getType() instanceof ArrowType.LargeUtf8 && targetClass == String.class) {
-            // object is org.apache.arrow.vector.util.Text
-            return obj -> obj.toString();
-        } else if (field.getType() instanceof ArrowType.LargeBinary
-                && targetClass == BigDecimal.class) {
-            // object is byte[]
-            return obj -> new BigDecimal(new String((byte[]) obj));
-        } else if (field.getType() instanceof ArrowType.Date && targetClass == LocalDate.class) {
-            // object is Integer
-            return obj -> LocalDate.ofEpochDay((int) obj);
-        } else if (field.getType() instanceof ArrowType.Time && targetClass == LocalTime.class) {
-            // object is Long
-            return obj -> LocalTime.ofNanoOfDay((long) obj * 1000);
-        } else if (field.getType() instanceof ArrowType.Interval
-                && targetClass == PeriodDuration.class) {
-            // object is arrow PeriodDuration
-            return obj -> new PeriodDuration((org.apache.arrow.vector.PeriodDuration) obj);
-        } else if (field.getType() instanceof ArrowType.List) {
-            // object is List
-            var subfield = field.getChildren().get(0);
-            var subfunc = processFunc(subfield, targetClass.getComponentType());
-            if (subfield.getType() instanceof ArrowType.Bool) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(Boolean[]::new);
-            } else if (subfield.getType().equals(new ArrowType.Int(16, true))) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(Short[]::new);
-            } else if (subfield.getType().equals(new ArrowType.Int(32, true))) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(Integer[]::new);
-            } else if (subfield.getType().equals(new ArrowType.Int(64, true))) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(Long[]::new);
-            } else if (subfield.getType()
-                    .equals(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE))) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(Float[]::new);
-            } else if (subfield.getType()
-                    .equals(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE))) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(Double[]::new);
-            } else if (subfield.getType() instanceof ArrowType.LargeBinary) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(BigDecimal[]::new);
-            } else if (subfield.getType() instanceof ArrowType.Date) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(LocalDate[]::new);
-            } else if (subfield.getType() instanceof ArrowType.Time) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(LocalTime[]::new);
-            } else if (subfield.getType() instanceof ArrowType.Timestamp) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(LocalDateTime[]::new);
-            } else if (subfield.getType() instanceof ArrowType.Interval) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(PeriodDuration[]::new);
-            } else if (subfield.getType() instanceof ArrowType.Utf8) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(String[]::new);
-            } else if (subfield.getType() instanceof ArrowType.LargeUtf8) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(String[]::new);
-            } else if (subfield.getType() instanceof ArrowType.Binary) {
-                return obj -> ((List<?>) obj).stream().map(subfunc).toArray(byte[][]::new);
-            } else if (subfield.getType() instanceof ArrowType.Struct) {
-                return obj -> {
-                    var list = (List<?>) obj;
-                    Object array = Array.newInstance(targetClass.getComponentType(), list.size());
-                    for (int i = 0; i < list.size(); i++) {
-                        Array.set(array, i, subfunc.apply(list.get(i)));
-                    }
-                    return array;
-                };
-            }
-            throw new IllegalArgumentException("Unsupported type: " + subfield.getType());
-        } else if (field.getType() instanceof ArrowType.Struct) {
-            // object is org.apache.arrow.vector.util.JsonStringHashMap
-            var subfields = field.getChildren();
-            @SuppressWarnings("unchecked")
-            Function<Object, Object>[] subfunc = new Function[subfields.size()];
-            for (int i = 0; i < subfields.size(); i++) {
-                subfunc[i] = processFunc(subfields.get(i), targetClass.getFields()[i].getType());
-            }
-            return obj -> {
-                var map = (AbstractMap<?, ?>) obj;
-                try {
-                    var row = targetClass.getDeclaredConstructor().newInstance();
-                    for (int i = 0; i < subfields.size(); i++) {
-                        var field0 = targetClass.getFields()[i];
-                        var val = subfunc[i].apply(map.get(field0.getName()));
-                        field0.set(row, val);
-                    }
-                    return row;
-                } catch (InstantiationException
-                        | IllegalAccessException
-                        | InvocationTargetException
-                        | NoSuchMethodException e) {
-                    throw new RuntimeException(e);
-                }
-            };
-        }
-        return Function.identity();
-    }
-}
diff --git a/java/udf/src/main/java/com/risingwave/functions/UdfProducer.java b/java/udf/src/main/java/com/risingwave/functions/UdfProducer.java
deleted file mode 100644
index 692d898acaf8a..0000000000000
--- a/java/udf/src/main/java/com/risingwave/functions/UdfProducer.java
+++ /dev/null
@@ -1,108 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.functions;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import org.apache.arrow.flight.*;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.VectorLoader;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.Schema;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-class UdfProducer extends NoOpFlightProducer {
-
-    private BufferAllocator allocator;
-    private HashMap<String, UserDefinedFunctionBatch> functions = new HashMap<>();
-    private static final Logger logger = LoggerFactory.getLogger(UdfServer.class);
-
-    UdfProducer(BufferAllocator allocator) {
-        this.allocator = allocator;
-    }
-
-    void addFunction(String name, UserDefinedFunction function) throws IllegalArgumentException {
-        UserDefinedFunctionBatch udf;
-        if (function instanceof ScalarFunction) {
-            udf = new ScalarFunctionBatch((ScalarFunction) function);
-        } else if (function instanceof TableFunction) {
-            udf = new TableFunctionBatch((TableFunction) function);
-        } else {
-            throw new IllegalArgumentException(
-                    "Unknown function type: " + function.getClass().getName());
-        }
-        if (functions.containsKey(name)) {
-            throw new IllegalArgumentException("Function already exists: " + name);
-        }
-        functions.put(name, udf);
-    }
-
-    @Override
-    public FlightInfo getFlightInfo(CallContext context, FlightDescriptor descriptor) {
-        try {
-            var functionName = descriptor.getPath().get(0);
-            var udf = functions.get(functionName);
-            if (udf == null) {
-                throw new IllegalArgumentException("Unknown function: " + functionName);
-            }
-            var fields = new ArrayList<Field>();
-            fields.addAll(udf.getInputSchema().getFields());
-            fields.addAll(udf.getOutputSchema().getFields());
-            var fullSchema = new Schema(fields);
-            var inputLen = udf.getInputSchema().getFields().size();
-
-            return new FlightInfo(fullSchema, descriptor, Collections.emptyList(), 0, inputLen);
-        } catch (Exception e) {
-            logger.error("Error occurred during getFlightInfo", e);
-            throw e;
-        }
-    }
-
-    @Override
-    public void doExchange(CallContext context, FlightStream reader, ServerStreamListener writer) {
-        try (var allocator = this.allocator.newChildAllocator("exchange", 0, Long.MAX_VALUE)) {
-            var functionName = reader.getDescriptor().getPath().get(0);
-            logger.debug("call function: " + functionName);
-
-            var udf = this.functions.get(functionName);
-            try (var root = VectorSchemaRoot.create(udf.getOutputSchema(), allocator)) {
-                var loader = new VectorLoader(root);
-                writer.start(root);
-                while (reader.next()) {
-                    try (var input = reader.getRoot()) {
-                        var outputBatches = udf.evalBatch(input, allocator);
-                        while (outputBatches.hasNext()) {
-                            try (var outputRoot = outputBatches.next()) {
-                                var unloader = new VectorUnloader(outputRoot);
-                                try (var outputBatch = unloader.getRecordBatch()) {
-                                    loader.load(outputBatch);
-                                }
-                            }
-                            writer.putNext();
-                        }
-                    }
-                }
-                writer.completed();
-            }
-        } catch (Exception e) {
-            logger.error("Error occurred during UDF execution", e);
-            writer.error(e);
-        }
-    }
-}
diff --git a/java/udf/src/main/java/com/risingwave/functions/UdfServer.java b/java/udf/src/main/java/com/risingwave/functions/UdfServer.java
deleted file mode 100644
index 66f2a8d3bb0dd..0000000000000
--- a/java/udf/src/main/java/com/risingwave/functions/UdfServer.java
+++ /dev/null
@@ -1,81 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.functions;
-
-import java.io.IOException;
-import org.apache.arrow.flight.*;
-import org.apache.arrow.memory.RootAllocator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** A server that exposes user-defined functions over Apache Arrow Flight. */
-public class UdfServer implements AutoCloseable {
-
-    private FlightServer server;
-    private UdfProducer producer;
-    private static final Logger logger = LoggerFactory.getLogger(UdfServer.class);
-
-    public UdfServer(String host, int port) {
-        var location = Location.forGrpcInsecure(host, port);
-        var allocator = new RootAllocator();
-        this.producer = new UdfProducer(allocator);
-        this.server = FlightServer.builder(allocator, location, this.producer).build();
-    }
-
-    /**
-     * Add a user-defined function to the server.
-     *
-     * @param name the name of the function
-     * @param udf the function to add
-     * @throws IllegalArgumentException if a function with the same name already exists
-     */
-    public void addFunction(String name, UserDefinedFunction udf) throws IllegalArgumentException {
-        logger.info("added function: " + name);
-        this.producer.addFunction(name, udf);
-    }
-
-    /**
-     * Start the server.
-     *
-     * @throws IOException if the server fails to start
-     */
-    public void start() throws IOException {
-        this.server.start();
-        logger.info("listening on " + this.server.getLocation().toSocketAddress());
-    }
-
-    /**
-     * Get the port the server is listening on.
-     *
-     * @return the port number
-     */
-    public int getPort() {
-        return this.server.getPort();
-    }
-
-    /**
-     * Wait for the server to terminate.
-     *
-     * @throws InterruptedException if the thread is interrupted while waiting
-     */
-    public void awaitTermination() throws InterruptedException {
-        this.server.awaitTermination();
-    }
-
-    /** Close the server. */
-    public void close() throws InterruptedException {
-        this.server.close();
-    }
-}
diff --git a/java/udf/src/main/java/com/risingwave/functions/UserDefinedFunction.java b/java/udf/src/main/java/com/risingwave/functions/UserDefinedFunction.java
deleted file mode 100644
index 3db6f1714cd83..0000000000000
--- a/java/udf/src/main/java/com/risingwave/functions/UserDefinedFunction.java
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.functions;
-
-/**
- * Base interface for all user-defined functions.
- *
- * @see ScalarFunction
- * @see TableFunction
- */
-public interface UserDefinedFunction {}
diff --git a/java/udf/src/main/java/com/risingwave/functions/UserDefinedFunctionBatch.java b/java/udf/src/main/java/com/risingwave/functions/UserDefinedFunctionBatch.java
deleted file mode 100644
index e2c513a7954ad..0000000000000
--- a/java/udf/src/main/java/com/risingwave/functions/UserDefinedFunctionBatch.java
+++ /dev/null
@@ -1,87 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.functions;
-
-import java.lang.invoke.MethodHandle;
-import java.lang.invoke.MethodHandles;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.util.ArrayList;
-import java.util.Iterator;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.types.pojo.Schema;
-
-/** Base class for a batch-processing user-defined function. */
-abstract class UserDefinedFunctionBatch {
-    protected Schema inputSchema;
-    protected Schema outputSchema;
-
-    /** Get the input schema of the function. */
-    Schema getInputSchema() {
-        return inputSchema;
-    }
-
-    /** Get the output schema of the function. */
-    Schema getOutputSchema() {
-        return outputSchema;
-    }
-
-    /**
-     * Evaluate the function by processing a batch of input data.
-     *
-     * @param batch the input data batch to process
-     * @param allocator the allocator to use for allocating output data
-     * @return an iterator over the output data batches
-     */
-    abstract Iterator<VectorSchemaRoot> evalBatch(
-            VectorSchemaRoot batch, BufferAllocator allocator);
-}
-
-/** Utility class for reflection. */
-class Reflection {
-    /** Get the method named <code>eval</code>. */
-    static Method getEvalMethod(UserDefinedFunction obj) {
-        var methods = new ArrayList<Method>();
-        for (Method method : obj.getClass().getDeclaredMethods()) {
-            if (method.getName().equals("eval")) {
-                methods.add(method);
-            }
-        }
-        if (methods.size() != 1) {
-            throw new IllegalArgumentException(
-                    "Exactly one eval method must be defined for class "
-                            + obj.getClass().getName());
-        }
-        var method = methods.get(0);
-        if (Modifier.isStatic(method.getModifiers())) {
-            throw new IllegalArgumentException(
-                    "The eval method should not be static for class " + obj.getClass().getName());
-        }
-        return method;
-    }
-
-    /** Get the method handle of the given method. */
-    static MethodHandle getMethodHandle(Method method) {
-        var lookup = MethodHandles.lookup();
-        try {
-            return lookup.unreflect(method);
-        } catch (IllegalAccessException e) {
-            throw new IllegalArgumentException(
-                    "The eval method must be public for class "
-                            + method.getDeclaringClass().getName());
-        }
-    }
-}
diff --git a/java/udf/src/test/java/com/risingwave/functions/TestUdfServer.java b/java/udf/src/test/java/com/risingwave/functions/TestUdfServer.java
deleted file mode 100644
index 5722efa1dd702..0000000000000
--- a/java/udf/src/test/java/com/risingwave/functions/TestUdfServer.java
+++ /dev/null
@@ -1,286 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.functions;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.LocalTime;
-import java.util.Iterator;
-import java.util.stream.IntStream;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.*;
-import org.apache.arrow.vector.complex.StructVector;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.FieldType;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-
-/** Unit test for UDF server. */
-public class TestUdfServer {
-    private static UdfClient client;
-    private static UdfServer server;
-    private static BufferAllocator allocator = new RootAllocator();
-
-    @BeforeAll
-    public static void setup() throws IOException {
-        server = new UdfServer("localhost", 0);
-        server.addFunction("gcd", new Gcd());
-        server.addFunction("return_all", new ReturnAll());
-        server.addFunction("series", new Series());
-        server.start();
-
-        client = new UdfClient("localhost", server.getPort());
-    }
-
-    @AfterAll
-    public static void teardown() throws InterruptedException {
-        client.close();
-        server.close();
-    }
-
-    public static class Gcd implements ScalarFunction {
-        public int eval(int a, int b) {
-            while (b != 0) {
-                int temp = b;
-                b = a % b;
-                a = temp;
-            }
-            return a;
-        }
-    }
-
-    @Test
-    public void gcd() throws Exception {
-        var c0 = new IntVector("", allocator);
-        c0.allocateNew(1);
-        c0.set(0, 15);
-        c0.setValueCount(1);
-
-        var c1 = new IntVector("", allocator);
-        c1.allocateNew(1);
-        c1.set(0, 12);
-        c1.setValueCount(1);
-
-        var input = VectorSchemaRoot.of(c0, c1);
-
-        try (var stream = client.call("gcd", input)) {
-            var output = stream.getRoot();
-            assertTrue(stream.next());
-            assertEquals("3", output.contentToTSVString().trim());
-        }
-    }
-
-    public static class ReturnAll implements ScalarFunction {
-        public static class Row {
-            public Boolean bool;
-            public Short i16;
-            public Integer i32;
-            public Long i64;
-            public Float f32;
-            public Double f64;
-            public BigDecimal decimal;
-            public LocalDate date;
-            public LocalTime time;
-            public LocalDateTime timestamp;
-            public PeriodDuration interval;
-            public String str;
-            public byte[] bytes;
-            public @DataTypeHint("JSONB") String jsonb;
-            public Struct struct;
-        }
-
-        public static class Struct {
-            public Integer f1;
-            public Integer f2;
-
-            public String toString() {
-                return String.format("(%d, %d)", f1, f2);
-            }
-        }
-
-        public Row eval(
-                Boolean bool,
-                Short i16,
-                Integer i32,
-                Long i64,
-                Float f32,
-                Double f64,
-                BigDecimal decimal,
-                LocalDate date,
-                LocalTime time,
-                LocalDateTime timestamp,
-                PeriodDuration interval,
-                String str,
-                byte[] bytes,
-                @DataTypeHint("JSONB") String jsonb,
-                Struct struct) {
-            var row = new Row();
-            row.bool = bool;
-            row.i16 = i16;
-            row.i32 = i32;
-            row.i64 = i64;
-            row.f32 = f32;
-            row.f64 = f64;
-            row.decimal = decimal;
-            row.date = date;
-            row.time = time;
-            row.timestamp = timestamp;
-            row.interval = interval;
-            row.str = str;
-            row.bytes = bytes;
-            row.jsonb = jsonb;
-            row.struct = struct;
-            return row;
-        }
-    }
-
-    @Test
-    public void all_types() throws Exception {
-        var c0 = new BitVector("", allocator);
-        c0.allocateNew(2);
-        c0.set(0, 1);
-        c0.setValueCount(2);
-
-        var c1 = new SmallIntVector("", allocator);
-        c1.allocateNew(2);
-        c1.set(0, 1);
-        c1.setValueCount(2);
-
-        var c2 = new IntVector("", allocator);
-        c2.allocateNew(2);
-        c2.set(0, 1);
-        c2.setValueCount(2);
-
-        var c3 = new BigIntVector("", allocator);
-        c3.allocateNew(2);
-        c3.set(0, 1);
-        c3.setValueCount(2);
-
-        var c4 = new Float4Vector("", allocator);
-        c4.allocateNew(2);
-        c4.set(0, 1);
-        c4.setValueCount(2);
-
-        var c5 = new Float8Vector("", allocator);
-        c5.allocateNew(2);
-        c5.set(0, 1);
-        c5.setValueCount(2);
-
-        var c6 = new LargeVarBinaryVector("", allocator);
-        c6.allocateNew(2);
-        c6.set(0, "1.234".getBytes());
-        c6.setValueCount(2);
-
-        var c7 = new DateDayVector("", allocator);
-        c7.allocateNew(2);
-        c7.set(0, (int) LocalDate.of(2023, 1, 1).toEpochDay());
-        c7.setValueCount(2);
-
-        var c8 = new TimeMicroVector("", allocator);
-        c8.allocateNew(2);
-        c8.set(0, LocalTime.of(1, 2, 3).toNanoOfDay() / 1000);
-        c8.setValueCount(2);
-
-        var c9 = new TimeStampMicroVector("", allocator);
-        c9.allocateNew(2);
-        var ts = LocalDateTime.of(2023, 1, 1, 1, 2, 3);
-        c9.set(
-                0,
-                ts.toLocalDate().toEpochDay() * 24 * 3600 * 1000000
-                        + ts.toLocalTime().toNanoOfDay() / 1000);
-        c9.setValueCount(2);
-
-        var c10 =
-                new IntervalMonthDayNanoVector(
-                        "",
-                        FieldType.nullable(MinorType.INTERVALMONTHDAYNANO.getType()),
-                        allocator);
-        c10.allocateNew(2);
-        c10.set(0, 1000, 2000, 3000);
-        c10.setValueCount(2);
-
-        var c11 = new VarCharVector("", allocator);
-        c11.allocateNew(2);
-        c11.set(0, "string".getBytes());
-        c11.setValueCount(2);
-
-        var c12 = new VarBinaryVector("", allocator);
-        c12.allocateNew(2);
-        c12.set(0, "bytes".getBytes());
-        c12.setValueCount(2);
-
-        var c13 = new LargeVarCharVector("", allocator);
-        c13.allocateNew(2);
-        c13.set(0, "{ key: 1 }".getBytes());
-        c13.setValueCount(2);
-
-        var c14 =
-                new StructVector(
-                        "", allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), null);
-        c14.allocateNew();
-        var f1 = c14.addOrGet("f1", FieldType.nullable(MinorType.INT.getType()), IntVector.class);
-        var f2 = c14.addOrGet("f2", FieldType.nullable(MinorType.INT.getType()), IntVector.class);
-        f1.allocateNew(2);
-        f2.allocateNew(2);
-        f1.set(0, 1);
-        f2.set(0, 2);
-        c14.setIndexDefined(0);
-        c14.setValueCount(2);
-
-        var input =
-                VectorSchemaRoot.of(
-                        c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14);
-
-        try (var stream = client.call("return_all", input)) {
-            var output = stream.getRoot();
-            assertTrue(stream.next());
-            assertEquals(
-                    "{\"bool\":true,\"i16\":1,\"i32\":1,\"i64\":1,\"f32\":1.0,\"f64\":1.0,\"decimal\":\"MS4yMzQ=\",\"date\":19358,\"time\":3723000000,\"timestamp\":[2023,1,1,1,2,3],\"interval\":{\"period\":\"P1000M2000D\",\"duration\":0.000003000},\"str\":\"string\",\"bytes\":\"Ynl0ZXM=\",\"jsonb\":\"{ key: 1 }\",\"struct\":{\"f1\":1,\"f2\":2}}\n{}",
-                    output.contentToTSVString().trim());
-        }
-    }
-
-    public static class Series implements TableFunction {
-        public Iterator<Integer> eval(int n) {
-            return IntStream.range(0, n).iterator();
-        }
-    }
-
-    @Test
-    public void series() throws Exception {
-        var c0 = new IntVector("", allocator);
-        c0.allocateNew(3);
-        c0.set(0, 0);
-        c0.set(1, 1);
-        c0.set(2, 2);
-        c0.setValueCount(3);
-
-        var input = VectorSchemaRoot.of(c0);
-
-        try (var stream = client.call("series", input)) {
-            var output = stream.getRoot();
-            assertTrue(stream.next());
-            assertEquals("row_index\t\n1\t0\n2\t0\n2\t1\n", output.contentToTSVString());
-        }
-    }
-}
diff --git a/java/udf/src/test/java/com/risingwave/functions/UdfClient.java b/java/udf/src/test/java/com/risingwave/functions/UdfClient.java
deleted file mode 100644
index 12728bf64fbec..0000000000000
--- a/java/udf/src/test/java/com/risingwave/functions/UdfClient.java
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.risingwave.functions;
-
-import org.apache.arrow.flight.*;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-
-public class UdfClient implements AutoCloseable {
-
-    private FlightClient client;
-
-    public UdfClient(String host, int port) {
-        var allocator = new RootAllocator();
-        var location = Location.forGrpcInsecure(host, port);
-        this.client = FlightClient.builder(allocator, location).build();
-    }
-
-    public void close() throws InterruptedException {
-        this.client.close();
-    }
-
-    public FlightInfo getFlightInfo(String functionName) {
-        var descriptor = FlightDescriptor.command(functionName.getBytes());
-        return client.getInfo(descriptor);
-    }
-
-    public FlightStream call(String functionName, VectorSchemaRoot root) {
-        var descriptor = FlightDescriptor.path(functionName);
-        var readerWriter = client.doExchange(descriptor);
-        var writer = readerWriter.getWriter();
-        var reader = readerWriter.getReader();
-
-        writer.start(root);
-        writer.putNext();
-        writer.completed();
-        return reader;
-    }
-}
diff --git a/lints/Cargo.lock b/lints/Cargo.lock
index b8fd465fecd7b..3ffb99c6c425e 100644
--- a/lints/Cargo.lock
+++ b/lints/Cargo.lock
@@ -20,6 +20,55 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "anstream"
+version = "0.6.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19"
+dependencies = [
+ "anstyle",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.75"
@@ -40,9 +89,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.4.1"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
+checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
 
 [[package]]
 name = "block-buffer"
@@ -113,8 +162,8 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clippy_config"
-version = "0.1.77"
-source = "git+https://github.com/rust-lang/rust-clippy?rev=6fd0258e45105161b7e759a22e7350958e5cb0b1#6fd0258e45105161b7e759a22e7350958e5cb0b1"
+version = "0.1.79"
+source = "git+https://github.com/rust-lang/rust-clippy?rev=9d6f41691ed9dbfaec2a2df2661c42451f2fe0d3#9d6f41691ed9dbfaec2a2df2661c42451f2fe0d3"
 dependencies = [
  "rustc-semver",
  "serde",
@@ -123,15 +172,21 @@ dependencies = [
 
 [[package]]
 name = "clippy_utils"
-version = "0.1.77"
-source = "git+https://github.com/rust-lang/rust-clippy?rev=6fd0258e45105161b7e759a22e7350958e5cb0b1#6fd0258e45105161b7e759a22e7350958e5cb0b1"
+version = "0.1.79"
+source = "git+https://github.com/rust-lang/rust-clippy?rev=9d6f41691ed9dbfaec2a2df2661c42451f2fe0d3#9d6f41691ed9dbfaec2a2df2661c42451f2fe0d3"
 dependencies = [
  "arrayvec",
  "clippy_config",
- "itertools 0.11.0",
+ "itertools",
  "rustc-semver",
 ]
 
+[[package]]
+name = "colorchoice"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
+
 [[package]]
 name = "compiletest_rs"
 version = "0.10.2"
@@ -233,9 +288,9 @@ dependencies = [
 
 [[package]]
 name = "dylint"
-version = "2.6.0"
+version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71fdb7b800ab13925402f0048ed0911068db2e5ba6168dd93962269d4f39541d"
+checksum = "4c6720f18fdd779ad137ab6bc448c11042b1b32eea625ea618c8b953660bba56"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -254,12 +309,13 @@ dependencies = [
 
 [[package]]
 name = "dylint_internal"
-version = "2.6.0"
+version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5154dada2bee2a69f75f54eae57479f56f93ca1db80725a1d82cdb5fe231ef73"
+checksum = "395dade88bc1a3103ef91b442498943d0072df869736a7c0107a5753c05d006f"
 dependencies = [
  "ansi_term",
  "anyhow",
+ "bitflags 2.5.0",
  "cargo_metadata",
  "git2",
  "home",
@@ -267,15 +323,18 @@ dependencies = [
  "is-terminal",
  "log",
  "once_cell",
+ "regex",
  "rust-embed",
- "sedregex",
+ "serde",
+ "thiserror",
+ "toml 0.8.8",
 ]
 
 [[package]]
 name = "dylint_linting"
-version = "2.6.0"
+version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d203baeb8770847314632f652e0e62dd7fec6a21102a116472eec0d6931f5dd9"
+checksum = "3d070f934310ccf8f04a940affcce0cd196c1068b6d19c5ae6d975f926968b66"
 dependencies = [
  "cargo_metadata",
  "dylint_internal",
@@ -288,9 +347,9 @@ dependencies = [
 
 [[package]]
 name = "dylint_testing"
-version = "2.6.0"
+version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1208b1f2c40fc2f3c3fa0d5631efbc7a95721d619410dc2da5b0496810d6a941"
+checksum = "7fc8a64c781bde7f1e445dc1205754994012e15a2a6e659ab7166b380c1c22ab"
 dependencies = [
  "anyhow",
  "cargo_metadata",
@@ -310,17 +369,27 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 
+[[package]]
+name = "env_filter"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea"
+dependencies = [
+ "log",
+ "regex",
+]
+
 [[package]]
 name = "env_logger"
-version = "0.10.1"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece"
+checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9"
 dependencies = [
+ "anstream",
+ "anstyle",
+ "env_filter",
  "humantime",
- "is-terminal",
  "log",
- "regex",
- "termcolor",
 ]
 
 [[package]]
@@ -402,7 +471,7 @@ version = "0.18.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd"
 dependencies = [
- "bitflags 2.4.1",
+ "bitflags 2.5.0",
  "libc",
  "libgit2-sys",
  "log",
@@ -489,13 +558,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "itertools"
-version = "0.11.0"
+name = "is_terminal_polyfill"
+version = "1.70.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
-dependencies = [
- "either",
-]
+checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
 
 [[package]]
 name = "itertools"
@@ -529,9 +595,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.152"
+version = "0.2.154"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"
+checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346"
 
 [[package]]
 name = "libgit2-sys"
@@ -553,7 +619,7 @@ version = "0.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
 dependencies = [
- "bitflags 2.4.1",
+ "bitflags 2.5.0",
  "libc",
  "redox_syscall 0.4.1",
 ]
@@ -592,7 +658,7 @@ dependencies = [
  "clippy_utils",
  "dylint_linting",
  "dylint_testing",
- "itertools 0.12.0",
+ "itertools",
  "thiserror-ext",
  "tracing",
 ]
@@ -605,9 +671,9 @@ checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
 
 [[package]]
 name = "log"
-version = "0.4.20"
+version = "0.4.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
+checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
 
 [[package]]
 name = "memchr"
@@ -766,9 +832,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
 
 [[package]]
 name = "rust-embed"
-version = "8.2.0"
+version = "8.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a82c0bbc10308ed323529fd3c1dce8badda635aa319a5ff0e6466f33b8101e3f"
+checksum = "19549741604902eb99a7ed0ee177a0663ee1eda51a29f71401f166e47e77806a"
 dependencies = [
  "rust-embed-impl",
  "rust-embed-utils",
@@ -777,9 +843,9 @@ dependencies = [
 
 [[package]]
 name = "rust-embed-impl"
-version = "8.2.0"
+version = "8.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6227c01b1783cdfee1bcf844eb44594cd16ec71c35305bf1c9fb5aade2735e16"
+checksum = "cb9f96e283ec64401f30d3df8ee2aaeb2561f34c824381efa24a35f79bf40ee4"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -790,9 +856,9 @@ dependencies = [
 
 [[package]]
 name = "rust-embed-utils"
-version = "8.2.0"
+version = "8.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cb0a25bfbb2d4b4402179c2cf030387d9990857ce08a32592c6238db9fa8665"
+checksum = "38c74a686185620830701348de757fd36bef4aa9680fd23c49fc539ddcc1af32"
 dependencies = [
  "globset",
  "sha2",
@@ -819,11 +885,11 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.38.30"
+version = "0.38.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca"
+checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
 dependencies = [
- "bitflags 2.4.1",
+ "bitflags 2.5.0",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -851,15 +917,6 @@ dependencies = [
  "winapi-util",
 ]
 
-[[package]]
-name = "sedregex"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19411e23596093f03bbd11dc45603b6329bb4bfec77b9fd13e2b9fc9b02efe3e"
-dependencies = [
- "regex",
-]
-
 [[package]]
 name = "semver"
 version = "1.0.20"
@@ -933,13 +990,12 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.9.0"
+version = "3.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa"
+checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
 dependencies = [
  "cfg-if",
  "fastrand",
- "redox_syscall 0.4.1",
  "rustix",
  "windows-sys 0.52.0",
 ]
@@ -955,15 +1011,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "termcolor"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449"
-dependencies = [
- "winapi-util",
-]
-
 [[package]]
 name = "tester"
 version = "0.9.1"
@@ -1168,6 +1215,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "utf8parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
@@ -1182,9 +1235,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "walkdir"
-version = "2.4.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
 dependencies = [
  "same-file",
  "winapi-util",
diff --git a/lints/Cargo.toml b/lints/Cargo.toml
index 758dc2bd6d7b4..2581c1d3f0483 100644
--- a/lints/Cargo.toml
+++ b/lints/Cargo.toml
@@ -14,12 +14,12 @@ path = "ui/format_error.rs"
 # See `README.md` before bumping the version.
 # Remember to update the version in `ci/Dockerfile` as well.
 [dependencies]
-clippy_utils = { git = "https://github.com/rust-lang/rust-clippy", rev = "6fd0258e45105161b7e759a22e7350958e5cb0b1" }
-dylint_linting = "2.6.0"
-itertools = { workspace = true }
+clippy_utils = { git = "https://github.com/rust-lang/rust-clippy", rev = "9d6f41691ed9dbfaec2a2df2661c42451f2fe0d3" }
+dylint_linting = "3.1.0"
+itertools = "0.12"
 
 [dev-dependencies]
-dylint_testing = "2.6.0"
+dylint_testing = "3.1.0"
 
 # UI test dependencies
 anyhow = "1"
diff --git a/lints/rust-toolchain b/lints/rust-toolchain
index ea1f0e928e5c3..e79f69d40f86e 100644
--- a/lints/rust-toolchain
+++ b/lints/rust-toolchain
@@ -1,5 +1,5 @@
 # See `README.md` before bumping the version.
 
 [toolchain]
-channel = "nightly-2024-01-11"
+channel = "nightly-2024-03-21"
 components = ["llvm-tools-preview", "rustc-dev"]
diff --git a/lints/ui/format_error.rs b/lints/ui/format_error.rs
index 22ea8c5f88e8b..7963a96018ba7 100644
--- a/lints/ui/format_error.rs
+++ b/lints/ui/format_error.rs
@@ -80,4 +80,13 @@ fn main() {
     let _ = anyhow!("some error occurred: {}", err.as_report());
     let _ = anyhow!("{:?}", anyhow_err.as_report());
     let _ = anyhow!("some error occurred: {:?}", anyhow_err.as_report());
+
+    let box_dyn_err_1: Box<dyn Error> = Box::new(err.clone());
+    let box_dyn_err_2: Box<dyn Error + Send> = Box::new(err.clone());
+    let box_dyn_err_3: Box<dyn Error + Send + Sync> = Box::new(err.clone());
+
+    // TODO: fail to lint
+    let _ = format!("{}", box_dyn_err_1);
+    info!("{}", box_dyn_err_2);
+    let _ = box_dyn_err_3.to_string();
 }
diff --git a/proto/backup_service.proto b/proto/backup_service.proto
index 75420149042d9..24d410b38f115 100644
--- a/proto/backup_service.proto
+++ b/proto/backup_service.proto
@@ -2,6 +2,8 @@ syntax = "proto3";
 
 package backup_service;
 
+import "hummock.proto";
+
 option java_package = "com.risingwave.proto";
 option optimize_for = SPEED;
 
@@ -49,6 +51,8 @@ message MetaSnapshotMetadata {
   uint64 safe_epoch = 4;
   optional uint32 format_version = 5;
   optional string remarks = 6;
+  optional string rw_version = 7;
+  map<uint32, hummock.StateTableInfo> state_table_info = 8;
 }
 
 service BackupService {
diff --git a/proto/batch_plan.proto b/proto/batch_plan.proto
index b6c00b1a14aa5..8a196eecf5021 100644
--- a/proto/batch_plan.proto
+++ b/proto/batch_plan.proto
@@ -71,6 +71,18 @@ message FilterNode {
   expr.ExprNode search_condition = 1;
 }
 
+message LogRowSeqScanNode {
+  plan_common.StorageTableDesc table_desc = 1;
+  // This records the mandatory column_ids of the original table, excluding op
+  repeated int32 column_ids = 2;
+  // The partition to read for scan tasks.
+  //
+  // Will be filled by the scheduler.
+  common.Buffer vnode_bitmap = 3;
+  common.BatchQueryEpoch old_epoch = 4;
+  common.BatchQueryEpoch new_epoch = 5;
+}
+
 message InsertNode {
   // Id of the table to perform inserting.
   uint32 table_id = 1;
@@ -213,7 +225,7 @@ message TableFunctionNode {
 message TaskId {
   string query_id = 1;
   uint32 stage_id = 2;
-  uint32 task_id = 3;
+  uint64 task_id = 3;
 }
 
 // Every task will create N buffers (channels) for parent operators to fetch results from,
@@ -221,7 +233,7 @@ message TaskId {
 message TaskOutputId {
   TaskId task_id = 1;
   // The id of output channel to fetch from
-  uint32 output_id = 2;
+  uint64 output_id = 2;
 }
 
 message LocalExecutePlan {
@@ -258,7 +270,7 @@ message LocalLookupJoinNode {
   repeated uint32 inner_side_key = 4;
   uint32 lookup_prefix_len = 5;
   plan_common.StorageTableDesc inner_side_table_desc = 6;
-  repeated uint32 inner_side_vnode_mapping = 7;
+  repeated uint64 inner_side_vnode_mapping = 7;
   repeated int32 inner_side_column_ids = 8;
   repeated uint32 output_indices = 9;
   repeated common.WorkerNode worker_nodes = 10;
@@ -326,6 +338,7 @@ message PlanNode {
     SourceNode source = 34;
     SortOverWindowNode sort_over_window = 35;
     MaxOneRowNode max_one_row = 36;
+    LogRowSeqScanNode log_row_seq_scan = 37;
     // The following nodes are used for testing.
     bool block_executor = 100;
     bool busy_loop_executor = 101;
diff --git a/proto/catalog.proto b/proto/catalog.proto
index fd7d77116c1a3..7dfefa003217d 100644
--- a/proto/catalog.proto
+++ b/proto/catalog.proto
@@ -62,10 +62,12 @@ message StreamSourceInfo {
   SchemaRegistryNameStrategy name_strategy = 10;
   optional string key_message_name = 11;
   plan_common.ExternalTableDesc external_table = 12;
-  // **This field should now be called `is_shared`.** Not renamed for backwards compatibility.
+  // **This field should now be called `is_shared`.** Not renamed for backwards
+  // compatibility.
   //
   // Whether the stream source is a shared source (it has a streaming job).
-  // This is related with [RFC: Reusable Source Executor](https://github.com/risingwavelabs/rfcs/pull/72).
+  // This is related with [RFC: Reusable Source
+  // Executor](https://github.com/risingwavelabs/rfcs/pull/72).
   //
   // Currently, the following sources can be shared:
   //
@@ -80,6 +82,9 @@ message StreamSourceInfo {
   bool is_distributed = 15;
   // Options specified by user in the FORMAT ENCODE clause.
   map<string, string> format_encode_options = 14;
+
+  // Handle the source relies on any sceret. The key is the propertity name and the value is the secret id.
+  map<string, uint32> secret_ref = 16;
 }
 
 message Source {
@@ -134,6 +139,7 @@ message SinkFormatDesc {
   plan_common.FormatType format = 1;
   plan_common.EncodeType encode = 2;
   map<string, string> options = 3;
+  optional plan_common.EncodeType key_encode = 4;
 }
 
 // the catalog of the sink. There are two kind of schema here. The full schema is all columns
@@ -173,29 +179,31 @@ message Sink {
 
   // Whether it should use background ddl or block until backfill finishes.
   CreateType create_type = 24;
+
+  // Handle the sink relies on any sceret. The key is the propertity name and the value is the secret id.
+  map<string, uint32> secret_ref = 25;
 }
 
 message Subscription {
+  enum SubscriptionState {
+    UNSPECIFIED = 0;
+    INIT = 1;
+    CREATED = 2;
+  }
   uint32 id = 1;
   string name = 2;
   string definition = 3;
-  repeated common.ColumnOrder plan_pk = 4;
-  repeated int32 distribution_key = 5;
-  map<string, string> properties = 6;
-  repeated plan_common.ColumnCatalog column_catalogs = 7;
+  uint64 retention_seconds = 6;
   uint32 database_id = 8;
   uint32 schema_id = 9;
-  repeated uint32 dependent_relations = 10;
+  uint32 dependent_table_id = 10;
   optional uint64 initialized_at_epoch = 11;
   optional uint64 created_at_epoch = 12;
   uint32 owner = 13;
-  StreamJobStatus stream_job_status = 14;
-
   optional string initialized_at_cluster_version = 15;
   optional string created_at_cluster_version = 16;
 
-  string subscription_from_name = 17;
-  optional string subscription_internal_table_name = 18;
+  SubscriptionState subscription_state = 19;
 }
 
 message Connection {
@@ -239,7 +247,8 @@ message Index {
   optional uint64 created_at_epoch = 11;
   StreamJobStatus stream_job_status = 12;
 
-  // Use to record the prefix len of the index_item to reconstruct index columns provided by users.
+  // Use to record the prefix len of the index_item to reconstruct index columns
+  // provided by users.
   uint32 index_columns_len = 13;
   // Cluster version (tracked by git commit) when initialized/created
   optional string initialized_at_cluster_version = 14;
@@ -319,8 +328,8 @@ message Table {
   // an optional column index which is the vnode of each row computed by the
   // table's consistent hash distribution
   optional uint32 vnode_col_index = 18;
-  // An optional column index of row id. If the primary key is specified by users,
-  // this will be `None`.
+  // An optional column index of row id. If the primary key is specified by
+  // users, this will be `None`.
   optional uint32 row_id_index = 19;
   // The column indices which are stored in the state store's value with
   // row-encoding. Currently is not supported yet and expected to be
@@ -329,23 +338,26 @@ message Table {
   string definition = 21;
   // Used to control whether handling pk conflict for incoming data.
   HandleConflictBehavior handle_pk_conflict_behavior = 22;
-  // Anticipated read prefix pattern (number of fields) for the table, which can be utilized
-  // for implementing the table's bloom filter or other storage optimization techniques.
+  // Anticipated read prefix pattern (number of fields) for the table, which can
+  // be utilized for implementing the table's bloom filter or other storage
+  // optimization techniques.
   uint32 read_prefix_len_hint = 23;
   repeated int32 watermark_indices = 24;
   repeated int32 dist_key_in_pk = 25;
-  // A dml fragment id corresponds to the table, used to decide where the dml statement is executed.
+  // A dml fragment id corresponds to the table, used to decide where the dml
+  // statement is executed.
   optional uint32 dml_fragment_id = 26;
   // The range of row count of the table.
-  // This field is not always present due to backward compatibility. Use `Cardinality::unknown` in this case.
+  // This field is not always present due to backward compatibility. Use
+  // `Cardinality::unknown` in this case.
   plan_common.Cardinality cardinality = 27;
 
   optional uint64 initialized_at_epoch = 28;
   optional uint64 created_at_epoch = 29;
 
-  // This field is introduced in v1.2.0. It is used to indicate whether the table should use
-  // watermark_cache to avoid state cleaning as a performance optimization.
-  // In older versions we can just initialize without it.
+  // This field is introduced in v1.2.0. It is used to indicate whether the
+  // table should use watermark_cache to avoid state cleaning as a performance
+  // optimization. In older versions we can just initialize without it.
   bool cleaned_by_watermark = 30;
 
   // Used to filter created / creating tables in meta.
@@ -363,14 +375,18 @@ message Table {
   optional string initialized_at_cluster_version = 35;
   optional string created_at_cluster_version = 36;
 
-  // TTL of the record in the table, to ensure the consistency with other tables in the streaming plan, it only applies to append-only tables.
+  // TTL of the record in the table, to ensure the consistency with other tables
+  // in the streaming plan, it only applies to append-only tables.
   optional uint32 retention_seconds = 37;
 
-  // This field specifies the index of the column set in the "with version column" within all the columns. It is used for filtering during "on conflict" operations.
+  // This field specifies the index of the column set in the "with version
+  // column" within all the columns. It is used for filtering during "on
+  // conflict" operations.
   optional uint32 version_column_index = 38;
 
-  // Per-table catalog version, used by schema change. `None` for internal tables and tests.
-  // Not to be confused with the global catalog version for notification service.
+  // Per-table catalog version, used by schema change. `None` for internal
+  // tables and tests. Not to be confused with the global catalog version for
+  // notification service.
   TableVersion version = 100;
 }
 
@@ -415,3 +431,13 @@ message Comment {
   optional uint32 column_index = 4;
   optional string description = 5;
 }
+
+message Secret {
+  uint32 id = 1;
+  string name = 2;
+  uint32 database_id = 3;
+  // The secret here is encrypted to bytes.
+  bytes value = 4;
+  uint32 owner = 5;
+  uint32 schema_id = 6;
+}
diff --git a/proto/common.proto b/proto/common.proto
index 4f0d56b4823a9..164150379c484 100644
--- a/proto/common.proto
+++ b/proto/common.proto
@@ -92,6 +92,12 @@ message ParallelUnitMapping {
   repeated uint32 data = 2;
 }
 
+// Vnode mapping for stream fragments. Stores mapping from virtual node to (worker id, slot index).
+message WorkerSlotMapping {
+  repeated uint32 original_indices = 1;
+  repeated uint64 data = 2;
+}
+
 message BatchQueryEpoch {
   oneof epoch {
     uint64 committed = 1;
diff --git a/proto/connector_service.proto b/proto/connector_service.proto
index 4deb0d6fb6096..da2c2b88087ea 100644
--- a/proto/connector_service.proto
+++ b/proto/connector_service.proto
@@ -4,7 +4,6 @@ package connector_service;
 
 import "catalog.proto";
 import "common.proto";
-import "data.proto";
 import "plan_common.proto";
 
 option java_outer_classname = "ConnectorServiceProto";
@@ -30,34 +29,21 @@ message SinkParam {
   string sink_name = 8;
 }
 
-enum SinkPayloadFormat {
-  FORMAT_UNSPECIFIED = 0;
-  JSON = 1;
-  STREAM_CHUNK = 2;
-}
-
 message SinkWriterStreamRequest {
   message StartSink {
     SinkParam sink_param = 1;
-    SinkPayloadFormat format = 2;
+    // Deprecated: SinkPayloadFormat format = 2;
+    reserved "format";
+    reserved 2;
     TableSchema payload_schema = 3;
   }
 
   message WriteBatch {
-    message JsonPayload {
-      message RowOp {
-        data.Op op_type = 1;
-        string line = 2;
-      }
-      repeated RowOp row_ops = 1;
-    }
-
     message StreamChunkPayload {
       bytes binary_data = 1;
     }
 
     oneof payload {
-      JsonPayload json_payload = 1;
       StreamChunkPayload stream_chunk_payload = 2;
       // This is a reference pointer to a StreamChunk. The StreamChunk is owned
       // by the JniSinkWriterStreamRequest, which should handle the release of StreamChunk.
@@ -65,6 +51,10 @@ message SinkWriterStreamRequest {
       int64 stream_chunk_ref_pointer = 5;
     }
 
+    // Deprecated in oneof payload: JsonPayload json_payload = 1;
+    reserved "json_payload";
+    reserved 1;
+
     uint64 batch_id = 3;
     uint64 epoch = 4;
   }
diff --git a/proto/ddl_service.proto b/proto/ddl_service.proto
index 80c46dd676a13..46c2a5c22ff6d 100644
--- a/proto/ddl_service.proto
+++ b/proto/ddl_service.proto
@@ -100,7 +100,6 @@ message DropSinkResponse {
 
 message CreateSubscriptionRequest {
   catalog.Subscription subscription = 1;
-  stream_plan.StreamFragmentGraph fragment_graph = 2;
 }
 
 message CreateSubscriptionResponse {
@@ -358,6 +357,26 @@ message GetDdlProgressResponse {
   repeated DdlProgress ddl_progress = 1;
 }
 
+message CreateSecretRequest {
+  string name = 1;
+  bytes value = 2;
+  uint32 database_id = 3;
+  uint32 schema_id = 4;
+  uint32 owner_id = 5;
+}
+
+message CreateSecretResponse {
+  uint64 version = 1;
+}
+
+message DropSecretRequest {
+  uint32 secret_id = 1;
+}
+
+message DropSecretResponse {
+  uint64 version = 1;
+}
+
 message CreateConnectionRequest {
   message PrivateLink {
     catalog.Connection.PrivateLinkService.PrivateLinkProvider provider = 1;
@@ -428,6 +447,8 @@ service DdlService {
   rpc CreateMaterializedView(CreateMaterializedViewRequest) returns (CreateMaterializedViewResponse);
   rpc DropMaterializedView(DropMaterializedViewRequest) returns (DropMaterializedViewResponse);
   rpc CreateTable(CreateTableRequest) returns (CreateTableResponse);
+  rpc CreateSecret(CreateSecretRequest) returns (CreateSecretResponse);
+  rpc DropSecret(DropSecretRequest) returns (DropSecretResponse);
   rpc AlterName(AlterNameRequest) returns (AlterNameResponse);
   rpc AlterSource(AlterSourceRequest) returns (AlterSourceResponse);
   rpc AlterOwner(AlterOwnerRequest) returns (AlterOwnerResponse);
diff --git a/proto/expr.proto b/proto/expr.proto
index c8a9887c1663e..602c712975ecd 100644
--- a/proto/expr.proto
+++ b/proto/expr.proto
@@ -194,6 +194,8 @@ message ExprNode {
     ENCRYPT = 325;
     INET_ATON = 328;
     INET_NTOA = 329;
+    QUOTE_LITERAL = 330;
+    QUOTE_NULLABLE = 331;
 
     // Unary operators
     NEG = 401;
@@ -278,6 +280,7 @@ message ExprNode {
     JSONB_PATH_QUERY_FIRST = 623;
     JSONB_POPULATE_RECORD = 629;
     JSONB_TO_RECORD = 630;
+    JSONB_SET = 631;
 
     // Non-pure functions below (> 1000)
     // ------------------------
@@ -299,6 +302,9 @@ message ExprNode {
     PG_INDEXES_SIZE = 2404;
     PG_RELATION_SIZE = 2405;
     PG_GET_SERIAL_SEQUENCE = 2406;
+    HAS_TABLE_PRIVILEGE = 2407;
+    HAS_ANY_COLUMN_PRIVILEGE = 2408;
+    HAS_SCHEMA_PRIVILEGE = 2409;
 
     // EXTERNAL
     ICEBERG_TRANSFORM = 2201;
@@ -325,6 +331,7 @@ message TableFunction {
     GENERATE_SUBSCRIPTS = 5;
     // buf:lint:ignore ENUM_VALUE_UPPER_SNAKE_CASE
     _PG_EXPANDARRAY = 6;
+    PG_GET_KEYWORDS = 18;
     // Jsonb functions
     JSONB_ARRAY_ELEMENTS = 10;
     JSONB_ARRAY_ELEMENTS_TEXT = 11;
@@ -335,13 +342,13 @@ message TableFunction {
     JSONB_POPULATE_RECORDSET = 16;
     JSONB_TO_RECORDSET = 17;
     // User defined table function
-    UDTF = 100;
+    USER_DEFINED = 100;
   }
   Type function_type = 1;
   repeated expr.ExprNode args = 2;
   data.DataType return_type = 3;
-  // optional. only used when the type is UDTF.
-  UserDefinedTableFunction udtf = 4;
+  // optional. only used when the type is USER_DEFINED.
+  UserDefinedFunctionMetadata udf = 4;
 }
 
 // Reference to an upstream column, containing its index and data type.
@@ -423,6 +430,8 @@ message AggCall {
     LAST_VALUE = 25;
     GROUPING = 26;
     INTERNAL_LAST_SEEN_VALUE = 27;
+    // user defined aggregate function
+    USER_DEFINED = 100;
   }
   Type type = 1;
   repeated InputRef args = 2;
@@ -431,6 +440,8 @@ message AggCall {
   repeated common.ColumnOrder order_by = 5;
   ExprNode filter = 6;
   repeated Constant direct_args = 7;
+  // optional. only used when the type is USER_DEFINED.
+  UserDefinedFunctionMetadata udf = 8;
 }
 
 message WindowFrame {
@@ -441,6 +452,7 @@ message WindowFrame {
 
     TYPE_ROWS = 5;
     TYPE_RANGE = 10;
+    TYPE_SESSION = 15;
   }
   enum BoundType {
     BOUND_TYPE_UNSPECIFIED = 0;
@@ -486,6 +498,13 @@ message WindowFrame {
     BoundType type = 1;
     optional data.Datum offset = 3;
   }
+  message SessionFrameBounds {
+    data.Datum gap = 1;
+
+    data.DataType order_data_type = 10;
+    common.OrderType order_type = 15;
+    data.DataType gap_data_type = 20;
+  }
 
   Type type = 1;
 
@@ -497,6 +516,7 @@ message WindowFrame {
   oneof bounds {
     RowsFrameBounds rows = 10;
     RangeFrameBounds range = 15;
+    SessionFrameBounds session = 20;
   }
 }
 
@@ -523,7 +543,7 @@ message WindowFunction {
 }
 
 // Note: due to historic reasons, UserDefinedFunction is a oneof variant parallel to FunctionCall,
-// while UserDefinedTableFunction is embedded as a field in TableFunction.
+// while UserDefinedFunctionMetadata is embedded as a field in TableFunction and AggCall.
 
 message UserDefinedFunction {
   repeated ExprNode children = 1;
@@ -549,8 +569,8 @@ message UserDefinedFunction {
   optional string function_type = 12;
 }
 
-// Additional information for user defined table functions.
-message UserDefinedTableFunction {
+// Additional information for user defined table/aggregate functions.
+message UserDefinedFunctionMetadata {
   repeated string arg_names = 8;
   repeated data.DataType arg_types = 3;
   string language = 4;
diff --git a/proto/hummock.proto b/proto/hummock.proto
index dc376cd694b8b..149944831a4f9 100644
--- a/proto/hummock.proto
+++ b/proto/hummock.proto
@@ -69,25 +69,28 @@ message IntraLevelDelta {
 enum CompatibilityVersion {
   VERSION_UNSPECIFIED = 0;
   NO_TRIVIAL_SPLIT = 1;
+  NO_MEMBER_TABLE_IDS = 2;
 }
 
 message GroupConstruct {
   CompactionConfig group_config = 1;
   // If parent_group_id is not 0, it means parent_group_id splits into parent_group_id and this group, so this group is not empty initially.
   uint64 parent_group_id = 2;
-  repeated uint32 table_ids = 3;
+  repeated uint32 table_ids = 3 [deprecated = true];
   uint64 group_id = 4;
   uint64 new_sst_start_id = 5;
   CompatibilityVersion version = 6;
 }
 
 message GroupMetaChange {
-  repeated uint32 table_ids_add = 1;
-  repeated uint32 table_ids_remove = 2;
+  option deprecated = true;
+  repeated uint32 table_ids_add = 1 [deprecated = true];
+  repeated uint32 table_ids_remove = 2 [deprecated = true];
 }
 
 message GroupTableChange {
-  repeated uint32 table_ids = 1;
+  option deprecated = true;
+  repeated uint32 table_ids = 1 [deprecated = true];
   uint64 target_group_id = 2;
   uint64 origin_group_id = 3;
   uint64 new_sst_start_id = 4;
@@ -102,7 +105,7 @@ message GroupDelta {
     GroupConstruct group_construct = 2;
     GroupDestroy group_destroy = 3;
     GroupMetaChange group_meta_change = 4;
-    GroupTableChange group_table_change = 5;
+    GroupTableChange group_table_change = 5 [deprecated = true];
   }
 }
 
@@ -149,13 +152,25 @@ message TableChangeLog {
   repeated EpochNewChangeLog change_logs = 1;
 }
 
+message StateTableInfo {
+  uint64 committed_epoch = 1;
+  uint64 safe_epoch = 2;
+  uint64 compaction_group_id = 3;
+}
+
+message StateTableInfoDelta {
+  uint64 committed_epoch = 1;
+  uint64 safe_epoch = 2;
+  uint64 compaction_group_id = 3;
+}
+
 message HummockVersion {
   message Levels {
     repeated Level levels = 1;
     OverlappingLevel l0 = 2;
     uint64 group_id = 3;
     uint64 parent_group_id = 4;
-    repeated uint32 member_table_ids = 5;
+    repeated uint32 member_table_ids = 5 [deprecated = true];
   }
   uint64 id = 1;
   // Levels of each compaction group
@@ -166,6 +181,7 @@ message HummockVersion {
   uint64 safe_epoch = 4;
   map<uint32, TableWatermarks> table_watermarks = 5;
   map<uint32, TableChangeLog> table_change_logs = 6;
+  map<uint32, StateTableInfo> state_table_info = 7;
 }
 
 message HummockVersionDelta {
@@ -185,6 +201,13 @@ message HummockVersionDelta {
   reserved "gc_object_ids";
   map<uint32, TableWatermarks> new_table_watermarks = 8;
   repeated uint32 removed_table_ids = 9;
+  message ChangeLogDelta {
+    EpochNewChangeLog new_log = 1;
+    // only logs in epoch later than truncate_epoch will be preserved
+    uint64 truncate_epoch = 2;
+  }
+  map<uint32, ChangeLogDelta> change_log_delta = 10;
+  map<uint32, StateTableInfoDelta> state_table_info_delta = 11;
 }
 
 message HummockVersionDeltas {
@@ -322,6 +345,7 @@ message CompactTask {
     JOIN_HANDLE_FAILED = 11;
     TRACK_SST_OBJECT_ID_FAILED = 12;
     NO_AVAIL_CPU_RESOURCE_CANCELED = 13;
+    HEARTBEAT_PROGRESS_CANCELED = 14;
   }
   // SSTs to be compacted, which will be removed from LSM after compaction
   repeated InputLevel input_ssts = 1;
@@ -376,6 +400,8 @@ message CompactTask {
   map<uint32, TableWatermarks> table_watermarks = 24;
   // The table schemas that are at least as new as the one used to create `input_ssts`.
   map<uint32, TableSchema> table_schemas = 25;
+  // Max sub compaction task numbers
+  uint32 max_sub_compaction = 26;
 }
 
 message LevelHandler {
@@ -663,6 +689,11 @@ message RiseCtlListCompactionGroupResponse {
 }
 
 message RiseCtlUpdateCompactionConfigRequest {
+  message CompressionAlgorithm {
+    uint32 level = 1;
+    string compression_algorithm = 2;
+  }
+
   message MutableConfig {
     oneof mutable_config {
       uint64 max_bytes_for_level_base = 1;
@@ -680,6 +711,8 @@ message RiseCtlUpdateCompactionConfigRequest {
       uint64 level0_max_compact_file_number = 14;
       bool enable_emergency_picker = 15;
       uint32 tombstone_reclaim_ratio = 16;
+      CompressionAlgorithm compression_algorithm = 17;
+      uint32 max_l0_compact_level_count = 18;
     }
   }
   repeated uint64 compaction_group_ids = 1;
@@ -827,6 +860,7 @@ service HummockManagerService {
   rpc ListCompactTaskAssignment(ListCompactTaskAssignmentRequest) returns (ListCompactTaskAssignmentResponse);
   rpc ListCompactTaskProgress(ListCompactTaskProgressRequest) returns (ListCompactTaskProgressResponse);
   rpc CancelCompactTask(CancelCompactTaskRequest) returns (CancelCompactTaskResponse);
+  rpc ListChangeLogEpochs(ListChangeLogEpochsRequest) returns (ListChangeLogEpochsResponse);
 }
 
 message CompactionConfig {
@@ -858,6 +892,9 @@ message CompactionConfig {
   uint32 level0_overlapping_sub_level_compact_level_count = 18;
   uint32 tombstone_reclaim_ratio = 19;
   bool enable_emergency_picker = 20;
+
+  // The limitation of the level count of l0 compaction
+  uint32 max_l0_compact_level_count = 21;
 }
 
 message TableStats {
@@ -882,7 +919,17 @@ message WriteLimits {
 
 message BranchedObject {
   uint64 object_id = 1;
-  uint64 sst_id = 2;
+  repeated uint64 sst_id = 2;
   // Compaction group id the SST belongs to.
   uint64 compaction_group_id = 3;
 }
+
+message ListChangeLogEpochsRequest {
+  uint32 table_id = 1;
+  uint64 min_epoch = 2;
+  uint32 max_count = 3;
+}
+
+message ListChangeLogEpochsResponse {
+  repeated uint64 epochs = 1;
+}
diff --git a/proto/meta.proto b/proto/meta.proto
index dadc5b364c623..9ad18cb3df7d5 100644
--- a/proto/meta.proto
+++ b/proto/meta.proto
@@ -111,6 +111,16 @@ message FragmentParallelUnitMappings {
   repeated FragmentParallelUnitMapping mappings = 1;
 }
 
+/// Worker slot mapping with fragment id, used for notification.
+message FragmentWorkerSlotMapping {
+  uint32 fragment_id = 1;
+  common.WorkerSlotMapping mapping = 2;
+}
+
+message FragmentWorkerSlotMappings {
+  repeated FragmentWorkerSlotMapping mappings = 1;
+}
+
 // TODO: remove this when dashboard refactored.
 message ActorLocation {
   common.WorkerNode node = 1;
@@ -254,6 +264,7 @@ enum ThrottleTarget {
   THROTTLE_TARGET_UNSPECIFIED = 0;
   SOURCE = 1;
   MV = 2;
+  TABLE_WITH_SOURCE = 3;
 }
 
 message ApplyThrottleRequest {
@@ -377,8 +388,10 @@ message SubscribeRequest {
 message MetaSnapshot {
   message SnapshotVersion {
     uint64 catalog_version = 1;
-    uint64 parallel_unit_mapping_version = 2;
+    reserved 2;
+    reserved "parallel_unit_mapping_version";
     uint64 worker_node_version = 3;
+    uint64 streaming_worker_slot_mapping_version = 4;
   }
   repeated catalog.Database databases = 1;
   repeated catalog.Schema schemas = 2;
@@ -391,16 +404,21 @@ message MetaSnapshot {
   repeated catalog.Connection connections = 17;
   repeated catalog.Subscription subscriptions = 19;
   repeated user.UserInfo users = 8;
+  reserved 9;
+  reserved "parallel_unit_mappings";
   GetSessionParamsResponse session_params = 20;
-  // for streaming
-  repeated FragmentParallelUnitMapping parallel_unit_mappings = 9;
+  repeated catalog.Secret secrets = 23;
   repeated common.WorkerNode nodes = 10;
   hummock.HummockSnapshot hummock_snapshot = 11;
   hummock.HummockVersion hummock_version = 12;
   backup_service.MetaBackupManifestId meta_backup_manifest_id = 14;
   hummock.WriteLimits hummock_write_limits = 16;
-  // for serving
-  repeated FragmentParallelUnitMapping serving_parallel_unit_mappings = 18;
+  reserved 18;
+  reserved "serving_parallel_unit_mappings";
+
+  // for streaming
+  repeated FragmentWorkerSlotMapping streaming_worker_slot_mappings = 21;
+  repeated FragmentWorkerSlotMapping serving_worker_slot_mappings = 22;
 
   SnapshotVersion version = 13;
 }
@@ -439,8 +457,6 @@ message SubscribeResponse {
     catalog.Function function = 6;
     user.UserInfo user = 11;
     SetSessionParamRequest session_param = 26;
-    // for streaming
-    FragmentParallelUnitMapping parallel_unit_mapping = 12;
     common.WorkerNode node = 13;
     hummock.HummockSnapshot hummock_snapshot = 14;
     hummock.HummockVersionDeltas hummock_version_deltas = 15;
@@ -450,10 +466,16 @@ message SubscribeResponse {
     hummock.WriteLimits hummock_write_limits = 20;
     RelationGroup relation_group = 21;
     catalog.Connection connection = 22;
-    FragmentParallelUnitMappings serving_parallel_unit_mappings = 23;
     hummock.HummockVersionStats hummock_stats = 24;
     Recovery recovery = 25;
+    FragmentWorkerSlotMapping streaming_worker_slot_mapping = 27;
+    FragmentWorkerSlotMappings serving_worker_slot_mappings = 28;
+    catalog.Secret secret = 29;
   }
+  reserved 12;
+  reserved "parallel_unit_mapping";
+  reserved 23;
+  reserved "serving_parallel_unit_mappings";
 }
 
 service NotificationService {
@@ -628,8 +650,10 @@ service SessionParamService {
 message GetServingVnodeMappingsRequest {}
 
 message GetServingVnodeMappingsResponse {
-  repeated FragmentParallelUnitMapping mappings = 1;
+  reserved 1;
+  reserved "mappings";
   map<uint32, uint32> fragment_to_table = 2;
+  repeated FragmentWorkerSlotMapping worker_slot_mappings = 3;
 }
 
 service ServingService {
diff --git a/proto/monitor_service.proto b/proto/monitor_service.proto
index 6a531e1bbab93..03013c127fd88 100644
--- a/proto/monitor_service.proto
+++ b/proto/monitor_service.proto
@@ -12,6 +12,8 @@ message StackTraceResponse {
   map<string, string> rpc_traces = 2;
   map<string, string> compaction_task_traces = 3;
   map<uint64, string> inflight_barrier_traces = 4;
+  map<uint32, string> barrier_worker_state = 5; // key: worker id
+  map<uint32, string> jvm_stack_traces = 6; // key: worker id. Might be empty if the worker doesn't run JVM.
 }
 
 // CPU profiling
diff --git a/proto/plan_common.proto b/proto/plan_common.proto
index 79a1b1622704e..31718ed9ac5cc 100644
--- a/proto/plan_common.proto
+++ b/proto/plan_common.proto
@@ -147,6 +147,7 @@ enum EncodeType {
   ENCODE_TYPE_BYTES = 6;
   ENCODE_TYPE_TEMPLATE = 7;
   ENCODE_TYPE_NONE = 8;
+  ENCODE_TYPE_TEXT = 9;
 }
 
 enum RowFormatType {
@@ -202,6 +203,15 @@ message AdditionalColumnHeader {
   data.DataType data_type = 2;
 }
 
+// metadata column for cdc table
+message AdditionalDatabaseName {}
+
+message AdditionalSchemaName {}
+
+message AdditionalTableName {}
+
+message AdditionalCollectionName {}
+
 // this type means we read all headers as a whole
 message AdditionalColumnHeaders {}
 
@@ -214,6 +224,10 @@ message AdditionalColumn {
     AdditionalColumnHeader header_inner = 5;
     AdditionalColumnFilename filename = 6;
     AdditionalColumnHeaders headers = 7;
+    AdditionalDatabaseName database_name = 8;
+    AdditionalSchemaName schema_name = 9;
+    AdditionalTableName table_name = 10;
+    AdditionalCollectionName collection_name = 11;
   }
 }
 
diff --git a/proto/secret.proto b/proto/secret.proto
new file mode 100644
index 0000000000000..f5065009519fd
--- /dev/null
+++ b/proto/secret.proto
@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+package secret;
+
+message SecretMetaBackend {
+  bytes value = 1;
+}
+
+message SecretHashicropValutBackend {
+  string host = 1;
+  string vault_token = 2;
+}
+
+message Secret {
+  // the message is stored in meta as encrypted bytes and is interpreted as bytes by catalog
+  oneof secret_backend {
+    SecretMetaBackend meta = 1;
+    SecretHashicropValutBackend hashicorp_vault = 2;
+  }
+}
diff --git a/proto/stream_plan.proto b/proto/stream_plan.proto
index 89c0521378972..223a23813f68d 100644
--- a/proto/stream_plan.proto
+++ b/proto/stream_plan.proto
@@ -94,6 +94,16 @@ message CombinedMutation {
   repeated BarrierMutation mutations = 1;
 }
 
+message CreateSubscriptionMutation {
+  uint32 subscription_id = 1;
+  uint32 upstream_mv_table_id = 2;
+}
+
+message DropSubscriptionMutation {
+  uint32 subscription_id = 1;
+  uint32 upstream_mv_table_id = 2;
+}
+
 message BarrierMutation {
   oneof mutation {
     // Add new dispatchers to some actors, used for creating materialized views.
@@ -111,6 +121,10 @@ message BarrierMutation {
     ResumeMutation resume = 8;
     // Throttle specific source exec or chain exec.
     ThrottleMutation throttle = 10;
+    // Create subscription on mv
+    CreateSubscriptionMutation create_subscription = 11;
+    // Drop subscription on mv
+    DropSubscriptionMutation drop_subscription = 12;
     // Combined mutation.
     CombinedMutation combined = 100;
   }
@@ -264,6 +278,9 @@ message ProjectNode {
   repeated uint32 watermark_input_cols = 2;
   repeated uint32 watermark_output_cols = 3;
   repeated uint32 nondecreasing_exprs = 4;
+  // Whether there are likely no-op updates in the output chunks, so that eliminating them with
+  // `StreamChunk::eliminate_adjacent_noop_update` could be beneficial.
+  bool noop_update_hint = 5;
 }
 
 message FilterNode {
@@ -561,6 +578,16 @@ message StreamScanNode {
   catalog.Table arrangement_table = 10;
 }
 
+// Config options for CDC backfill
+message StreamCdcScanOptions {
+  // Whether skip the backfill and only consume from upstream.
+  bool disable_backfill = 1;
+
+  uint32 snapshot_barrier_interval = 2;
+
+  uint32 snapshot_batch_size = 3;
+}
+
 message StreamCdcScanNode {
   uint32 table_id = 1;
 
@@ -581,7 +608,10 @@ message StreamCdcScanNode {
   optional uint32 rate_limit = 6;
 
   // Whether skip the backfill and only consume from upstream.
+  // keep it for backward compatibility, new stream plan will use `options.disable_backfill`
   bool disable_backfill = 7;
+
+  StreamCdcScanOptions options = 8;
 }
 
 // BatchPlanNode is used for mv on mv snapshot read.
@@ -738,12 +768,6 @@ message OverWindowNode {
   OverWindowCachePolicy cache_policy = 5;
 }
 
-message SubscriptionNode {
-  catalog.Subscription subscription_catalog = 1;
-  // log store should have a table.
-  catalog.Table log_store_table = 2;
-}
-
 message StreamNode {
   oneof node_body {
     SourceNode source = 100;
@@ -787,7 +811,6 @@ message StreamNode {
     StreamFsFetchNode stream_fs_fetch = 138;
     StreamCdcScanNode stream_cdc_scan = 139;
     CdcFilterNode cdc_filter = 140;
-    SubscriptionNode subscription = 141;
     SourceBackfillNode source_backfill = 142;
   }
   // The id for the operator. This is local per mview.
@@ -884,7 +907,6 @@ enum FragmentTypeFlag {
   FRAGMENT_TYPE_FLAG_VALUES = 64;
   FRAGMENT_TYPE_FLAG_DML = 128;
   FRAGMENT_TYPE_FLAG_CDC_FILTER = 256;
-  FRAGMENT_TYPE_FLAG_SUBSCRIPTION = 512;
   FRAGMENT_TYPE_FLAG_SOURCE_SCAN = 1024;
 }
 
diff --git a/proto/stream_service.proto b/proto/stream_service.proto
index 5990fe1e2cbcf..99c3f68fa34bf 100644
--- a/proto/stream_service.proto
+++ b/proto/stream_service.proto
@@ -9,10 +9,18 @@ import "stream_plan.proto";
 option java_package = "com.risingwave.proto";
 option optimize_for = SPEED;
 
+message BuildActorInfo {
+  stream_plan.StreamActor actor = 1;
+  message SubscriptionIds {
+    repeated uint32 subscription_ids = 1;
+  }
+  map<uint32, SubscriptionIds> related_subscriptions = 2;
+}
+
 // Describe the fragments which will be running on this node
 message UpdateActorsRequest {
   string request_id = 1;
-  repeated stream_plan.StreamActor actors = 2;
+  repeated BuildActorInfo actors = 2;
 }
 
 message UpdateActorsResponse {
@@ -69,6 +77,7 @@ message BarrierCompleteResponse {
   repeated GroupedSstableInfo synced_sstables = 4;
   uint32 worker_id = 5;
   map<uint32, hummock.TableWatermarks> table_watermarks = 6;
+  repeated hummock.SstableInfo old_value_sstables = 7;
 }
 
 // Before starting streaming, the leader node broadcast the actor-host table to needed workers.
diff --git a/proto/user.proto b/proto/user.proto
index 383e78cb57b28..6218fea4fd98d 100644
--- a/proto/user.proto
+++ b/proto/user.proto
@@ -66,7 +66,7 @@ message GrantPrivilege {
 
     uint32 all_tables_schema_id = 11;
     uint32 all_sources_schema_id = 12;
-    uint32 all_dml_tables_schema_id = 13;
+    uint32 all_dml_relations_schema_id = 13;
     uint32 subscription_id = 14;
   }
   repeated ActionWithGrantOption action_with_opts = 7;
diff --git a/risedev.yml b/risedev.yml
index 721043ccf7fa2..cdc115ca30010 100644
--- a/risedev.yml
+++ b/risedev.yml
@@ -43,9 +43,6 @@ profile:
       # - use: compactor
 
       # If you want to create source from Kafka, uncomment the following lines
-      # Note that kafka depends on zookeeper, so zookeeper must be started beforehand.
-      # - use: zookeeper
-      #   persist-data: true
       # - use: kafka
       #   persist-data: true
 
@@ -108,13 +105,12 @@ profile:
       - use: minio
       - use: etcd
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
       - use: frontend
       - use: compactor
       - use: prometheus
       - use: grafana
-      - use: zookeeper
-        persist-data: true
       - use: kafka
         persist-data: true
 
@@ -124,6 +120,7 @@ profile:
       - use: etcd
       - use: meta-node
         user-managed: true
+        meta-backend: etcd
       - use: compute-node
         user-managed: true
       - use: frontend
@@ -132,8 +129,6 @@ profile:
         user-managed: true
       - use: prometheus
       - use: grafana
-      - use: zookeeper
-        persist-data: true
       - use: kafka
         persist-data: true
 
@@ -143,6 +138,7 @@ profile:
       - use: etcd
       - use: meta-node
         user-managed: true
+        meta-backend: etcd
       - use: compute-node
         user-managed: true
       - use: frontend
@@ -156,6 +152,7 @@ profile:
       - use: etcd
       - use: meta-node
         user-managed: true
+        meta-backend: etcd
       - use: compute-node
         user-managed: true
       - use: frontend
@@ -260,6 +257,7 @@ profile:
       - use: minio
       - use: etcd
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
       - use: frontend
       - use: compactor
@@ -268,8 +266,6 @@ profile:
         remote-write-region: "ap-southeast-1"
         remote-write-url: "https://aps-workspaces.ap-southeast-1.amazonaws.com/workspaces/ws-f3841dad-6a5c-420f-8f62-8f66487f512a/api/v1/remote_write"
       - use: grafana
-      - use: zookeeper
-        persist-data: true
       - use: kafka
         persist-data: true
 
@@ -295,6 +291,7 @@ profile:
         port: 5690
         dashboard-port: 5691
         exporter-port: 1250
+        meta-backend: etcd
       - use: meta-node
         port: 15690
         dashboard-port: 15691
@@ -342,6 +339,7 @@ profile:
         port: 5690
         dashboard-port: 5691
         exporter-port: 1250
+        meta-backend: etcd
       - use: meta-node
         port: 15690
         dashboard-port: 15691
@@ -362,6 +360,7 @@ profile:
         port: 5690
         dashboard-port: 5691
         exporter-port: 1250
+        meta-backend: sqlite
       - use: compactor
       - use: compute-node
       - use: frontend
@@ -375,6 +374,75 @@ profile:
         port: 5690
         dashboard-port: 5691
         exporter-port: 1250
+        meta-backend: sqlite
+      - use: compactor
+      - use: compute-node
+      - use: frontend
+
+  meta-1cn-1fe-pg-backend:
+    steps:
+      - use: minio
+      - use: postgres
+        port: 8432
+        user: postgres
+        database: metadata
+      - use: meta-node
+        port: 5690
+        dashboard-port: 5691
+        exporter-port: 1250
+        meta-backend: postgres
+      - use: compactor
+      - use: compute-node
+      - use: frontend
+
+  meta-1cn-1fe-pg-backend-with-recovery:
+    config-path: src/config/ci-recovery.toml
+    steps:
+      - use: minio
+      - use: postgres
+        port: 8432
+        user: postgres
+        database: metadata
+      - use: meta-node
+        port: 5690
+        dashboard-port: 5691
+        exporter-port: 1250
+        meta-backend: postgres
+      - use: compactor
+      - use: compute-node
+      - use: frontend
+
+  meta-1cn-1fe-mysql-backend:
+    steps:
+      - use: minio
+      - use: mysql
+        port: 4306
+        user: root
+        database: metadata
+        application: metastore
+      - use: meta-node
+        port: 5690
+        dashboard-port: 5691
+        exporter-port: 1250
+        meta-backend: mysql
+      - use: compactor
+      - use: compute-node
+      - use: frontend
+
+  meta-1cn-1fe-mysql-backend-with-recovery:
+    config-path: src/config/ci-recovery.toml
+    steps:
+      - use: minio
+      - use: mysql
+        port: 4306
+        user: root
+        database: metadata
+        application: metastore
+      - use: meta-node
+        port: 5690
+        dashboard-port: 5691
+        exporter-port: 1250
+        meta-backend: mysql
       - use: compactor
       - use: compute-node
       - use: frontend
@@ -401,16 +469,15 @@ profile:
       - use: minio
       - use: etcd
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         parallelism: 8
       - use: frontend
       - use: compactor
       - use: prometheus
       - use: grafana
-      # Do not use kafka and zookeeper here, we will spawn it separately,
+      # Do not use kafka here, we will spawn it separately,
       # so we don't have to re-generate data each time.
-      # - use: zookeeper
-      #   persist-data: true
       # RW will still be ale to talk to it.
       # - use: kafka
       #   port: 9092
@@ -561,6 +628,7 @@ profile:
       - use: etcd
         unsafe-no-fsync: true
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         enable-tiered-cache: true
       - use: frontend
@@ -573,6 +641,7 @@ profile:
       - use: etcd
         unsafe-no-fsync: true
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         port: 5687
         exporter-port: 1222
@@ -595,6 +664,7 @@ profile:
       - use: etcd
         unsafe-no-fsync: true
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         port: 5687
         exporter-port: 1222
@@ -621,6 +691,7 @@ profile:
       - use: etcd
         unsafe-no-fsync: true
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         port: 5687
         exporter-port: 1222
@@ -645,6 +716,7 @@ profile:
       - use: etcd
         unsafe-no-fsync: true
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         port: 5687
         exporter-port: 1222
@@ -669,6 +741,7 @@ profile:
       - use: etcd
         unsafe-no-fsync: true
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         port: 5687
         exporter-port: 1222
@@ -728,6 +801,7 @@ profile:
       - use: etcd
         unsafe-no-fsync: true
       - use: meta-node
+        meta-backend: etcd
       - use: opendal
         engine: fs
         bucket: "/tmp/rw_ci"
@@ -761,6 +835,7 @@ profile:
       - use: etcd
         unsafe-no-fsync: true
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         port: 5687
         exporter-port: 1222
@@ -805,13 +880,35 @@ profile:
         health-check-port: 6788
       - use: compactor
 
-  ci-pubsub-kafka:
+  ci-kafka:
     config-path: src/config/ci.toml
     steps:
       - use: minio
       - use: etcd
         unsafe-no-fsync: true
       - use: meta-node
+        meta-backend: etcd
+      - use: compute-node
+        enable-tiered-cache: true
+      - use: frontend
+      - use: compactor
+      - use: kafka
+        user-managed: true
+        address: message_queue
+        port: 29092
+      - use: schema-registry
+        user-managed: true
+        address: schemaregistry
+        port: 8082
+
+  ci-inline-source-test:
+    config-path: src/config/ci-recovery.toml
+    steps:
+      - use: minio
+      - use: etcd
+        unsafe-no-fsync: true
+      - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         enable-tiered-cache: true
       - use: frontend
@@ -822,6 +919,16 @@ profile:
         user-managed: true
         address: message_queue
         port: 29092
+      - use: schema-registry
+        user-managed: true
+        address: schemaregistry
+        port: 8082
+      - use: mysql
+        port: 3306
+        address: mysql
+        user: root
+        password: 123456
+        user-managed: true
 
   ci-redis:
     config-path: src/config/ci.toml
@@ -830,6 +937,7 @@ profile:
       - use: etcd
         unsafe-no-fsync: true
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         enable-tiered-cache: true
       - use: frontend
@@ -843,6 +951,7 @@ profile:
       - use: etcd
         unsafe-no-fsync: true
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         enable-tiered-cache: true
         total-memory-bytes: 17179869184
@@ -855,6 +964,7 @@ profile:
       - use: minio
       - use: etcd
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         enable-tiered-cache: true
       - use: frontend
@@ -867,6 +977,7 @@ profile:
       - use: etcd
         unsafe-no-fsync: true
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         port: 5687
         exporter-port: 1222
@@ -882,37 +993,56 @@ profile:
       - use: frontend
       - use: compactor
 
-  ci-1cn-1fe-kafka-with-recovery:
+  ci-1cn-1fe-user-kafka-with-recovery:
     config-path: src/config/ci-recovery.toml
     steps:
       - use: minio
       - use: etcd
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
         enable-tiered-cache: true
       - use: frontend
       - use: compactor
-      - use: zookeeper
-        persist-data: true
       - use: kafka
-        persist-data: true
+        user-managed: true
+        address: message_queue
+        port: 29092
 
-  ci-meta-backup-test:
+  ci-meta-backup-test-etcd:
     config-path: src/config/ci-meta-backup-test.toml
     steps:
       - use: etcd
       - use: minio
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
       - use: frontend
       - use: compactor
 
-  ci-meta-backup-test-restore:
+  ci-meta-backup-test-restore-etcd:
     config-path: src/config/ci-meta-backup-test.toml
     steps:
       - use: etcd
       - use: minio
 
+  ci-meta-backup-test-sql:
+    config-path: src/config/ci-meta-backup-test.toml
+    steps:
+      - use: sqlite
+      - use: minio
+      - use: meta-node
+        meta-backend: sqlite
+      - use: compute-node
+      - use: frontend
+      - use: compactor
+
+  ci-meta-backup-test-restore-sql:
+    config-path: src/config/ci-meta-backup-test.toml
+    steps:
+      - use: sqlite
+      - use: minio
+
   ci-meta-etcd-for-migration:
     config-path: src/config/ci.toml
     steps:
@@ -953,6 +1083,7 @@ profile:
       - use: minio
       - use: etcd
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
       - use: frontend
       - use: compactor
@@ -964,6 +1095,7 @@ profile:
         bucket: renjie-iceberg-bench
       - use: etcd
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
       - use: frontend
       - use: compactor
@@ -976,6 +1108,7 @@ profile:
       - use: minio
       - use: etcd
       - use: meta-node
+        meta-backend: etcd
       - use: compute-node
       - use: frontend
       - use: compactor
@@ -1136,12 +1269,21 @@ template:
     # If `user-managed` is true, this service will be started by user with the above config
     user-managed: false
 
+    # meta backend type, requires extra config for provided backend
+    meta-backend: "memory"
+
     # Etcd backend config
     provide-etcd-backend: "etcd*"
 
     # Sqlite backend config
     provide-sqlite-backend: "sqlite*"
 
+    # Postgres backend config
+    provide-postgres-backend: "postgres*"
+
+    # Mysql backend config
+    provide-mysql-backend: "mysql*"
+
     # Prometheus nodes used by dashboard service
     provide-prometheus: "prometheus*"
 
@@ -1324,7 +1466,7 @@ template:
 
     # access key, secret key and region should be set in aws config (either by env var or .aws/config)
 
-  # Apache Kafka service
+  # Apache Kafka service backed by docker.
   kafka:
     # Id to be picked-up by services
     id: kafka-${port}
@@ -1335,45 +1477,47 @@ template:
     # Listen port of Kafka
     port: 29092
 
-    # Listen address
-    listen-address: ${address}
+    # Listen port of KRaft controller
+    controller-port: 29093
+    # Listen port for other services in docker (schema-registry)
+    docker-port: 29094
 
-    # ZooKeeper used by this Kafka instance
-    provide-zookeeper: "zookeeper*"
+    # The docker image. Can be overridden to use a different version.
+    image: "confluentinc/cp-kafka:7.6.1"
 
     # If set to true, data will be persisted at data/{id}.
     persist-data: true
 
-    # Kafka broker id. If there are multiple instances of Kafka, we will need to set.
-    broker-id: 0
+    # Kafka node id. If there are multiple instances of Kafka, we will need to set.
+    node-id: 0
 
     user-managed: false
 
-  # Google pubsub emulator service
-  pubsub:
-    id: pubsub-${port}
+  schema-registry:
+    # Id to be picked-up by services
+    id: schema-registry-${port}
 
+    # Advertise address
     address: "127.0.0.1"
 
-    port: 5980
+    # Listen port of Schema Registry
+    port: 8081
 
-    persist-data: true
+    # The docker image. Can be overridden to use a different version.
+    image: "confluentinc/cp-schema-registry:7.6.1"
 
-  # Apache ZooKeeper service
-  zookeeper:
-    # Id to be picked-up by services
-    id: zookeeper-${port}
+    user-managed: false
 
-    # Advertise address of ZooKeeper
-    address: "127.0.0.1"
+    provide-kafka: "kafka*"
 
-    # Listen address
-    listen-address: ${address}
+  # Google pubsub emulator service
+  pubsub:
+    id: pubsub-${port}
 
-    # Listen port of ZooKeeper
-    port: 2181
+    address: "127.0.0.1"
+
+    port: 5980
 
-    # If set to true, data will be persisted at data/{id}.
     persist-data: true
 
   # Only supported in RiseDev compose
@@ -1406,3 +1550,71 @@ template:
 
     # address of redis
     address: "127.0.0.1"
+
+  # MySQL service backed by docker.
+  mysql:
+    # Id to be picked-up by services
+    id: mysql-${port}
+
+    # address of mysql
+    address: "127.0.0.1"
+
+    # listen port of mysql
+    port: 8306
+
+    # Note:
+    # - This will be used to initialize the MySQL instance.
+    #   * If the user is "root", the password will be used as the root password.
+    #   * Otherwise, a regular user will be created with the given password. The root password will be empty.
+    #   Note that this only applies to fresh instances, i.e., the data directory is empty.
+    # - In user-managed mode, these configs are not validated by risedev.
+    #   They are passed as-is to risedev-env default user for MySQL operations.
+    # - This is not used in RISEDEV_MYSQL_WITH_OPTIONS_COMMON.
+    user: root
+    password: ""
+    database: "risedev"
+
+    # Which application to use. Can be overridden for metastore purpose.
+    application: "connector"
+
+    # The docker image. Can be overridden to use a different version.
+    image: "mysql:8.0"
+
+    # If set to true, data will be persisted at data/{id}.
+    persist-data: true
+
+    # If `user-managed` is true, user is responsible for starting the service
+    # to serve at the above address and port in any way they see fit.
+    user-managed: false
+
+  # PostgreSQL service backed by docker.
+  postgres:
+    # Id to be picked-up by services
+    id: postgres-${port}
+
+    # address of pg
+    address: "127.0.0.1"
+
+    # listen port of pg
+    port: 8432
+
+    # Note:
+    # - This will be used to initialize the PostgreSQL instance if it's fresh.
+    # - In user-managed mode, these configs are not validated by risedev.
+    #   They are passed as-is to risedev-env default user for PostgreSQL operations.
+    user: postgres
+    password: ""
+    database: "postgres"
+
+    # Which application to use. Can be overridden for connector purpose.
+    application: "metastore"
+
+    # The docker image. Can be overridden to use a different version.
+    image: "postgres:15-alpine"
+
+    # If set to true, data will be persisted at data/{id}.
+    persist-data: true
+
+    # If `user-managed` is true, user is responsible for starting the service
+    # to serve at the above address and port in any way they see fit.
+    user-managed: false
diff --git a/scripts/install/install-risingwave.sh b/scripts/install/install-risingwave.sh
index e2095593d597e..b610ea6365cb0 100755
--- a/scripts/install/install-risingwave.sh
+++ b/scripts/install/install-risingwave.sh
@@ -11,7 +11,7 @@ META_STORE_PATH="${HOME}/.risingwave/meta_store"
 
 VERSION=$(curl -s https://api.github.com/repos/risingwavelabs/risingwave/releases/latest \
  | grep '.tag_name' \
- | sed -E -n 's/.*(v[0-9]+.[0-9]+.[0-9])\",/\1/p')
+ | sed -E -n 's/.*(v[0-9]+.[0-9]+.[0-9]+.*)\",/\1/p')
 
 BASE_URL="https://github.com/risingwavelabs/risingwave/releases/download"
 
@@ -48,11 +48,11 @@ fi
 
 ############# BREW INSTALL
 if [ "${USE_BREW}" -eq 1 ]; then
-  echo "Installing RisingWave@${VERSION} using Homebrew."
+  echo "Installing RisingWave using Homebrew."
   brew tap risingwavelabs/risingwave
   brew install risingwave
   echo
-  echo "Successfully installed RisingWave@${VERSION} using Homebrew."
+  echo "Successfully installed RisingWave using Homebrew."
   echo
   echo "Run RisingWave:"
   echo
@@ -97,7 +97,7 @@ echo "  rm -r ~/.risingwave"
 echo
 echo "To view available options, run:"
 echo
-echo "  risingwave single-node --help"
+echo "  ./risingwave single-node --help"
 echo
 # Check if $JAVA_HOME is set, if not, prompt user to install Java, and set $JAVA_HOME.
 if [ -z "${JAVA_HOME}" ]; then
diff --git a/scripts/source/README.md b/scripts/source/README.md
index 4676aa7723b37..311c058a8230b 100644
--- a/scripts/source/README.md
+++ b/scripts/source/README.md
@@ -3,3 +3,10 @@ This folder contains scripts to prepare data for testing sources.
 ## Kafka
 
 `scripts/source/test_data` contains the data. Filename's convention is `<topic_name>.<n_partitions>`.
+
+- If `<topic_name>` ends with `bin`, the whole file is a message with binary data.
+- If `<topic_name>` ends with `avro_json` or `json_schema`:
+    - The first line is the schema. Key and value are separated by `^`.
+    - The rest of the lines are messages in JSON format. Key and value are separated by `^`.
+    - Produced to Kafka with `schema_registry_producer.py` (serialized to Avro or JSON)
+- Otherwise, each line is a message, and key/value is separated by `^`.
diff --git a/scripts/source/prepare_ci_kafka.sh b/scripts/source/prepare_ci_kafka.sh
index c3d72cca3d250..7bcb7871ab1c2 100755
--- a/scripts/source/prepare_ci_kafka.sh
+++ b/scripts/source/prepare_ci_kafka.sh
@@ -5,9 +5,12 @@ set -e
 
 SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
 cd "$SCRIPT_PATH/.." || exit 1
+# cwd is /scripts
 
 echo "$SCRIPT_PATH"
 
+source ../.risingwave/config/risedev-env
+
 if [ "$1" == "compress" ]; then
   echo "Compress test_data/ into test_data.zip"
   cd ./source
@@ -37,15 +40,14 @@ for filename in $kafka_data_files; do
 
     # always ok
     echo "Drop topic $topic"
-    risedev kafka-topics --topic "$topic" --delete || true
+    risedev rpk topic delete "$topic" || true
 
     echo "Recreate topic $topic with partition $partition"
-    risedev kafka-topics --topic "$topic" --create --partitions "$partition") &
+    risedev rpk topic create "$topic" --partitions "$partition") &
 done
 wait
 
 echo "Fulfill kafka topics"
-python3 -m pip install --break-system-packages requests fastavro confluent_kafka jsonschema
 for filename in $kafka_data_files; do
     ([ -e "$filename" ]
     base=$(basename "$filename")
@@ -54,27 +56,17 @@ for filename in $kafka_data_files; do
     echo "Fulfill kafka topic $topic with data from $base"
     # binary data, one message a file, filename/topic ends with "bin"
     if [[ "$topic" = *bin ]]; then
-        kcat -P -b message_queue:29092 -t "$topic" "$filename"
+        kcat -P -b "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" -t "$topic" "$filename"
     elif [[ "$topic" = *avro_json ]]; then
-        python3 source/schema_registry_producer.py "message_queue:29092" "http://message_queue:8081" "$filename" "topic" "avro"
+        python3 source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "$filename" "topic" "avro"
     elif [[ "$topic" = *json_schema ]]; then
-        python3 source/schema_registry_producer.py "kafka:9093" "http://schemaregistry:8082" "$filename" "topic" "json"
+        python3 source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "$filename" "topic" "json"
     else
-        cat "$filename" | kcat -P -K ^  -b message_queue:29092 -t "$topic"
+        cat "$filename" | kcat -P -K ^  -b "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" -t "$topic"
     fi
     ) &
 done
 
 # test additional columns: produce messages with headers
 ADDI_COLUMN_TOPIC="kafka_additional_columns"
-for i in {0..100}; do echo "key$i:{\"a\": $i}" | kcat -P -b message_queue:29092 -t ${ADDI_COLUMN_TOPIC} -K : -H "header1=v1" -H "header2=v2"; done
-
-# write schema with name strategy
-
-## topic: upsert_avro_json-record, key subject: string, value subject: CPLM.OBJ_ATTRIBUTE_VALUE
-(python3 source/schema_registry_producer.py  "message_queue:29092" "http://message_queue:8081" source/test_data/upsert_avro_json.1 "record" "avro") &
-## topic: upsert_avro_json-topic-record,
-## key subject: upsert_avro_json-topic-record-string
-## value subject: upsert_avro_json-topic-record-CPLM.OBJ_ATTRIBUTE_VALUE
-(python3 source/schema_registry_producer.py  "message_queue:29092" "http://message_queue:8081" source/test_data/upsert_avro_json.1 "topic-record" "avro") &
-wait
+for i in {0..100}; do echo "key$i:{\"a\": $i}" | kcat -P -b "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" -t ${ADDI_COLUMN_TOPIC} -K : -H "header1=v1" -H "header2=v2"; done
diff --git a/scripts/source/prepare_ci_pubsub/Cargo.toml b/scripts/source/prepare_ci_pubsub/Cargo.toml
deleted file mode 100644
index ff3fd2b311468..0000000000000
--- a/scripts/source/prepare_ci_pubsub/Cargo.toml
+++ /dev/null
@@ -1,28 +0,0 @@
-[package]
-name = "prepare_ci_pubsub"
-version = "0.1.0"
-edition = "2021"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[package.metadata.cargo-machete]
-ignored = ["workspace-hack"]
-
-[package.metadata.cargo-udeps.ignore]
-normal = ["workspace-hack"]
-
-[dependencies]
-anyhow = "1"
-google-cloud-googleapis = { version = "0.12", features = ["pubsub"] }
-google-cloud-pubsub = "0.24"
-tokio = { version = "0.2", package = "madsim-tokio", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-    "fs",
-] }
-
-[lints]
-workspace = true
diff --git a/scripts/source/prepare_ci_pubsub/src/main.rs b/scripts/source/prepare_ci_pubsub/src/main.rs
deleted file mode 100644
index 5357e2d4b6065..0000000000000
--- a/scripts/source/prepare_ci_pubsub/src/main.rs
+++ /dev/null
@@ -1,64 +0,0 @@
-use std::fs::File;
-use std::io::prelude::*;
-use std::io::BufReader;
-
-use google_cloud_googleapis::pubsub::v1::PubsubMessage;
-use google_cloud_pubsub::client::Client;
-use google_cloud_pubsub::subscription::SubscriptionConfig;
-
-const TOPIC: &str = "test-topic";
-
-const SUBSCRIPTION_COUNT: usize = 50;
-
-#[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    std::env::set_var("PUBSUB_EMULATOR_HOST", "127.0.0.1:5980");
-
-    let client = Client::new(Default::default()).await?;
-
-    // delete and create "test-topic"
-    let topic = client.topic(TOPIC);
-    for subscription in topic.subscriptions(None).await? {
-        subscription.delete(None).await?;
-    }
-
-    let _ = topic.delete(None).await;
-    topic.create(Some(Default::default()), None).await?;
-    for i in 0..SUBSCRIPTION_COUNT {
-        let _ = client
-            .create_subscription(
-                format!("test-subscription-{}", i).as_str(),
-                TOPIC,
-                SubscriptionConfig {
-                    retain_acked_messages: true,
-                    ..Default::default()
-                },
-                None,
-            )
-            .await?;
-    }
-
-    let path = std::env::current_exe()?
-        .parent()
-        .and_then(|p| p.parent())
-        .and_then(|p| p.parent())
-        .unwrap()
-        .join("scripts/source/test_data/pubsub_1_test_topic.1");
-
-    let file = File::open(path)?;
-    let file = BufReader::new(file);
-
-    let publisher = topic.new_publisher(Default::default());
-    for line in file.lines().map_while(Result::ok) {
-        let a = publisher
-            .publish(PubsubMessage {
-                data: line.clone().into_bytes(),
-                ..Default::default()
-            })
-            .await;
-        a.get().await?;
-        println!("published {}", line);
-    }
-
-    Ok(())
-}
diff --git a/scripts/source/schema_registry_producer.py b/scripts/source/schema_registry_producer.py
index 79a3d4db1b40f..a88861b65bd26 100644
--- a/scripts/source/schema_registry_producer.py
+++ b/scripts/source/schema_registry_producer.py
@@ -39,8 +39,11 @@ def load_avro_json(encoded, schema):
 
 
 if __name__ == '__main__':
-    if len(sys.argv) < 5:
-        print("datagen.py <brokerlist> <schema-registry-url> <file> <name-strategy> <json/avro>")
+    if len(sys.argv) <= 5:
+        print(
+            "usage: schema_registry_producer.py <brokerlist> <schema-registry-url> <file> <name-strategy> <json/avro>"
+        )
+        exit(1)
     broker_list = sys.argv[1]
     schema_registry_url = sys.argv[2]
     file = sys.argv[3]
diff --git a/scripts/source/test_data/avro_c_bin.1 b/scripts/source/test_data/avro_complex_schema_bin.1
similarity index 100%
rename from scripts/source/test_data/avro_c_bin.1
rename to scripts/source/test_data/avro_complex_schema_bin.1
diff --git a/scripts/source/test_data/avro_bin.1 b/scripts/source/test_data/avro_simple_schema_bin.1
similarity index 100%
rename from scripts/source/test_data/avro_bin.1
rename to scripts/source/test_data/avro_simple_schema_bin.1
diff --git a/scripts/source/test_data/kafka_json_schema.1 b/scripts/source/test_data/kafka_json_schema.1
index e71485885a654..afa96107864cb 100644
--- a/scripts/source/test_data/kafka_json_schema.1
+++ b/scripts/source/test_data/kafka_json_schema.1
@@ -1,2 +1,2 @@
-{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"productId":{"description":"The unique identifier for a product","type":"integer"},"productName":{"description":"Name of the product","type":"string"},"price":{"description":"The price of the product","type":"number","exclusiveMinimum":0},"tags":{"description":"Tags for the product","type":"array","items":{"type":"string"},"minItems":1,"uniqueItems":true},"dimensions":{"type":"object","properties":{"length":{"type":"number"},"width":{"type":"number"},"height":{"type":"number"}},"required":["length","width","height"]}},"required":["productId","productName","price"]}
-{"productId":1,"productName":"An ice sculpture","price":12.5,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5}}
\ No newline at end of file
+{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"productId":{"description":"The unique identifier for a product","type":"integer"},"productName":{"description":"Name of the product","type":"string"},"price":{"description":"The price of the product","type":"number","exclusiveMinimum":0},"tags":{"description":"Tags for the product","type":"array","items":{"type":"string"},"minItems":1,"uniqueItems":true},"dimensions":{"type":"object","properties":{"length":{"type":"number"},"width":{"type":"number"},"height":{"type":"number"}},"required":["length","width","height"]},"map":{"type":"object","additionalProperties":{"type":"string"}},"notMap":{"type":"object","additionalProperties":{"type":"string"},"properties":{"a":{"type":"string"}}}},"required":["productId","productName","price"]}
+{"productId":1,"productName":"An ice sculpture","price":12.5,"tags":["cold","ice"],"dimensions":{"length":7,"width":12,"height":9.5},"map":{"foo":"bar"},"notMap":{"a":"b","ignored":"c"}}
\ No newline at end of file
diff --git a/scripts/source/test_data/pubsub_1_test_topic.1 b/scripts/source/test_data/pubsub_1_test_topic.1
deleted file mode 100644
index fdc299d97d826..0000000000000
--- a/scripts/source/test_data/pubsub_1_test_topic.1
+++ /dev/null
@@ -1,20 +0,0 @@
-{"v1":1,"v2":"name0"}
-{"v1":2,"v2":"name0"}
-{"v1":6,"v2":"name3"}
-{"v1":0,"v2":"name5"}
-{"v1":5,"v2":"name8"}
-{"v1":6,"v2":"name4"}
-{"v1":8,"v2":"name9"}
-{"v1":9,"v2":"name2"}
-{"v1":4,"v2":"name6"}
-{"v1":5,"v2":"name3"}
-{"v1":8,"v2":"name8"}
-{"v1":9,"v2":"name2"}
-{"v1":2,"v2":"name3"}
-{"v1":4,"v2":"name7"}
-{"v1":7,"v2":"name0"}
-{"v1":0,"v2":"name9"}
-{"v1":3,"v2":"name2"}
-{"v1":7,"v2":"name5"}
-{"v1":1,"v2":"name7"}
-{"v1":3,"v2":"name9"}
diff --git a/src/batch/Cargo.toml b/src/batch/Cargo.toml
index 019c33253466b..2ca8ed1be4e77 100644
--- a/src/batch/Cargo.toml
+++ b/src/batch/Cargo.toml
@@ -20,6 +20,7 @@ arrow-schema = { workspace = true }
 assert_matches = "1"
 async-recursion = "1"
 async-trait = "0.1"
+bytes = "1"
 either = "1"
 foyer = { workspace = true }
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
@@ -30,9 +31,11 @@ hytra = "0.1.2"
 icelake = { workspace = true }
 itertools = { workspace = true }
 memcomparable = "0.2"
+opendal = "0.45.1"
 parking_lot = { workspace = true }
 paste = "1"
 prometheus = { version = "0.13", features = ["process"] }
+prost = "0.12"
 rand = { workspace = true }
 risingwave_common = { workspace = true }
 risingwave_common_estimate_size = { workspace = true }
@@ -62,6 +65,8 @@ tokio-stream = "0.1"
 tokio-util = { workspace = true }
 tonic = { workspace = true }
 tracing = "0.1"
+twox-hash = "1"
+uuid = { version = "1", features = ["v4"] }
 
 [target.'cfg(not(madsim))'.dependencies]
 workspace-hack = { path = "../workspace-hack" }
diff --git a/src/batch/benches/hash_agg.rs b/src/batch/benches/hash_agg.rs
index a918541beea18..e91564692dc95 100644
--- a/src/batch/benches/hash_agg.rs
+++ b/src/batch/benches/hash_agg.rs
@@ -13,6 +13,8 @@
 // limitations under the License.
 pub mod utils;
 
+use std::sync::Arc;
+
 use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion};
 use itertools::Itertools;
 use risingwave_batch::executor::aggregation::build as build_agg;
@@ -49,6 +51,7 @@ fn create_agg_call(
         order_by: vec![],
         filter: None,
         direct_args: vec![],
+        udf: None,
     }
 }
 
@@ -95,7 +98,7 @@ fn create_hash_agg_executor(
     let schema = Schema { fields };
 
     Box::new(HashAggExecutor::<hash::Key64>::new(
-        agg_init_states,
+        Arc::new(agg_init_states),
         group_key_columns,
         group_key_types,
         schema,
@@ -103,6 +106,7 @@ fn create_hash_agg_executor(
         "HashAggExecutor".to_string(),
         CHUNK_SIZE,
         MemoryContext::none(),
+        false,
         ShutdownToken::empty(),
     ))
 }
diff --git a/src/batch/benches/hash_join.rs b/src/batch/benches/hash_join.rs
index 723080638e670..03e252d94f557 100644
--- a/src/batch/benches/hash_join.rs
+++ b/src/batch/benches/hash_join.rs
@@ -61,7 +61,7 @@ fn create_hash_join_executor(
         _ => vec![0, 1],
     };
 
-    let cond = with_cond.then(|| build_from_pretty("(greater_than:int8 $0:int8 100:int8)"));
+    let cond = with_cond.then(|| build_from_pretty("(greater_than:int8 $0:int8 100:int8)").into());
 
     Box::new(HashJoinExecutor::<hash::Key64>::new(
         join_type,
@@ -74,6 +74,7 @@ fn create_hash_join_executor(
         cond,
         "HashJoinExecutor".into(),
         CHUNK_SIZE,
+        false,
         ShutdownToken::empty(),
         MemoryContext::none(),
     ))
diff --git a/src/batch/src/error.rs b/src/batch/src/error.rs
index 5751ab86ea9aa..27f355aed48b3 100644
--- a/src/batch/src/error.rs
+++ b/src/batch/src/error.rs
@@ -136,6 +136,16 @@ pub enum BatchError {
 
     #[error("Streaming vnode mapping not found for fragment {0}")]
     StreamingVnodeMappingNotFound(FragmentId),
+
+    #[error("Not enough memory to run this query, batch memory limit is {0} bytes")]
+    OutOfMemory(u64),
+
+    #[error("Failed to spill out to disk")]
+    Spill(
+        #[from]
+        #[backtrace]
+        opendal::Error,
+    ),
 }
 
 // Serialize/deserialize error.
diff --git a/src/batch/src/executor/aggregation/distinct.rs b/src/batch/src/executor/aggregation/distinct.rs
index 6792ff7a80f69..728ea2631b498 100644
--- a/src/batch/src/executor/aggregation/distinct.rs
+++ b/src/batch/src/executor/aggregation/distinct.rs
@@ -62,12 +62,12 @@ impl AggregateFunction for Distinct {
         self.inner.return_type()
     }
 
-    fn create_state(&self) -> AggregateState {
-        AggregateState::Any(Box::new(State {
-            inner: self.inner.create_state(),
+    fn create_state(&self) -> Result<AggregateState> {
+        Ok(AggregateState::Any(Box::new(State {
+            inner: self.inner.create_state()?,
             exists: HashSet::new(),
             exists_estimated_heap_size: 0,
-        }))
+        })))
     }
 
     async fn update(&self, state: &mut AggregateState, input: &StreamChunk) -> Result<()> {
@@ -203,7 +203,7 @@ mod tests {
 
     fn test_agg(pretty: &str, input: StreamChunk, expected: Datum) {
         let agg = build(&AggCall::from_pretty(pretty)).unwrap();
-        let mut state = agg.create_state();
+        let mut state = agg.create_state().unwrap();
         agg.update(&mut state, &input)
             .now_or_never()
             .unwrap()
diff --git a/src/batch/src/executor/aggregation/filter.rs b/src/batch/src/executor/aggregation/filter.rs
index d7b5efc592a8f..2fbc50372a0e1 100644
--- a/src/batch/src/executor/aggregation/filter.rs
+++ b/src/batch/src/executor/aggregation/filter.rs
@@ -41,7 +41,7 @@ impl AggregateFunction for Filter {
         self.inner.return_type()
     }
 
-    fn create_state(&self) -> AggregateState {
+    fn create_state(&self) -> Result<AggregateState> {
         self.inner.create_state()
     }
 
@@ -86,7 +86,7 @@ mod tests {
             condition.into(),
             build_append_only(&AggCall::from_pretty("(count:int8 $0:int8)")).unwrap(),
         );
-        let mut state = agg.create_state();
+        let mut state = agg.create_state()?;
 
         let chunk = StreamChunk::from_pretty(
             " I
@@ -115,7 +115,7 @@ mod tests {
             expr.into(),
             build_append_only(&AggCall::from_pretty("(count:int8 $0:int8)")).unwrap(),
         );
-        let mut state = agg.create_state();
+        let mut state = agg.create_state()?;
 
         let chunk = StreamChunk::from_pretty(
             " I
@@ -147,7 +147,7 @@ mod tests {
             expr.into(),
             build_append_only(&AggCall::from_pretty("(count:int8 $0:int8)")).unwrap(),
         );
-        let mut state = agg.create_state();
+        let mut state = agg.create_state()?;
 
         let chunk = StreamChunk::from_pretty(
             " I
diff --git a/src/batch/src/executor/aggregation/orderby.rs b/src/batch/src/executor/aggregation/orderby.rs
index e9cb90eb7fd79..7f564100b8c99 100644
--- a/src/batch/src/executor/aggregation/orderby.rs
+++ b/src/batch/src/executor/aggregation/orderby.rs
@@ -93,11 +93,11 @@ impl AggregateFunction for ProjectionOrderBy {
         self.inner.return_type()
     }
 
-    fn create_state(&self) -> AggregateState {
-        AggregateState::Any(Box::new(State {
+    fn create_state(&self) -> Result<AggregateState> {
+        Ok(AggregateState::Any(Box::new(State {
             unordered_values: vec![],
             unordered_values_estimated_heap_size: 0,
-        }))
+        })))
     }
 
     async fn update(&self, state: &mut AggregateState, input: &StreamChunk) -> Result<()> {
@@ -127,7 +127,7 @@ impl AggregateFunction for ProjectionOrderBy {
 
     async fn get_result(&self, state: &AggregateState) -> Result<Datum> {
         let state = state.downcast_ref::<State>();
-        let mut inner_state = self.inner.create_state();
+        let mut inner_state = self.inner.create_state()?;
         // sort
         let mut rows = state.unordered_values.clone();
         rows.sort_unstable_by(|(key_a, _), (key_b, _)| key_a.cmp(key_b));
@@ -168,7 +168,7 @@ mod tests {
             "(array_agg:int4[] $0:int4 orderby $1:asc $0:desc)",
         ))
         .unwrap();
-        let mut state = agg.create_state();
+        let mut state = agg.create_state().unwrap();
         agg.update(&mut state, &chunk).await.unwrap();
         assert_eq!(
             agg.get_result(&state).await.unwrap(),
@@ -189,7 +189,7 @@ mod tests {
             "(string_agg:varchar $0:varchar $1:varchar orderby $2:asc $3:desc $0:desc)",
         ))
         .unwrap();
-        let mut state = agg.create_state();
+        let mut state = agg.create_state().unwrap();
         agg.update(&mut state, &chunk).await.unwrap();
         assert_eq!(
             agg.get_result(&state).await.unwrap(),
diff --git a/src/batch/src/executor/aggregation/projection.rs b/src/batch/src/executor/aggregation/projection.rs
index af2b49faf9cdb..02235163bc6b7 100644
--- a/src/batch/src/executor/aggregation/projection.rs
+++ b/src/batch/src/executor/aggregation/projection.rs
@@ -36,7 +36,7 @@ impl AggregateFunction for Projection {
         self.inner.return_type()
     }
 
-    fn create_state(&self) -> AggregateState {
+    fn create_state(&self) -> Result<AggregateState> {
         self.inner.create_state()
     }
 
diff --git a/src/batch/src/executor/delete.rs b/src/batch/src/executor/delete.rs
index 00b7ffb1ca519..f85fad254a750 100644
--- a/src/batch/src/executor/delete.rs
+++ b/src/batch/src/executor/delete.rs
@@ -39,6 +39,7 @@ pub struct DeleteExecutor {
     table_version_id: TableVersionId,
     dml_manager: DmlManagerRef,
     child: BoxedExecutor,
+    #[expect(dead_code)]
     chunk_size: usize,
     schema: Schema,
     identity: String,
@@ -195,8 +196,6 @@ mod tests {
     use std::sync::Arc;
 
     use futures::StreamExt;
-    use itertools::Itertools;
-    use risingwave_common::array::Array;
     use risingwave_common::catalog::{
         schema_test_utils, ColumnDesc, ColumnId, INITIAL_TABLE_VERSION_ID,
     };
diff --git a/src/batch/src/executor/generic_exchange.rs b/src/batch/src/executor/generic_exchange.rs
index 24ff47958dd3f..9a13fd198f9aa 100644
--- a/src/batch/src/executor/generic_exchange.rs
+++ b/src/batch/src/executor/generic_exchange.rs
@@ -45,6 +45,7 @@ pub struct GenericExchangeExecutor<CS, C> {
     context: C,
 
     schema: Schema,
+    #[expect(dead_code)]
     task_id: TaskId,
     identity: String,
 
@@ -277,9 +278,8 @@ impl<CS: 'static + Send + CreateSource, C: BatchTaskContext> GenericExchangeExec
 #[cfg(test)]
 mod tests {
 
-    use futures::StreamExt;
     use rand::Rng;
-    use risingwave_common::array::{Array, DataChunk, I32Array};
+    use risingwave_common::array::{Array, I32Array};
     use risingwave_common::types::DataType;
 
     use super::*;
diff --git a/src/batch/src/executor/group_top_n.rs b/src/batch/src/executor/group_top_n.rs
index 2b33ac6f28319..b1f4791358131 100644
--- a/src/batch/src/executor/group_top_n.rs
+++ b/src/batch/src/executor/group_top_n.rs
@@ -15,7 +15,6 @@
 use std::marker::PhantomData;
 use std::mem::swap;
 use std::sync::Arc;
-use std::vec::Vec;
 
 use futures_async_stream::try_stream;
 use hashbrown::HashMap;
@@ -240,11 +239,9 @@ impl<K: HashKey> GroupTopNExecutor<K> {
 #[cfg(test)]
 mod tests {
     use futures::stream::StreamExt;
-    use risingwave_common::array::DataChunk;
-    use risingwave_common::catalog::{Field, Schema};
+    use risingwave_common::catalog::Field;
     use risingwave_common::metrics::LabelGuardedIntGauge;
     use risingwave_common::test_prelude::DataChunkTestExt;
-    use risingwave_common::types::DataType;
     use risingwave_common::util::sort_util::OrderType;
 
     use super::*;
@@ -254,7 +251,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_group_top_n_executor() {
-        let parent_mem = MemoryContext::root(LabelGuardedIntGauge::<4>::test_int_gauge());
+        let parent_mem = MemoryContext::root(LabelGuardedIntGauge::<4>::test_int_gauge(), u64::MAX);
         {
             let schema = Schema {
                 fields: vec![
diff --git a/src/batch/src/executor/hash_agg.rs b/src/batch/src/executor/hash_agg.rs
index 402d96934a1b1..ce54ea215a430 100644
--- a/src/batch/src/executor/hash_agg.rs
+++ b/src/batch/src/executor/hash_agg.rs
@@ -12,25 +12,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::hash::BuildHasher;
 use std::marker::PhantomData;
+use std::sync::Arc;
 
+use bytes::Bytes;
 use futures_async_stream::try_stream;
+use hashbrown::hash_map::Entry;
 use itertools::Itertools;
+use prost::Message;
 use risingwave_common::array::{DataChunk, StreamChunk};
+use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::{Field, Schema};
 use risingwave_common::hash::{HashKey, HashKeyDispatcher, PrecomputedBuildHasher};
 use risingwave_common::memory::MemoryContext;
-use risingwave_common::types::DataType;
+use risingwave_common::row::{OwnedRow, Row, RowExt};
+use risingwave_common::types::{DataType, ToOwnedDatum};
+use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common_estimate_size::EstimateSize;
 use risingwave_expr::aggregate::{AggCall, AggregateState, BoxedAggregateFunction};
 use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::HashAggNode;
+use risingwave_pb::data::DataChunk as PbDataChunk;
 
 use crate::error::{BatchError, Result};
 use crate::executor::aggregation::build as build_agg;
 use crate::executor::{
     BoxedDataChunkStream, BoxedExecutor, BoxedExecutorBuilder, Executor, ExecutorBuilder,
+    WrapStreamExecutor,
+};
+use crate::spill::spill_op::{
+    SpillBuildHasher, SpillOp, DEFAULT_SPILL_PARTITION_NUM, SPILL_AT_LEAST_MEMORY,
 };
 use crate::task::{BatchTaskContext, ShutdownToken, TaskId};
 
@@ -42,7 +55,7 @@ impl HashKeyDispatcher for HashAggExecutorBuilder {
 
     fn dispatch_impl<K: HashKey>(self) -> Self::Output {
         Box::new(HashAggExecutor::<K>::new(
-            self.aggs,
+            Arc::new(self.aggs),
             self.group_key_columns,
             self.group_key_types,
             self.schema,
@@ -50,6 +63,7 @@ impl HashKeyDispatcher for HashAggExecutorBuilder {
             self.identity,
             self.chunk_size,
             self.mem_context,
+            self.enable_spill,
             self.shutdown_rx,
         ))
     }
@@ -65,10 +79,12 @@ pub struct HashAggExecutorBuilder {
     group_key_types: Vec<DataType>,
     child: BoxedExecutor,
     schema: Schema,
+    #[expect(dead_code)]
     task_id: TaskId,
     identity: String,
     chunk_size: usize,
     mem_context: MemoryContext,
+    enable_spill: bool,
     shutdown_rx: ShutdownToken,
 }
 
@@ -80,6 +96,7 @@ impl HashAggExecutorBuilder {
         identity: String,
         chunk_size: usize,
         mem_context: MemoryContext,
+        enable_spill: bool,
         shutdown_rx: ShutdownToken,
     ) -> Result<BoxedExecutor> {
         let aggs: Vec<_> = hash_agg_node
@@ -118,6 +135,7 @@ impl HashAggExecutorBuilder {
             identity,
             chunk_size,
             mem_context,
+            enable_spill,
             shutdown_rx,
         };
 
@@ -147,6 +165,7 @@ impl BoxedExecutorBuilder for HashAggExecutorBuilder {
             identity.clone(),
             source.context.get_config().developer.chunk_size,
             source.context.create_executor_mem_context(identity),
+            source.context.get_config().enable_spill,
             source.shutdown_rx.clone(),
         )
     }
@@ -155,7 +174,7 @@ impl BoxedExecutorBuilder for HashAggExecutorBuilder {
 /// `HashAggExecutor` implements the hash aggregate algorithm.
 pub struct HashAggExecutor<K> {
     /// Aggregate functions.
-    aggs: Vec<BoxedAggregateFunction>,
+    aggs: Arc<Vec<BoxedAggregateFunction>>,
     /// Column indexes that specify a group
     group_key_columns: Vec<usize>,
     /// Data types of group key columns
@@ -163,23 +182,60 @@ pub struct HashAggExecutor<K> {
     /// Output schema
     schema: Schema,
     child: BoxedExecutor,
+    /// Used to initialize the state of the aggregation from the spilled files.
+    init_agg_state_executor: Option<BoxedExecutor>,
     identity: String,
     chunk_size: usize,
     mem_context: MemoryContext,
+    enable_spill: bool,
+    /// The upper bound of memory usage for this executor.
+    memory_upper_bound: Option<u64>,
     shutdown_rx: ShutdownToken,
     _phantom: PhantomData<K>,
 }
 
 impl<K> HashAggExecutor<K> {
     pub fn new(
-        aggs: Vec<BoxedAggregateFunction>,
+        aggs: Arc<Vec<BoxedAggregateFunction>>,
+        group_key_columns: Vec<usize>,
+        group_key_types: Vec<DataType>,
+        schema: Schema,
+        child: BoxedExecutor,
+        identity: String,
+        chunk_size: usize,
+        mem_context: MemoryContext,
+        enable_spill: bool,
+        shutdown_rx: ShutdownToken,
+    ) -> Self {
+        Self::new_inner(
+            aggs,
+            group_key_columns,
+            group_key_types,
+            schema,
+            child,
+            None,
+            identity,
+            chunk_size,
+            mem_context,
+            enable_spill,
+            None,
+            shutdown_rx,
+        )
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn new_inner(
+        aggs: Arc<Vec<BoxedAggregateFunction>>,
         group_key_columns: Vec<usize>,
         group_key_types: Vec<DataType>,
         schema: Schema,
         child: BoxedExecutor,
+        init_agg_state_executor: Option<BoxedExecutor>,
         identity: String,
         chunk_size: usize,
         mem_context: MemoryContext,
+        enable_spill: bool,
+        memory_upper_bound: Option<u64>,
         shutdown_rx: ShutdownToken,
     ) -> Self {
         HashAggExecutor {
@@ -188,9 +244,12 @@ impl<K> HashAggExecutor<K> {
             group_key_types,
             schema,
             child,
+            init_agg_state_executor,
             identity,
             chunk_size,
             mem_context,
+            enable_spill,
+            memory_upper_bound,
             shutdown_rx,
             _phantom: PhantomData,
         }
@@ -211,18 +270,243 @@ impl<K: HashKey + Send + Sync> Executor for HashAggExecutor<K> {
     }
 }
 
+/// `AggSpillManager` is used to manage how to write spill data file and read them back.
+/// The spill data first need to be partitioned. Each partition contains 2 files: `agg_state_file` and `input_chunks_file`.
+/// The spill file consume a data chunk and serialize the chunk into a protobuf bytes.
+/// Finally, spill file content will look like the below.
+/// The file write pattern is append-only and the read pattern is sequential scan.
+/// This can maximize the disk IO performance.
+///
+/// ```text
+/// [proto_len]
+/// [proto_bytes]
+/// ...
+/// [proto_len]
+/// [proto_bytes]
+/// ```
+pub struct AggSpillManager {
+    op: SpillOp,
+    partition_num: usize,
+    agg_state_writers: Vec<opendal::Writer>,
+    agg_state_chunk_builder: Vec<DataChunkBuilder>,
+    input_writers: Vec<opendal::Writer>,
+    input_chunk_builders: Vec<DataChunkBuilder>,
+    spill_build_hasher: SpillBuildHasher,
+    group_key_types: Vec<DataType>,
+    child_data_types: Vec<DataType>,
+    agg_data_types: Vec<DataType>,
+    spill_chunk_size: usize,
+}
+
+impl AggSpillManager {
+    fn new(
+        agg_identity: &String,
+        partition_num: usize,
+        group_key_types: Vec<DataType>,
+        agg_data_types: Vec<DataType>,
+        child_data_types: Vec<DataType>,
+        spill_chunk_size: usize,
+    ) -> Result<Self> {
+        let suffix_uuid = uuid::Uuid::new_v4();
+        let dir = format!("/{}-{}/", agg_identity, suffix_uuid);
+        let op = SpillOp::create(dir)?;
+        let agg_state_writers = Vec::with_capacity(partition_num);
+        let agg_state_chunk_builder = Vec::with_capacity(partition_num);
+        let input_writers = Vec::with_capacity(partition_num);
+        let input_chunk_builders = Vec::with_capacity(partition_num);
+        // Use uuid to generate an unique hasher so that when recursive spilling happens they would use a different hasher to avoid data skew.
+        let spill_build_hasher = SpillBuildHasher(suffix_uuid.as_u64_pair().1);
+        Ok(Self {
+            op,
+            partition_num,
+            agg_state_writers,
+            agg_state_chunk_builder,
+            input_writers,
+            input_chunk_builders,
+            spill_build_hasher,
+            group_key_types,
+            child_data_types,
+            agg_data_types,
+            spill_chunk_size,
+        })
+    }
+
+    async fn init_writers(&mut self) -> Result<()> {
+        for i in 0..self.partition_num {
+            let agg_state_partition_file_name = format!("agg-state-p{}", i);
+            let w = self.op.writer_with(&agg_state_partition_file_name).await?;
+            self.agg_state_writers.push(w);
+
+            let partition_file_name = format!("input-chunks-p{}", i);
+            let w = self.op.writer_with(&partition_file_name).await?;
+            self.input_writers.push(w);
+            self.input_chunk_builders.push(DataChunkBuilder::new(
+                self.child_data_types.clone(),
+                self.spill_chunk_size,
+            ));
+            self.agg_state_chunk_builder.push(DataChunkBuilder::new(
+                self.group_key_types
+                    .iter()
+                    .cloned()
+                    .chain(self.agg_data_types.iter().cloned())
+                    .collect(),
+                self.spill_chunk_size,
+            ));
+        }
+        Ok(())
+    }
+
+    async fn write_agg_state_row(&mut self, row: impl Row, hash_code: u64) -> Result<()> {
+        let partition = hash_code as usize % self.partition_num;
+        if let Some(output_chunk) = self.agg_state_chunk_builder[partition].append_one_row(row) {
+            let chunk_pb: PbDataChunk = output_chunk.to_protobuf();
+            let buf = Message::encode_to_vec(&chunk_pb);
+            let len_bytes = Bytes::copy_from_slice(&(buf.len() as u32).to_le_bytes());
+            self.agg_state_writers[partition].write(len_bytes).await?;
+            self.agg_state_writers[partition].write(buf).await?;
+        }
+        Ok(())
+    }
+
+    async fn write_input_chunk(&mut self, chunk: DataChunk, hash_codes: Vec<u64>) -> Result<()> {
+        let (columns, vis) = chunk.into_parts_v2();
+        for partition in 0..self.partition_num {
+            let new_vis = vis.clone()
+                & Bitmap::from_iter(
+                    hash_codes
+                        .iter()
+                        .map(|hash_code| (*hash_code as usize % self.partition_num) == partition),
+                );
+            let new_chunk = DataChunk::from_parts(columns.clone(), new_vis);
+            for output_chunk in self.input_chunk_builders[partition].append_chunk(new_chunk) {
+                let chunk_pb: PbDataChunk = output_chunk.to_protobuf();
+                let buf = Message::encode_to_vec(&chunk_pb);
+                let len_bytes = Bytes::copy_from_slice(&(buf.len() as u32).to_le_bytes());
+                self.input_writers[partition].write(len_bytes).await?;
+                self.input_writers[partition].write(buf).await?;
+            }
+        }
+        Ok(())
+    }
+
+    async fn close_writers(&mut self) -> Result<()> {
+        for partition in 0..self.partition_num {
+            if let Some(output_chunk) = self.agg_state_chunk_builder[partition].consume_all() {
+                let chunk_pb: PbDataChunk = output_chunk.to_protobuf();
+                let buf = Message::encode_to_vec(&chunk_pb);
+                let len_bytes = Bytes::copy_from_slice(&(buf.len() as u32).to_le_bytes());
+                self.agg_state_writers[partition].write(len_bytes).await?;
+                self.agg_state_writers[partition].write(buf).await?;
+            }
+
+            if let Some(output_chunk) = self.input_chunk_builders[partition].consume_all() {
+                let chunk_pb: PbDataChunk = output_chunk.to_protobuf();
+                let buf = Message::encode_to_vec(&chunk_pb);
+                let len_bytes = Bytes::copy_from_slice(&(buf.len() as u32).to_le_bytes());
+                self.input_writers[partition].write(len_bytes).await?;
+                self.input_writers[partition].write(buf).await?;
+            }
+        }
+
+        for mut w in self.agg_state_writers.drain(..) {
+            w.close().await?;
+        }
+        for mut w in self.input_writers.drain(..) {
+            w.close().await?;
+        }
+        Ok(())
+    }
+
+    async fn read_agg_state_partition(&mut self, partition: usize) -> Result<BoxedDataChunkStream> {
+        let agg_state_partition_file_name = format!("agg-state-p{}", partition);
+        let r = self.op.reader_with(&agg_state_partition_file_name).await?;
+        Ok(SpillOp::read_stream(r))
+    }
+
+    async fn read_input_partition(&mut self, partition: usize) -> Result<BoxedDataChunkStream> {
+        let input_partition_file_name = format!("input-chunks-p{}", partition);
+        let r = self.op.reader_with(&input_partition_file_name).await?;
+        Ok(SpillOp::read_stream(r))
+    }
+
+    async fn estimate_partition_size(&self, partition: usize) -> Result<u64> {
+        let agg_state_partition_file_name = format!("agg-state-p{}", partition);
+        let agg_state_size = self
+            .op
+            .stat(&agg_state_partition_file_name)
+            .await?
+            .content_length();
+        let input_partition_file_name = format!("input-chunks-p{}", partition);
+        let input_size = self
+            .op
+            .stat(&input_partition_file_name)
+            .await?
+            .content_length();
+        Ok(agg_state_size + input_size)
+    }
+
+    async fn clear_partition(&mut self, partition: usize) -> Result<()> {
+        let agg_state_partition_file_name = format!("agg-state-p{}", partition);
+        self.op.delete(&agg_state_partition_file_name).await?;
+        let input_partition_file_name = format!("input-chunks-p{}", partition);
+        self.op.delete(&input_partition_file_name).await?;
+        Ok(())
+    }
+}
+
 impl<K: HashKey + Send + Sync> HashAggExecutor<K> {
     #[try_stream(boxed, ok = DataChunk, error = BatchError)]
     async fn do_execute(self: Box<Self>) {
+        let child_schema = self.child.schema().clone();
+        let mut need_to_spill = false;
+        // If the memory upper bound is less than 1MB, we don't need to check memory usage.
+        let check_memory = match self.memory_upper_bound {
+            Some(upper_bound) => upper_bound > SPILL_AT_LEAST_MEMORY,
+            None => true,
+        };
+
         // hash map for each agg groups
         let mut groups = AggHashMap::<K, _>::with_hasher_in(
             PrecomputedBuildHasher,
             self.mem_context.global_allocator(),
         );
 
+        if let Some(init_agg_state_executor) = self.init_agg_state_executor {
+            // `init_agg_state_executor` exists which means this is a sub `HashAggExecutor` used to consume spilling data.
+            // The spilled agg states by its parent executor need to be recovered first.
+            let mut init_agg_state_stream = init_agg_state_executor.execute();
+            #[for_await]
+            for chunk in &mut init_agg_state_stream {
+                let chunk = chunk?;
+                let group_key_indices = (0..self.group_key_columns.len()).collect_vec();
+                let keys = K::build_many(&group_key_indices, &chunk);
+                let mut memory_usage_diff = 0;
+                for (row_id, key) in keys.into_iter().enumerate() {
+                    let mut agg_states = vec![];
+                    for i in 0..self.aggs.len() {
+                        let agg = &self.aggs[i];
+                        let datum = chunk
+                            .row_at(row_id)
+                            .0
+                            .datum_at(self.group_key_columns.len() + i)
+                            .to_owned_datum();
+                        let agg_state = agg.decode_state(datum)?;
+                        memory_usage_diff += agg_state.estimated_size() as i64;
+                        agg_states.push(agg_state);
+                    }
+                    groups.try_insert(key, agg_states).unwrap();
+                }
+
+                if !self.mem_context.add(memory_usage_diff) && check_memory {
+                    warn!("not enough memory to load one partition agg state after spill which is not a normal case, so keep going");
+                }
+            }
+        }
+
+        let mut input_stream = self.child.execute();
         // consume all chunks to compute the agg result
         #[for_await]
-        for chunk in self.child.execute() {
+        for chunk in &mut input_stream {
             let chunk = StreamChunk::from(chunk?);
             let keys = K::build_many(self.group_key_columns.as_slice(), &chunk);
             let mut memory_usage_diff = 0;
@@ -235,10 +519,18 @@ impl<K: HashKey + Send + Sync> HashAggExecutor<K> {
                     continue;
                 }
                 let mut new_group = false;
-                let states = groups.entry(key).or_insert_with(|| {
-                    new_group = true;
-                    self.aggs.iter().map(|agg| agg.create_state()).collect()
-                });
+                let states = match groups.entry(key) {
+                    Entry::Occupied(entry) => entry.into_mut(),
+                    Entry::Vacant(entry) => {
+                        new_group = true;
+                        let states = self
+                            .aggs
+                            .iter()
+                            .map(|agg| agg.create_state())
+                            .try_collect()?;
+                        entry.insert(states)
+                    }
+                };
 
                 // TODO: currently not a vectorized implementation
                 for (agg, state) in self.aggs.iter().zip_eq_fast(states) {
@@ -250,52 +542,161 @@ impl<K: HashKey + Send + Sync> HashAggExecutor<K> {
                 }
             }
             // update memory usage
-            self.mem_context.add(memory_usage_diff);
+            if !self.mem_context.add(memory_usage_diff) && check_memory {
+                if self.enable_spill {
+                    need_to_spill = true;
+                    break;
+                } else {
+                    Err(BatchError::OutOfMemory(self.mem_context.mem_limit()))?;
+                }
+            }
         }
 
-        // Don't use `into_iter` here, it may cause memory leak.
-        let mut result = groups.iter_mut();
-        let cardinality = self.chunk_size;
-        loop {
-            let mut group_builders: Vec<_> = self
-                .group_key_types
-                .iter()
-                .map(|datatype| datatype.create_array_builder(cardinality))
-                .collect();
-
-            let mut agg_builders: Vec<_> = self
-                .aggs
-                .iter()
-                .map(|agg| agg.return_type().create_array_builder(cardinality))
-                .collect();
-
-            let mut has_next = false;
-            let mut array_len = 0;
-            for (key, states) in result.by_ref().take(cardinality) {
-                self.shutdown_rx.check()?;
-                has_next = true;
-                array_len += 1;
-                key.deserialize_to_builders(&mut group_builders[..], &self.group_key_types)?;
-                for ((agg, state), builder) in (self.aggs.iter())
-                    .zip_eq_fast(states)
-                    .zip_eq_fast(&mut agg_builders)
-                {
-                    let result = agg.get_result(state).await?;
-                    builder.append(result);
+        if need_to_spill {
+            // A spilling version of aggregation based on the RFC: Spill Hash Aggregation https://github.com/risingwavelabs/rfcs/pull/89
+            // When HashAggExecutor told memory is insufficient, AggSpillManager will start to partition the hash table and spill to disk.
+            // After spilling the hash table, AggSpillManager will consume all chunks from the input executor,
+            // partition and spill to disk with the same hash function as the hash table spilling.
+            // Finally, we would get e.g. 20 partitions. Each partition should contain a portion of the original hash table and input data.
+            // A sub HashAggExecutor would be used to consume each partition one by one.
+            // If memory is still not enough in the sub HashAggExecutor, it will spill its hash table and input recursively.
+            let mut agg_spill_manager = AggSpillManager::new(
+                &self.identity,
+                DEFAULT_SPILL_PARTITION_NUM,
+                self.group_key_types.clone(),
+                self.aggs.iter().map(|agg| agg.return_type()).collect(),
+                child_schema.data_types(),
+                self.chunk_size,
+            )?;
+            agg_spill_manager.init_writers().await?;
+
+            let mut memory_usage_diff = 0;
+            // Spill agg states.
+            for (key, states) in groups {
+                let key_row = key.deserialize(&self.group_key_types)?;
+                let mut agg_datums = vec![];
+                for (agg, state) in self.aggs.iter().zip_eq_fast(states) {
+                    let encode_state = agg.encode_state(&state)?;
+                    memory_usage_diff -= state.estimated_size() as i64;
+                    agg_datums.push(encode_state);
                 }
+                let agg_state_row = OwnedRow::from_iter(agg_datums.into_iter());
+                let hash_code = agg_spill_manager.spill_build_hasher.hash_one(key);
+                agg_spill_manager
+                    .write_agg_state_row(key_row.chain(agg_state_row), hash_code)
+                    .await?;
+            }
+
+            // Release memory occupied by agg hash map.
+            self.mem_context.add(memory_usage_diff);
+
+            // Spill input chunks.
+            #[for_await]
+            for chunk in input_stream {
+                let chunk: DataChunk = chunk?;
+                let hash_codes = chunk.get_hash_values(
+                    self.group_key_columns.as_slice(),
+                    agg_spill_manager.spill_build_hasher,
+                );
+                agg_spill_manager
+                    .write_input_chunk(
+                        chunk,
+                        hash_codes
+                            .into_iter()
+                            .map(|hash_code| hash_code.value())
+                            .collect(),
+                    )
+                    .await?;
             }
-            if !has_next {
-                break; // exit loop
+
+            agg_spill_manager.close_writers().await?;
+
+            // Process each partition one by one.
+            for i in 0..agg_spill_manager.partition_num {
+                let partition_size = agg_spill_manager.estimate_partition_size(i).await?;
+
+                let agg_state_stream = agg_spill_manager.read_agg_state_partition(i).await?;
+                let input_stream = agg_spill_manager.read_input_partition(i).await?;
+
+                let sub_hash_agg_executor: HashAggExecutor<K> = HashAggExecutor::new_inner(
+                    self.aggs.clone(),
+                    self.group_key_columns.clone(),
+                    self.group_key_types.clone(),
+                    self.schema.clone(),
+                    Box::new(WrapStreamExecutor::new(child_schema.clone(), input_stream)),
+                    Some(Box::new(WrapStreamExecutor::new(
+                        self.schema.clone(),
+                        agg_state_stream,
+                    ))),
+                    format!("{}-sub{}", self.identity.clone(), i),
+                    self.chunk_size,
+                    self.mem_context.clone(),
+                    self.enable_spill,
+                    Some(partition_size),
+                    self.shutdown_rx.clone(),
+                );
+
+                debug!(
+                    "create sub_hash_agg {} for hash_agg {} to spill",
+                    sub_hash_agg_executor.identity, self.identity
+                );
+
+                let sub_hash_agg_stream = Box::new(sub_hash_agg_executor).execute();
+
+                #[for_await]
+                for chunk in sub_hash_agg_stream {
+                    let chunk = chunk?;
+                    yield chunk;
+                }
+
+                // Clear files of the current partition.
+                agg_spill_manager.clear_partition(i).await?;
             }
+        } else {
+            // Don't use `into_iter` here, it may cause memory leak.
+            let mut result = groups.iter_mut();
+            let cardinality = self.chunk_size;
+            loop {
+                let mut group_builders: Vec<_> = self
+                    .group_key_types
+                    .iter()
+                    .map(|datatype| datatype.create_array_builder(cardinality))
+                    .collect();
+
+                let mut agg_builders: Vec<_> = self
+                    .aggs
+                    .iter()
+                    .map(|agg| agg.return_type().create_array_builder(cardinality))
+                    .collect();
+
+                let mut has_next = false;
+                let mut array_len = 0;
+                for (key, states) in result.by_ref().take(cardinality) {
+                    self.shutdown_rx.check()?;
+                    has_next = true;
+                    array_len += 1;
+                    key.deserialize_to_builders(&mut group_builders[..], &self.group_key_types)?;
+                    for ((agg, state), builder) in (self.aggs.iter())
+                        .zip_eq_fast(states)
+                        .zip_eq_fast(&mut agg_builders)
+                    {
+                        let result = agg.get_result(state).await?;
+                        builder.append(result);
+                    }
+                }
+                if !has_next {
+                    break; // exit loop
+                }
 
-            let columns = group_builders
-                .into_iter()
-                .chain(agg_builders)
-                .map(|b| b.finish().into())
-                .collect::<Vec<_>>();
+                let columns = group_builders
+                    .into_iter()
+                    .chain(agg_builders)
+                    .map(|b| b.finish().into())
+                    .collect::<Vec<_>>();
 
-            let output = DataChunk::new(columns, array_len);
-            yield output;
+                let output = DataChunk::new(columns, array_len);
+                yield output;
+            }
         }
     }
 }
@@ -305,10 +706,8 @@ mod tests {
     use std::alloc::{AllocError, Allocator, Global, Layout};
     use std::ptr::NonNull;
     use std::sync::atomic::{AtomicBool, Ordering};
-    use std::sync::Arc;
 
     use futures_async_stream::for_await;
-    use risingwave_common::catalog::{Field, Schema};
     use risingwave_common::metrics::LabelGuardedIntGauge;
     use risingwave_common::test_prelude::DataChunkTestExt;
     use risingwave_pb::data::data_type::TypeName;
@@ -323,7 +722,7 @@ mod tests {
 
     #[tokio::test]
     async fn execute_int32_grouped() {
-        let parent_mem = MemoryContext::root(LabelGuardedIntGauge::<4>::test_int_gauge());
+        let parent_mem = MemoryContext::root(LabelGuardedIntGauge::<4>::test_int_gauge(), u64::MAX);
         {
             let src_exec = Box::new(MockExecutor::with_chunk(
                 DataChunk::from_pretty(
@@ -361,6 +760,7 @@ mod tests {
                 order_by: vec![],
                 filter: None,
                 direct_args: vec![],
+                udf: None,
             };
 
             let agg_prost = HashAggNode {
@@ -379,6 +779,7 @@ mod tests {
                 "HashAggExecutor".to_string(),
                 CHUNK_SIZE,
                 mem_context.clone(),
+                false,
                 ShutdownToken::empty(),
             )
             .unwrap();
@@ -436,6 +837,7 @@ mod tests {
             order_by: vec![],
             filter: None,
             direct_args: vec![],
+            udf: None,
         };
 
         let agg_prost = HashAggNode {
@@ -450,6 +852,7 @@ mod tests {
             "HashAggExecutor".to_string(),
             CHUNK_SIZE,
             MemoryContext::none(),
+            false,
             ShutdownToken::empty(),
         )
         .unwrap();
@@ -477,6 +880,7 @@ mod tests {
 
             #[derive(Clone)]
             struct MyAlloc {
+                #[expect(dead_code)]
                 inner: Arc<MyAllocInner>,
             }
 
@@ -549,6 +953,7 @@ mod tests {
             order_by: vec![],
             filter: None,
             direct_args: vec![],
+            udf: None,
         };
 
         let agg_prost = HashAggNode {
@@ -564,6 +969,7 @@ mod tests {
             "HashAggExecutor".to_string(),
             CHUNK_SIZE,
             MemoryContext::none(),
+            false,
             shutdown_rx,
         )
         .unwrap();
diff --git a/src/batch/src/executor/hop_window.rs b/src/batch/src/executor/hop_window.rs
index d287efc4667d7..5345aeaa90552 100644
--- a/src/batch/src/executor/hop_window.rs
+++ b/src/batch/src/executor/hop_window.rs
@@ -32,7 +32,6 @@ pub struct HopWindowExecutor {
     child: BoxedExecutor,
     identity: String,
     schema: Schema,
-    time_col_idx: usize,
     window_slide: Interval,
     window_size: Interval,
     window_start_exprs: Vec<BoxedExpression>,
@@ -52,7 +51,6 @@ impl BoxedExecutorBuilder for HopWindowExecutor {
             source.plan_node().get_node_body().unwrap(),
             NodeBody::HopWindow
         )?;
-        let time_col = hop_window_node.get_time_col() as usize;
         let window_slide = hop_window_node.get_window_slide()?.into();
         let window_size = hop_window_node.get_window_size()?.into();
         let output_indices = hop_window_node
@@ -74,6 +72,7 @@ impl BoxedExecutorBuilder for HopWindowExecutor {
             .try_collect()?;
         assert_eq!(window_start_exprs.len(), window_end_exprs.len());
 
+        let time_col = hop_window_node.get_time_col() as usize;
         let time_col_data_type = child.schema().fields()[time_col].data_type();
         let output_type = DataType::window_of(&time_col_data_type).unwrap();
         let original_schema: Schema = child
@@ -93,7 +92,6 @@ impl BoxedExecutorBuilder for HopWindowExecutor {
         Ok(Box::new(HopWindowExecutor::new(
             child,
             output_indices_schema,
-            time_col,
             window_slide,
             window_size,
             source.plan_node().get_identity().clone(),
@@ -109,7 +107,6 @@ impl HopWindowExecutor {
     fn new(
         child: BoxedExecutor,
         schema: Schema,
-        time_col_idx: usize,
         window_slide: Interval,
         window_size: Interval,
         identity: String,
@@ -121,7 +118,6 @@ impl HopWindowExecutor {
             child,
             identity,
             schema,
-            time_col_idx,
             window_slide,
             window_size,
             window_start_exprs,
@@ -209,10 +205,8 @@ impl HopWindowExecutor {
 #[cfg(test)]
 mod tests {
     use futures::stream::StreamExt;
-    use risingwave_common::array::{DataChunk, DataChunkTestExt};
-    use risingwave_common::catalog::{Field, Schema};
+    use risingwave_common::array::DataChunkTestExt;
     use risingwave_common::types::test_utils::IntervalTestExt;
-    use risingwave_common::types::DataType;
     use risingwave_expr::expr::test_utils::make_hop_window_expression;
 
     use super::*;
@@ -256,7 +250,6 @@ mod tests {
         Box::new(HopWindowExecutor::new(
             Box::new(mock_executor),
             schema,
-            2,
             window_slide,
             window_size,
             "test".to_string(),
diff --git a/src/batch/src/executor/iceberg_scan.rs b/src/batch/src/executor/iceberg_scan.rs
index e20b1aaedb8fe..31abcc73a2aac 100644
--- a/src/batch/src/executor/iceberg_scan.rs
+++ b/src/batch/src/executor/iceberg_scan.rs
@@ -13,13 +13,13 @@
 // limitations under the License.
 
 use std::hash::{DefaultHasher, Hash, Hasher};
-use std::sync::Arc;
 
 use anyhow::anyhow;
 use arrow_array::RecordBatch;
 use futures_async_stream::try_stream;
 use futures_util::stream::StreamExt;
 use icelake::io::{FileScan, TableScan};
+use risingwave_common::array::arrow::{FromArrow, IcebergArrowConvert};
 use risingwave_common::catalog::Schema;
 use risingwave_connector::sink::iceberg::IcebergConfig;
 
@@ -150,12 +150,7 @@ impl IcebergScanExecutor {
     }
 
     fn record_batch_to_chunk(record_batch: RecordBatch) -> Result<DataChunk, BatchError> {
-        let mut columns = Vec::with_capacity(record_batch.num_columns());
-        for array in record_batch.columns() {
-            let column = Arc::new(array.try_into()?);
-            columns.push(column);
-        }
-        Ok(DataChunk::new(columns, record_batch.num_rows()))
+        Ok(IcebergArrowConvert.from_record_batch(&record_batch)?)
     }
 }
 
diff --git a/src/batch/src/executor/insert.rs b/src/batch/src/executor/insert.rs
index aa4063be84bb5..5674e17557e6b 100644
--- a/src/batch/src/executor/insert.rs
+++ b/src/batch/src/executor/insert.rs
@@ -43,6 +43,7 @@ pub struct InsertExecutor {
     table_version_id: TableVersionId,
     dml_manager: DmlManagerRef,
     child: BoxedExecutor,
+    #[expect(dead_code)]
     chunk_size: usize,
     schema: Schema,
     identity: String,
@@ -264,18 +265,16 @@ impl BoxedExecutorBuilder for InsertExecutor {
 #[cfg(test)]
 mod tests {
     use std::ops::Bound;
-    use std::sync::Arc;
 
     use assert_matches::assert_matches;
-    use foyer::memory::CacheContext;
+    use foyer::CacheContext;
     use futures::StreamExt;
-    use itertools::Itertools;
     use risingwave_common::array::{Array, ArrayImpl, I32Array, StructArray};
     use risingwave_common::catalog::{
         schema_test_utils, ColumnDesc, ColumnId, INITIAL_TABLE_VERSION_ID,
     };
     use risingwave_common::transaction::transaction_message::TxnMsg;
-    use risingwave_common::types::{DataType, StructType};
+    use risingwave_common::types::StructType;
     use risingwave_dml::dml_manager::DmlManager;
     use risingwave_storage::hummock::CachePolicy;
     use risingwave_storage::memory::MemoryStateStore;
diff --git a/src/batch/src/executor/join/chunked_data.rs b/src/batch/src/executor/join/chunked_data.rs
index 066ca88d4bbaa..158f9cba9f289 100644
--- a/src/batch/src/executor/join/chunked_data.rs
+++ b/src/batch/src/executor/join/chunked_data.rs
@@ -122,17 +122,6 @@ impl<V> ChunkedData<V> {
             chunk_offsets: &self.chunk_offsets,
         }
     }
-
-    pub(super) fn next_row_id(&self, cur: RowId) -> Option<RowId> {
-        let current_chunk_row_count =
-            self.chunk_offsets[cur.chunk_id() + 1] - self.chunk_offsets[cur.chunk_id()];
-        let next = cur.next_row(current_chunk_row_count);
-        if (next.chunk_id() + 1) >= self.chunk_offsets.len() {
-            None
-        } else {
-            Some(next)
-        }
-    }
 }
 
 impl<V> Index<RowId> for ChunkedData<V> {
diff --git a/src/batch/src/executor/join/hash_join.rs b/src/batch/src/executor/join/hash_join.rs
index c9bc0caad7cef..f6440c74430a6 100644
--- a/src/batch/src/executor/join/hash_join.rs
+++ b/src/batch/src/executor/join/hash_join.rs
@@ -17,8 +17,10 @@ use std::iter::empty;
 use std::marker::PhantomData;
 use std::sync::Arc;
 
+use bytes::Bytes;
 use futures_async_stream::try_stream;
 use itertools::Itertools;
+use prost::Message;
 use risingwave_common::array::{Array, DataChunk, RowRef};
 use risingwave_common::buffer::{Bitmap, BitmapBuilder};
 use risingwave_common::catalog::Schema;
@@ -31,13 +33,18 @@ use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common_estimate_size::EstimateSize;
 use risingwave_expr::expr::{build_from_prost, BoxedExpression, Expression};
 use risingwave_pb::batch_plan::plan_node::NodeBody;
+use risingwave_pb::data::DataChunk as PbDataChunk;
 
 use super::{ChunkedData, JoinType, RowId};
 use crate::error::{BatchError, Result};
 use crate::executor::{
     BoxedDataChunkStream, BoxedExecutor, BoxedExecutorBuilder, Executor, ExecutorBuilder,
+    WrapStreamExecutor,
 };
 use crate::risingwave_common::hash::NullBitmap;
+use crate::spill::spill_op::{
+    SpillBuildHasher, SpillOp, DEFAULT_SPILL_PARTITION_NUM, SPILL_AT_LEAST_MEMORY,
+};
 use crate::task::{BatchTaskContext, ShutdownToken};
 
 /// Hash Join Executor
@@ -47,12 +54,13 @@ use crate::task::{BatchTaskContext, ShutdownToken};
 /// 2. Iterate over the probe side (i.e. left table) and compute the hash value of each row.
 ///    Then find the matched build side row for each probe side row in the hash map.
 /// 3. Concatenate the matched pair of probe side row and build side row into a single row and push
-/// it into the data chunk builder.
+///    it into the data chunk builder.
 /// 4. Yield chunks from the builder.
 pub struct HashJoinExecutor<K> {
     /// Join type e.g. inner, left outer, ...
     join_type: JoinType,
     /// Output schema without applying `output_indices`
+    #[expect(dead_code)]
     original_schema: Schema,
     /// Output schema after applying `output_indices`
     schema: Schema,
@@ -67,13 +75,17 @@ pub struct HashJoinExecutor<K> {
     /// Column indices of right keys in equi join
     build_key_idxs: Vec<usize>,
     /// Non-equi join condition (optional)
-    cond: Option<BoxedExpression>,
+    cond: Option<Arc<BoxedExpression>>,
     /// Whether or not to enable 'IS NOT DISTINCT FROM' semantics for a specific probe/build key
     /// column
     null_matched: Vec<bool>,
     identity: String,
     chunk_size: usize,
 
+    enable_spill: bool,
+    /// The upper bound of memory usage for this executor.
+    memory_upper_bound: Option<u64>,
+
     shutdown_rx: ShutdownToken,
 
     mem_ctx: MemoryContext,
@@ -220,22 +232,252 @@ struct RightNonEquiJoinState {
     build_row_matched: ChunkedData<bool>,
 }
 
+pub struct JoinSpillManager {
+    op: SpillOp,
+    partition_num: usize,
+    probe_side_writers: Vec<opendal::Writer>,
+    build_side_writers: Vec<opendal::Writer>,
+    probe_side_chunk_builders: Vec<DataChunkBuilder>,
+    build_side_chunk_builders: Vec<DataChunkBuilder>,
+    spill_build_hasher: SpillBuildHasher,
+    probe_side_data_types: Vec<DataType>,
+    build_side_data_types: Vec<DataType>,
+    spill_chunk_size: usize,
+}
+
+/// `JoinSpillManager` is used to manage how to write spill data file and read them back.
+/// The spill data first need to be partitioned. Each partition contains 2 files: `join_probe_side_file` and `join_build_side_file`.
+/// The spill file consume a data chunk and serialize the chunk into a protobuf bytes.
+/// Finally, spill file content will look like the below.
+/// The file write pattern is append-only and the read pattern is sequential scan.
+/// This can maximize the disk IO performance.
+///
+/// ```text
+/// [proto_len]
+/// [proto_bytes]
+/// ...
+/// [proto_len]
+/// [proto_bytes]
+/// ```
+impl JoinSpillManager {
+    pub fn new(
+        join_identity: &String,
+        partition_num: usize,
+        probe_side_data_types: Vec<DataType>,
+        build_side_data_types: Vec<DataType>,
+        spill_chunk_size: usize,
+    ) -> Result<Self> {
+        let suffix_uuid = uuid::Uuid::new_v4();
+        let dir = format!("/{}-{}/", join_identity, suffix_uuid);
+        let op = SpillOp::create(dir)?;
+        let probe_side_writers = Vec::with_capacity(partition_num);
+        let build_side_writers = Vec::with_capacity(partition_num);
+        let probe_side_chunk_builders = Vec::with_capacity(partition_num);
+        let build_side_chunk_builders = Vec::with_capacity(partition_num);
+        let spill_build_hasher = SpillBuildHasher(suffix_uuid.as_u64_pair().1);
+        Ok(Self {
+            op,
+            partition_num,
+            probe_side_writers,
+            build_side_writers,
+            probe_side_chunk_builders,
+            build_side_chunk_builders,
+            spill_build_hasher,
+            probe_side_data_types,
+            build_side_data_types,
+            spill_chunk_size,
+        })
+    }
+
+    pub async fn init_writers(&mut self) -> Result<()> {
+        for i in 0..self.partition_num {
+            let join_probe_side_partition_file_name = format!("join-probe-side-p{}", i);
+            let w = self
+                .op
+                .writer_with(&join_probe_side_partition_file_name)
+                .await?;
+            self.probe_side_writers.push(w);
+
+            let join_build_side_partition_file_name = format!("join-build-side-p{}", i);
+            let w = self
+                .op
+                .writer_with(&join_build_side_partition_file_name)
+                .await?;
+            self.build_side_writers.push(w);
+            self.probe_side_chunk_builders.push(DataChunkBuilder::new(
+                self.probe_side_data_types.clone(),
+                self.spill_chunk_size,
+            ));
+            self.build_side_chunk_builders.push(DataChunkBuilder::new(
+                self.build_side_data_types.clone(),
+                self.spill_chunk_size,
+            ));
+        }
+        Ok(())
+    }
+
+    pub async fn write_probe_side_chunk(
+        &mut self,
+        chunk: DataChunk,
+        hash_codes: Vec<u64>,
+    ) -> Result<()> {
+        let (columns, vis) = chunk.into_parts_v2();
+        for partition in 0..self.partition_num {
+            let new_vis = vis.clone()
+                & Bitmap::from_iter(
+                    hash_codes
+                        .iter()
+                        .map(|hash_code| (*hash_code as usize % self.partition_num) == partition),
+                );
+            let new_chunk = DataChunk::from_parts(columns.clone(), new_vis);
+            for output_chunk in self.probe_side_chunk_builders[partition].append_chunk(new_chunk) {
+                let chunk_pb: PbDataChunk = output_chunk.to_protobuf();
+                let buf = Message::encode_to_vec(&chunk_pb);
+                let len_bytes = Bytes::copy_from_slice(&(buf.len() as u32).to_le_bytes());
+                self.probe_side_writers[partition].write(len_bytes).await?;
+                self.probe_side_writers[partition].write(buf).await?;
+            }
+        }
+        Ok(())
+    }
+
+    pub async fn write_build_side_chunk(
+        &mut self,
+        chunk: DataChunk,
+        hash_codes: Vec<u64>,
+    ) -> Result<()> {
+        let (columns, vis) = chunk.into_parts_v2();
+        for partition in 0..self.partition_num {
+            let new_vis = vis.clone()
+                & Bitmap::from_iter(
+                    hash_codes
+                        .iter()
+                        .map(|hash_code| (*hash_code as usize % self.partition_num) == partition),
+                );
+            let new_chunk = DataChunk::from_parts(columns.clone(), new_vis);
+            for output_chunk in self.build_side_chunk_builders[partition].append_chunk(new_chunk) {
+                let chunk_pb: PbDataChunk = output_chunk.to_protobuf();
+                let buf = Message::encode_to_vec(&chunk_pb);
+                let len_bytes = Bytes::copy_from_slice(&(buf.len() as u32).to_le_bytes());
+                self.build_side_writers[partition].write(len_bytes).await?;
+                self.build_side_writers[partition].write(buf).await?;
+            }
+        }
+        Ok(())
+    }
+
+    pub async fn close_writers(&mut self) -> Result<()> {
+        for partition in 0..self.partition_num {
+            if let Some(output_chunk) = self.probe_side_chunk_builders[partition].consume_all() {
+                let chunk_pb: PbDataChunk = output_chunk.to_protobuf();
+                let buf = Message::encode_to_vec(&chunk_pb);
+                let len_bytes = Bytes::copy_from_slice(&(buf.len() as u32).to_le_bytes());
+                self.probe_side_writers[partition].write(len_bytes).await?;
+                self.probe_side_writers[partition].write(buf).await?;
+            }
+
+            if let Some(output_chunk) = self.build_side_chunk_builders[partition].consume_all() {
+                let chunk_pb: PbDataChunk = output_chunk.to_protobuf();
+                let buf = Message::encode_to_vec(&chunk_pb);
+                let len_bytes = Bytes::copy_from_slice(&(buf.len() as u32).to_le_bytes());
+                self.build_side_writers[partition].write(len_bytes).await?;
+                self.build_side_writers[partition].write(buf).await?;
+            }
+        }
+
+        for mut w in self.probe_side_writers.drain(..) {
+            w.close().await?;
+        }
+        for mut w in self.build_side_writers.drain(..) {
+            w.close().await?;
+        }
+        Ok(())
+    }
+
+    async fn read_probe_side_partition(
+        &mut self,
+        partition: usize,
+    ) -> Result<BoxedDataChunkStream> {
+        let join_probe_side_partition_file_name = format!("join-probe-side-p{}", partition);
+        let r = self
+            .op
+            .reader_with(&join_probe_side_partition_file_name)
+            .await?;
+        Ok(SpillOp::read_stream(r))
+    }
+
+    async fn read_build_side_partition(
+        &mut self,
+        partition: usize,
+    ) -> Result<BoxedDataChunkStream> {
+        let join_build_side_partition_file_name = format!("join-build-side-p{}", partition);
+        let r = self
+            .op
+            .reader_with(&join_build_side_partition_file_name)
+            .await?;
+        Ok(SpillOp::read_stream(r))
+    }
+
+    pub async fn estimate_partition_size(&self, partition: usize) -> Result<u64> {
+        let join_probe_side_partition_file_name = format!("join-probe-side-p{}", partition);
+        let probe_size = self
+            .op
+            .stat(&join_probe_side_partition_file_name)
+            .await?
+            .content_length();
+        let join_build_side_partition_file_name = format!("join-build-side-p{}", partition);
+        let build_size = self
+            .op
+            .stat(&join_build_side_partition_file_name)
+            .await?
+            .content_length();
+        Ok(probe_size + build_size)
+    }
+
+    async fn clear_partition(&mut self, partition: usize) -> Result<()> {
+        let join_probe_side_partition_file_name = format!("join-probe-side-p{}", partition);
+        self.op.delete(&join_probe_side_partition_file_name).await?;
+        let join_build_side_partition_file_name = format!("join-build-side-p{}", partition);
+        self.op.delete(&join_build_side_partition_file_name).await?;
+        Ok(())
+    }
+}
+
 impl<K: HashKey> HashJoinExecutor<K> {
     #[try_stream(boxed, ok = DataChunk, error = BatchError)]
     async fn do_execute(self: Box<Self>) {
+        let mut need_to_spill = false;
+        // If the memory upper bound is less than 1MB, we don't need to check memory usage.
+        let check_memory = match self.memory_upper_bound {
+            Some(upper_bound) => upper_bound > SPILL_AT_LEAST_MEMORY,
+            None => true,
+        };
+
+        let probe_schema = self.probe_side_source.schema().clone();
+        let build_schema = self.build_side_source.schema().clone();
         let probe_data_types = self.probe_side_source.schema().data_types();
         let build_data_types = self.build_side_source.schema().data_types();
         let full_data_types = [probe_data_types.clone(), build_data_types.clone()].concat();
 
         let mut build_side = Vec::new_in(self.mem_ctx.global_allocator());
         let mut build_row_count = 0;
+        let mut build_side_stream = self.build_side_source.execute();
         #[for_await]
-        for build_chunk in self.build_side_source.execute() {
+        for build_chunk in &mut build_side_stream {
             let build_chunk = build_chunk?;
             if build_chunk.cardinality() > 0 {
                 build_row_count += build_chunk.cardinality();
-                self.mem_ctx.add(build_chunk.estimated_heap_size() as i64);
+                let chunk_estimated_heap_size = build_chunk.estimated_heap_size();
+                // push build_chunk to build_side before checking memory limit, otherwise we will lose that chunk when spilling.
                 build_side.push(build_chunk);
+                if !self.mem_ctx.add(chunk_estimated_heap_size as i64) && check_memory {
+                    if self.enable_spill {
+                        need_to_spill = true;
+                        break;
+                    } else {
+                        Err(BatchError::OutOfMemory(self.mem_ctx.mem_limit()))?;
+                    }
+                }
             }
         }
         let mut hash_map = JoinHashMap::with_capacity_and_hasher_in(
@@ -246,92 +488,235 @@ impl<K: HashKey> HashJoinExecutor<K> {
         let mut next_build_row_with_same_key =
             ChunkedData::with_chunk_sizes(build_side.iter().map(|c| c.capacity()))?;
 
-        let null_matched = K::Bitmap::from_bool_vec(self.null_matched);
+        let null_matched = K::Bitmap::from_bool_vec(self.null_matched.clone());
 
-        // Build hash map
-        for (build_chunk_id, build_chunk) in build_side.iter().enumerate() {
-            let build_keys = K::build_many(&self.build_key_idxs, build_chunk);
+        let mut mem_added_by_hash_table = 0;
+        if !need_to_spill {
+            // Build hash map
+            for (build_chunk_id, build_chunk) in build_side.iter().enumerate() {
+                let build_keys = K::build_many(&self.build_key_idxs, build_chunk);
 
-            for (build_row_id, (build_key, visible)) in build_keys
-                .into_iter()
-                .zip_eq_fast(build_chunk.visibility().iter())
-                .enumerate()
-            {
-                self.shutdown_rx.check()?;
-                if !visible {
-                    continue;
-                }
-                // Only insert key to hash map if it is consistent with the null safe restriction.
-                if build_key.null_bitmap().is_subset(&null_matched) {
-                    let row_id = RowId::new(build_chunk_id, build_row_id);
-                    self.mem_ctx.add(build_key.estimated_heap_size() as i64);
-                    next_build_row_with_same_key[row_id] = hash_map.insert(build_key, row_id);
+                for (build_row_id, (build_key, visible)) in build_keys
+                    .into_iter()
+                    .zip_eq_fast(build_chunk.visibility().iter())
+                    .enumerate()
+                {
+                    self.shutdown_rx.check()?;
+                    if !visible {
+                        continue;
+                    }
+                    // Only insert key to hash map if it is consistent with the null safe restriction.
+                    if build_key.null_bitmap().is_subset(&null_matched) {
+                        let row_id = RowId::new(build_chunk_id, build_row_id);
+                        let build_key_size = build_key.estimated_heap_size() as i64;
+                        mem_added_by_hash_table += build_key_size;
+                        if !self.mem_ctx.add(build_key_size) && check_memory {
+                            if self.enable_spill {
+                                need_to_spill = true;
+                                break;
+                            } else {
+                                Err(BatchError::OutOfMemory(self.mem_ctx.mem_limit()))?;
+                            }
+                        }
+                        next_build_row_with_same_key[row_id] = hash_map.insert(build_key, row_id);
+                    }
                 }
             }
         }
 
-        let params = EquiJoinParams::new(
-            self.probe_side_source,
-            probe_data_types,
-            self.probe_key_idxs,
-            build_side,
-            build_data_types,
-            full_data_types,
-            hash_map,
-            next_build_row_with_same_key,
-            self.chunk_size,
-            self.shutdown_rx.clone(),
-        );
+        if need_to_spill {
+            // A spilling version of hash join based on the RFC: Spill Hash Join https://github.com/risingwavelabs/rfcs/pull/91
+            // When HashJoinExecutor told memory is insufficient, JoinSpillManager will start to partition the hash table and spill to disk.
+            // After spilling the hash table, JoinSpillManager will consume all chunks from its build side input executor and probe side input executor.
+            // Finally, we would get e.g. 20 partitions. Each partition should contain a portion of the original build side input and probr side input data.
+            // A sub HashJoinExecutor would be used to consume each partition one by one.
+            // If memory is still not enough in the sub HashJoinExecutor, it will spill its inputs recursively.
+            let mut join_spill_manager = JoinSpillManager::new(
+                &self.identity,
+                DEFAULT_SPILL_PARTITION_NUM,
+                probe_data_types.clone(),
+                build_data_types.clone(),
+                self.chunk_size,
+            )?;
+            join_spill_manager.init_writers().await?;
+
+            // Release memory occupied by the hash map
+            self.mem_ctx.add(-mem_added_by_hash_table);
+            drop(hash_map);
+            drop(next_build_row_with_same_key);
+
+            // Spill buffered build side chunks
+            for chunk in build_side {
+                // Release the memory occupied by the buffered chunks
+                self.mem_ctx.add(-(chunk.estimated_heap_size() as i64));
+                let hash_codes = chunk.get_hash_values(
+                    self.build_key_idxs.as_slice(),
+                    join_spill_manager.spill_build_hasher,
+                );
+                join_spill_manager
+                    .write_build_side_chunk(
+                        chunk,
+                        hash_codes
+                            .into_iter()
+                            .map(|hash_code| hash_code.value())
+                            .collect(),
+                    )
+                    .await?;
+            }
 
-        if let Some(cond) = self.cond.as_ref() {
-            let stream = match self.join_type {
-                JoinType::Inner => Self::do_inner_join_with_non_equi_condition(params, cond),
-                JoinType::LeftOuter => {
-                    Self::do_left_outer_join_with_non_equi_condition(params, cond)
-                }
-                JoinType::LeftSemi => Self::do_left_semi_join_with_non_equi_condition(params, cond),
-                JoinType::LeftAnti => Self::do_left_anti_join_with_non_equi_condition(params, cond),
-                JoinType::RightOuter => {
-                    Self::do_right_outer_join_with_non_equi_condition(params, cond)
-                }
-                JoinType::RightSemi => {
-                    Self::do_right_semi_anti_join_with_non_equi_condition::<false>(params, cond)
-                }
-                JoinType::RightAnti => {
-                    Self::do_right_semi_anti_join_with_non_equi_condition::<true>(params, cond)
-                }
-                JoinType::FullOuter => {
-                    Self::do_full_outer_join_with_non_equi_condition(params, cond)
-                }
-            };
-            // For non-equi join, we need an output chunk builder to align the output chunks.
-            let mut output_chunk_builder =
-                DataChunkBuilder::new(self.schema.data_types(), self.chunk_size);
+            // Spill build side chunks
             #[for_await]
-            for chunk in stream {
-                for output_chunk in
-                    output_chunk_builder.append_chunk(chunk?.project(&self.output_indices))
-                {
-                    yield output_chunk
-                }
+            for chunk in build_side_stream {
+                let chunk = chunk?;
+                let hash_codes = chunk.get_hash_values(
+                    self.build_key_idxs.as_slice(),
+                    join_spill_manager.spill_build_hasher,
+                );
+                join_spill_manager
+                    .write_build_side_chunk(
+                        chunk,
+                        hash_codes
+                            .into_iter()
+                            .map(|hash_code| hash_code.value())
+                            .collect(),
+                    )
+                    .await?;
             }
-            if let Some(output_chunk) = output_chunk_builder.consume_all() {
-                yield output_chunk
+
+            // Spill probe side chunks
+            #[for_await]
+            for chunk in self.probe_side_source.execute() {
+                let chunk = chunk?;
+                let hash_codes = chunk.get_hash_values(
+                    self.probe_key_idxs.as_slice(),
+                    join_spill_manager.spill_build_hasher,
+                );
+                join_spill_manager
+                    .write_probe_side_chunk(
+                        chunk,
+                        hash_codes
+                            .into_iter()
+                            .map(|hash_code| hash_code.value())
+                            .collect(),
+                    )
+                    .await?;
+            }
+
+            join_spill_manager.close_writers().await?;
+
+            // Process each partition one by one.
+            for i in 0..join_spill_manager.partition_num {
+                let partition_size = join_spill_manager.estimate_partition_size(i).await?;
+                let probe_side_stream = join_spill_manager.read_probe_side_partition(i).await?;
+                let build_side_stream = join_spill_manager.read_build_side_partition(i).await?;
+
+                let sub_hash_join_executor: HashJoinExecutor<K> = HashJoinExecutor::new_inner(
+                    self.join_type,
+                    self.output_indices.clone(),
+                    Box::new(WrapStreamExecutor::new(
+                        probe_schema.clone(),
+                        probe_side_stream,
+                    )),
+                    Box::new(WrapStreamExecutor::new(
+                        build_schema.clone(),
+                        build_side_stream,
+                    )),
+                    self.probe_key_idxs.clone(),
+                    self.build_key_idxs.clone(),
+                    self.null_matched.clone(),
+                    self.cond.clone(),
+                    format!("{}-sub{}", self.identity.clone(), i),
+                    self.chunk_size,
+                    self.enable_spill,
+                    Some(partition_size),
+                    self.shutdown_rx.clone(),
+                    self.mem_ctx.clone(),
+                );
+
+                debug!(
+                    "create sub_hash_join {} for hash_join {} to spill",
+                    sub_hash_join_executor.identity, self.identity
+                );
+
+                let sub_hash_join_executor = Box::new(sub_hash_join_executor).execute();
+
+                #[for_await]
+                for chunk in sub_hash_join_executor {
+                    let chunk = chunk?;
+                    yield chunk;
+                }
+
+                // Clear files of the current partition.
+                join_spill_manager.clear_partition(i).await?;
             }
         } else {
-            let stream = match self.join_type {
-                JoinType::Inner => Self::do_inner_join(params),
-                JoinType::LeftOuter => Self::do_left_outer_join(params),
-                JoinType::LeftSemi => Self::do_left_semi_anti_join::<false>(params),
-                JoinType::LeftAnti => Self::do_left_semi_anti_join::<true>(params),
-                JoinType::RightOuter => Self::do_right_outer_join(params),
-                JoinType::RightSemi => Self::do_right_semi_anti_join::<false>(params),
-                JoinType::RightAnti => Self::do_right_semi_anti_join::<true>(params),
-                JoinType::FullOuter => Self::do_full_outer_join(params),
-            };
-            #[for_await]
-            for chunk in stream {
-                yield chunk?.project(&self.output_indices)
+            let params = EquiJoinParams::new(
+                self.probe_side_source,
+                probe_data_types,
+                self.probe_key_idxs,
+                build_side,
+                build_data_types,
+                full_data_types,
+                hash_map,
+                next_build_row_with_same_key,
+                self.chunk_size,
+                self.shutdown_rx.clone(),
+            );
+
+            if let Some(cond) = self.cond.as_ref() {
+                let stream = match self.join_type {
+                    JoinType::Inner => Self::do_inner_join_with_non_equi_condition(params, cond),
+                    JoinType::LeftOuter => {
+                        Self::do_left_outer_join_with_non_equi_condition(params, cond)
+                    }
+                    JoinType::LeftSemi => {
+                        Self::do_left_semi_join_with_non_equi_condition(params, cond)
+                    }
+                    JoinType::LeftAnti => {
+                        Self::do_left_anti_join_with_non_equi_condition(params, cond)
+                    }
+                    JoinType::RightOuter => {
+                        Self::do_right_outer_join_with_non_equi_condition(params, cond)
+                    }
+                    JoinType::RightSemi => {
+                        Self::do_right_semi_anti_join_with_non_equi_condition::<false>(params, cond)
+                    }
+                    JoinType::RightAnti => {
+                        Self::do_right_semi_anti_join_with_non_equi_condition::<true>(params, cond)
+                    }
+                    JoinType::FullOuter => {
+                        Self::do_full_outer_join_with_non_equi_condition(params, cond)
+                    }
+                };
+                // For non-equi join, we need an output chunk builder to align the output chunks.
+                let mut output_chunk_builder =
+                    DataChunkBuilder::new(self.schema.data_types(), self.chunk_size);
+                #[for_await]
+                for chunk in stream {
+                    for output_chunk in
+                        output_chunk_builder.append_chunk(chunk?.project(&self.output_indices))
+                    {
+                        yield output_chunk
+                    }
+                }
+                if let Some(output_chunk) = output_chunk_builder.consume_all() {
+                    yield output_chunk
+                }
+            } else {
+                let stream = match self.join_type {
+                    JoinType::Inner => Self::do_inner_join(params),
+                    JoinType::LeftOuter => Self::do_left_outer_join(params),
+                    JoinType::LeftSemi => Self::do_left_semi_anti_join::<false>(params),
+                    JoinType::LeftAnti => Self::do_left_semi_anti_join::<true>(params),
+                    JoinType::RightOuter => Self::do_right_outer_join(params),
+                    JoinType::RightSemi => Self::do_right_semi_anti_join::<false>(params),
+                    JoinType::RightAnti => Self::do_right_semi_anti_join::<true>(params),
+                    JoinType::FullOuter => Self::do_full_outer_join(params),
+                };
+                #[for_await]
+                for chunk in stream {
+                    yield chunk?.project(&self.output_indices)
+                }
             }
         }
     }
@@ -580,7 +965,7 @@ impl<K: HashKey> HashJoinExecutor<K> {
                 }
                 shutdown_rx.check()?;
                 if !ANTI_JOIN {
-                    if hash_map.get(probe_key).is_some() {
+                    if hash_map.contains_key(probe_key) {
                         if let Some(spilled) = Self::append_one_probe_row(
                             &mut chunk_builder,
                             &probe_chunk,
@@ -1284,7 +1669,7 @@ impl<K: HashKey> HashJoinExecutor<K> {
     /// | 4 | 3 | 3 | - |
     ///
     /// 3. Remove duplicate rows with NULL build side. This is done by setting the visibility bitmap
-    /// of the chunk.
+    ///    of the chunk.
     ///
     /// | offset | v1 | v2 | v3 |
     /// |---|---|---|---|
@@ -1832,6 +2217,7 @@ impl BoxedExecutorBuilder for HashJoinExecutor<()> {
             identity: identity.clone(),
             right_key_types,
             chunk_size: context.context.get_config().developer.chunk_size,
+            enable_spill: context.context.get_config().enable_spill,
             shutdown_rx: context.shutdown_rx.clone(),
             mem_ctx: context.context.create_executor_mem_context(&identity),
         }
@@ -1851,6 +2237,7 @@ struct HashJoinExecutorArgs {
     identity: String,
     right_key_types: Vec<DataType>,
     chunk_size: usize,
+    enable_spill: bool,
     shutdown_rx: ShutdownToken,
     mem_ctx: MemoryContext,
 }
@@ -1867,9 +2254,10 @@ impl HashKeyDispatcher for HashJoinExecutorArgs {
             self.probe_key_idxs,
             self.build_key_idxs,
             self.null_matched,
-            self.cond,
+            self.cond.map(Arc::new),
             self.identity,
             self.chunk_size,
+            self.enable_spill,
             self.shutdown_rx,
             self.mem_ctx,
         ))
@@ -1890,9 +2278,45 @@ impl<K> HashJoinExecutor<K> {
         probe_key_idxs: Vec<usize>,
         build_key_idxs: Vec<usize>,
         null_matched: Vec<bool>,
-        cond: Option<BoxedExpression>,
+        cond: Option<Arc<BoxedExpression>>,
+        identity: String,
+        chunk_size: usize,
+        enable_spill: bool,
+        shutdown_rx: ShutdownToken,
+        mem_ctx: MemoryContext,
+    ) -> Self {
+        Self::new_inner(
+            join_type,
+            output_indices,
+            probe_side_source,
+            build_side_source,
+            probe_key_idxs,
+            build_key_idxs,
+            null_matched,
+            cond,
+            identity,
+            chunk_size,
+            enable_spill,
+            None,
+            shutdown_rx,
+            mem_ctx,
+        )
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn new_inner(
+        join_type: JoinType,
+        output_indices: Vec<usize>,
+        probe_side_source: BoxedExecutor,
+        build_side_source: BoxedExecutor,
+        probe_key_idxs: Vec<usize>,
+        build_key_idxs: Vec<usize>,
+        null_matched: Vec<bool>,
+        cond: Option<Arc<BoxedExpression>>,
         identity: String,
         chunk_size: usize,
+        enable_spill: bool,
+        memory_upper_bound: Option<u64>,
         shutdown_rx: ShutdownToken,
         mem_ctx: MemoryContext,
     ) -> Self {
@@ -1929,6 +2353,8 @@ impl<K> HashJoinExecutor<K> {
             identity,
             chunk_size,
             shutdown_rx,
+            enable_spill,
+            memory_upper_bound,
             mem_ctx,
             _phantom: PhantomData,
         }
@@ -1960,7 +2386,6 @@ mod tests {
     const CHUNK_SIZE: usize = 1024;
 
     struct DataChunkMerger {
-        data_types: Vec<DataType>,
         array_builders: Vec<ArrayBuilderImpl>,
         array_len: usize,
     }
@@ -1973,7 +2398,6 @@ mod tests {
                 .collect();
 
             Ok(Self {
-                data_types,
                 array_builders,
                 array_len: 0,
             })
@@ -2116,10 +2540,6 @@ mod tests {
             Box::new(executor)
         }
 
-        fn full_data_types(&self) -> Vec<DataType> {
-            [self.left_types.clone(), self.right_types.clone()].concat()
-        }
-
         fn output_data_types(&self) -> Vec<DataType> {
             let join_type = self.join_type;
             if join_type.keep_all() {
@@ -2157,7 +2577,7 @@ mod tests {
                 .collect();
 
             let cond = if has_non_equi_cond {
-                Some(Self::create_cond())
+                Some(Self::create_cond().into())
             } else {
                 None
             };
@@ -2175,6 +2595,7 @@ mod tests {
                 cond,
                 "HashJoinExecutor".to_string(),
                 chunk_size,
+                false,
                 shutdown_rx,
                 mem_ctx,
             ))
@@ -2204,7 +2625,7 @@ mod tests {
             right_executor: BoxedExecutor,
         ) {
             let parent_mem_context =
-                MemoryContext::root(LabelGuardedIntGauge::<4>::test_int_gauge());
+                MemoryContext::root(LabelGuardedIntGauge::<4>::test_int_gauge(), u64::MAX);
 
             {
                 let join_executor = self.create_join_executor_with_chunk_size_and_executors(
diff --git a/src/batch/src/executor/join/local_lookup_join.rs b/src/batch/src/executor/join/local_lookup_join.rs
index 17b257106fb5b..54d3185de3dca 100644
--- a/src/batch/src/executor/join/local_lookup_join.rs
+++ b/src/batch/src/executor/join/local_lookup_join.rs
@@ -21,7 +21,7 @@ use risingwave_common::buffer::BitmapBuilder;
 use risingwave_common::catalog::{ColumnDesc, Field, Schema};
 use risingwave_common::hash::table_distribution::TableDistribution;
 use risingwave_common::hash::{
-    ExpandedParallelUnitMapping, HashKey, HashKeyDispatcher, ParallelUnitId, VirtualNode,
+    ExpandedWorkerSlotMapping, HashKey, HashKeyDispatcher, VirtualNode, WorkerSlotId,
 };
 use risingwave_common::memory::MemoryContext;
 use risingwave_common::types::{DataType, Datum};
@@ -29,7 +29,6 @@ use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::scan_range::ScanRange;
 use risingwave_common::util::tracing::TracingContext;
-use risingwave_common::util::worker_util::get_pu_to_worker_mapping;
 use risingwave_expr::expr::{build_from_prost, BoxedExpression};
 use risingwave_pb::batch_plan::exchange_info::DistributionMode;
 use risingwave_pb::batch_plan::exchange_source::LocalExecutePlan::Plan;
@@ -52,7 +51,7 @@ use crate::task::{BatchTaskContext, ShutdownToken, TaskId};
 struct InnerSideExecutorBuilder<C> {
     table_desc: StorageTableDesc,
     table_distribution: TableDistribution,
-    vnode_mapping: ExpandedParallelUnitMapping,
+    vnode_mapping: ExpandedWorkerSlotMapping,
     outer_side_key_types: Vec<DataType>,
     inner_side_schema: Schema,
     inner_side_column_ids: Vec<i32>,
@@ -61,8 +60,9 @@ struct InnerSideExecutorBuilder<C> {
     context: C,
     task_id: TaskId,
     epoch: BatchQueryEpoch,
-    pu_to_worker_mapping: HashMap<ParallelUnitId, WorkerNode>,
-    pu_to_scan_range_mapping: HashMap<ParallelUnitId, Vec<(ScanRange, VirtualNode)>>,
+    worker_slot_mapping: HashMap<WorkerSlotId, WorkerNode>,
+    worker_slot_to_scan_range_mapping: HashMap<WorkerSlotId, Vec<(ScanRange, VirtualNode)>>,
+    #[expect(dead_code)]
     chunk_size: usize,
     shutdown_rx: ShutdownToken,
     next_stage_id: usize,
@@ -91,8 +91,8 @@ impl<C: BatchTaskContext> InnerSideExecutorBuilder<C> {
 
     /// Creates the `RowSeqScanNode` that will be used for scanning the inner side table
     /// based on the passed `scan_range` and virtual node.
-    fn create_row_seq_scan_node(&self, id: &ParallelUnitId) -> Result<NodeBody> {
-        let list = self.pu_to_scan_range_mapping.get(id).unwrap();
+    fn create_row_seq_scan_node(&self, id: &WorkerSlotId) -> Result<NodeBody> {
+        let list = self.worker_slot_to_scan_range_mapping.get(id).unwrap();
         let mut scan_ranges = vec![];
         let mut vnode_bitmap = BitmapBuilder::zeroed(self.vnode_mapping.len());
 
@@ -114,11 +114,11 @@ impl<C: BatchTaskContext> InnerSideExecutorBuilder<C> {
     }
 
     /// Creates the `PbExchangeSource` using the given `id`.
-    fn build_prost_exchange_source(&self, id: &ParallelUnitId) -> Result<PbExchangeSource> {
+    fn build_prost_exchange_source(&self, id: &WorkerSlotId) -> Result<PbExchangeSource> {
         let worker = self
-            .pu_to_worker_mapping
+            .worker_slot_mapping
             .get(id)
-            .context("No worker node found for the given parallel unit id.")?;
+            .context("No worker node found for the given worker slot id.")?;
 
         let local_execute_plan = LocalExecutePlan {
             plan: Some(PlanFragment {
@@ -145,7 +145,7 @@ impl<C: BatchTaskContext> InnerSideExecutorBuilder<C> {
                     // conflict.
                     query_id: self.task_id.query_id.clone(),
                     stage_id: self.task_id.stage_id + 10000 + self.next_stage_id as u32,
-                    task_id: *id,
+                    task_id: (*id).into(),
                 }),
                 output_id: 0,
             }),
@@ -160,7 +160,7 @@ impl<C: BatchTaskContext> InnerSideExecutorBuilder<C> {
 #[async_trait::async_trait]
 impl<C: BatchTaskContext> LookupExecutorBuilder for InnerSideExecutorBuilder<C> {
     fn reset(&mut self) {
-        self.pu_to_scan_range_mapping = HashMap::new();
+        self.worker_slot_to_scan_range_mapping = HashMap::new();
     }
 
     /// Adds the scan range made from the given `kwy_scalar_impls` into the parallel unit id
@@ -191,11 +191,11 @@ impl<C: BatchTaskContext> LookupExecutorBuilder for InnerSideExecutorBuilder<C>
         }
 
         let vnode = self.get_virtual_node(&scan_range)?;
-        let parallel_unit_id = self.vnode_mapping[vnode.to_index()];
+        let worker_slot_id = self.vnode_mapping[vnode.to_index()];
 
         let list = self
-            .pu_to_scan_range_mapping
-            .entry(parallel_unit_id)
+            .worker_slot_to_scan_range_mapping
+            .entry(worker_slot_id)
             .or_default();
         list.push((scan_range, vnode));
 
@@ -207,7 +207,7 @@ impl<C: BatchTaskContext> LookupExecutorBuilder for InnerSideExecutorBuilder<C>
     async fn build_executor(&mut self) -> Result<BoxedExecutor> {
         self.next_stage_id += 1;
         let mut sources = vec![];
-        for id in self.pu_to_scan_range_mapping.keys() {
+        for id in self.worker_slot_to_scan_range_mapping.keys() {
             sources.push(self.build_prost_exchange_source(id)?);
         }
 
@@ -368,11 +368,26 @@ impl BoxedExecutorBuilder for LocalLookupJoinExecutorBuilder {
 
         let null_safe = lookup_join_node.get_null_safe().to_vec();
 
-        let vnode_mapping = lookup_join_node.get_inner_side_vnode_mapping().to_vec();
+        let vnode_mapping = lookup_join_node
+            .get_inner_side_vnode_mapping()
+            .iter()
+            .copied()
+            .map(WorkerSlotId::from)
+            .collect_vec();
+
         assert!(!vnode_mapping.is_empty());
 
         let chunk_size = source.context.get_config().developer.chunk_size;
 
+        let worker_nodes = lookup_join_node.get_worker_nodes();
+        let worker_slot_mapping: HashMap<WorkerSlotId, WorkerNode> = worker_nodes
+            .iter()
+            .flat_map(|worker| {
+                (0..(worker.parallel_units.len()))
+                    .map(|i| (WorkerSlotId::new(worker.id, i), worker.clone()))
+            })
+            .collect();
+
         let inner_side_builder = InnerSideExecutorBuilder {
             table_desc: table_desc.clone(),
             table_distribution: TableDistribution::new_from_storage_table_desc(
@@ -388,11 +403,11 @@ impl BoxedExecutorBuilder for LocalLookupJoinExecutorBuilder {
             context: source.context().clone(),
             task_id: source.task_id.clone(),
             epoch: source.epoch(),
-            pu_to_worker_mapping: get_pu_to_worker_mapping(lookup_join_node.get_worker_nodes()),
-            pu_to_scan_range_mapping: HashMap::new(),
+            worker_slot_to_scan_range_mapping: HashMap::new(),
             chunk_size,
             shutdown_rx: source.shutdown_rx.clone(),
             next_stage_id: 0,
+            worker_slot_mapping,
         };
 
         let identity = source.plan_node().get_identity().clone();
@@ -491,10 +506,6 @@ mod tests {
 
     const CHUNK_SIZE: usize = 1024;
 
-    pub struct MockGatherExecutor {
-        chunks: Vec<DataChunk>,
-    }
-
     fn create_outer_side_input() -> BoxedExecutor {
         let schema = Schema {
             fields: vec![
diff --git a/src/batch/src/executor/join/lookup_join_base.rs b/src/batch/src/executor/join/lookup_join_base.rs
index 72af8ef418449..39a39b9a1424d 100644
--- a/src/batch/src/executor/join/lookup_join_base.rs
+++ b/src/batch/src/executor/join/lookup_join_base.rs
@@ -121,9 +121,9 @@ impl<K: HashKey> LookupJoinBase<K> {
             ]
             .concat();
 
-            // We need to temporary variable to record hash key heap size, since in each loop we
+            // We need to temporary variable to record heap size, since in each loop we
             // will free build side hash map, and the subtraction is not executed automatically.
-            let mut hash_key_heap_size = 0i64;
+            let mut tmp_heap_size = 0i64;
 
             let mut build_side = Vec::new_in(self.mem_ctx.global_allocator());
             let mut build_row_count = 0;
@@ -132,7 +132,9 @@ impl<K: HashKey> LookupJoinBase<K> {
                 let build_chunk = build_chunk?;
                 if build_chunk.cardinality() > 0 {
                     build_row_count += build_chunk.cardinality();
-                    self.mem_ctx.add(build_chunk.estimated_heap_size() as i64);
+                    let chunk_estimated_heap_size = build_chunk.estimated_heap_size() as i64;
+                    self.mem_ctx.add(chunk_estimated_heap_size);
+                    tmp_heap_size += chunk_estimated_heap_size;
                     build_side.push(build_chunk);
                 }
             }
@@ -160,8 +162,9 @@ impl<K: HashKey> LookupJoinBase<K> {
                     // restriction.
                     if build_key.null_bitmap().is_subset(&null_matched) {
                         let row_id = RowId::new(build_chunk_id, build_row_id);
-                        self.mem_ctx.add(build_key.estimated_heap_size() as i64);
-                        hash_key_heap_size += build_key.estimated_heap_size() as i64;
+                        let build_key_estimated_heap_size = build_key.estimated_heap_size() as i64;
+                        self.mem_ctx.add(build_key_estimated_heap_size);
+                        tmp_heap_size += build_key_estimated_heap_size;
                         next_build_row_with_same_key[row_id] = hash_map.insert(build_key, row_id);
                     }
                 }
@@ -230,7 +233,7 @@ impl<K: HashKey> LookupJoinBase<K> {
                 }
             }
 
-            self.mem_ctx.add(-hash_key_heap_size);
+            self.mem_ctx.add(-tmp_heap_size);
         }
     }
 }
diff --git a/src/batch/src/executor/join/mod.rs b/src/batch/src/executor/join/mod.rs
index 2f05174aca3db..6cead60aee0b4 100644
--- a/src/batch/src/executor/join/mod.rs
+++ b/src/batch/src/executor/join/mod.rs
@@ -52,14 +52,6 @@ pub enum JoinType {
 }
 
 impl JoinType {
-    #[inline(always)]
-    pub(super) fn need_join_remaining(self) -> bool {
-        matches!(
-            self,
-            JoinType::RightOuter | JoinType::RightAnti | JoinType::FullOuter
-        )
-    }
-
     pub fn from_prost(prost: JoinTypePb) -> Self {
         match prost {
             JoinTypePb::Inner => JoinType::Inner,
@@ -73,6 +65,19 @@ impl JoinType {
             JoinTypePb::Unspecified => unreachable!(),
         }
     }
+}
+
+#[cfg(test)]
+impl JoinType {
+    #![allow(dead_code)]
+
+    #[inline(always)]
+    pub(super) fn need_join_remaining(self) -> bool {
+        matches!(
+            self,
+            JoinType::RightOuter | JoinType::RightAnti | JoinType::FullOuter
+        )
+    }
 
     fn need_build(self) -> bool {
         match self {
diff --git a/src/batch/src/executor/join/nested_loop_join.rs b/src/batch/src/executor/join/nested_loop_join.rs
index 43b6448ad39f2..5d6d9b699b0e7 100644
--- a/src/batch/src/executor/join/nested_loop_join.rs
+++ b/src/batch/src/executor/join/nested_loop_join.rs
@@ -98,7 +98,9 @@ impl NestedLoopJoinExecutor {
             for chunk in self.left_child.execute() {
                 let c = chunk?;
                 trace!("Estimated chunk size is {:?}", c.estimated_heap_size());
-                self.mem_context.add(c.estimated_heap_size() as i64);
+                if !self.mem_context.add(c.estimated_heap_size() as i64) {
+                    Err(BatchError::OutOfMemory(self.mem_context.mem_limit()))?;
+                }
                 ret.push(c);
             }
             ret
@@ -527,8 +529,6 @@ mod tests {
     const CHUNK_SIZE: usize = 1024;
 
     struct TestFixture {
-        left_types: Vec<DataType>,
-        right_types: Vec<DataType>,
         join_type: JoinType,
     }
 
@@ -549,11 +549,7 @@ mod tests {
     /// ```
     impl TestFixture {
         fn with_join_type(join_type: JoinType) -> Self {
-            Self {
-                left_types: vec![DataType::Int32, DataType::Float32],
-                right_types: vec![DataType::Int32, DataType::Float64],
-                join_type,
-            }
+            Self { join_type }
         }
 
         fn create_left_executor(&self) -> BoxedExecutor {
diff --git a/src/batch/src/executor/limit.rs b/src/batch/src/executor/limit.rs
index eb57a78e1ab12..8c02ed082f2b2 100644
--- a/src/batch/src/executor/limit.rs
+++ b/src/batch/src/executor/limit.rs
@@ -150,8 +150,8 @@ mod tests {
     use std::vec;
 
     use futures_async_stream::for_await;
-    use risingwave_common::array::{Array, ArrayRef, BoolArray, DataChunk, PrimitiveArray};
-    use risingwave_common::catalog::{Field, Schema};
+    use risingwave_common::array::{Array, ArrayRef, BoolArray, PrimitiveArray};
+    use risingwave_common::catalog::Field;
     use risingwave_common::types::DataType;
     use risingwave_common::util::iter_util::ZipEqDebug;
 
diff --git a/src/batch/src/executor/log_row_seq_scan.rs b/src/batch/src/executor/log_row_seq_scan.rs
new file mode 100644
index 0000000000000..c46de5b3a2944
--- /dev/null
+++ b/src/batch/src/executor/log_row_seq_scan.rs
@@ -0,0 +1,250 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::ops::{Bound, Deref};
+use std::sync::Arc;
+
+use futures::prelude::stream::StreamExt;
+use futures_async_stream::try_stream;
+use futures_util::pin_mut;
+use itertools::Itertools;
+use prometheus::Histogram;
+use risingwave_common::array::DataChunk;
+use risingwave_common::buffer::Bitmap;
+use risingwave_common::catalog::{ColumnId, Field, Schema};
+use risingwave_common::row::{OwnedRow, Row};
+use risingwave_common::types::ScalarImpl;
+use risingwave_pb::batch_plan::plan_node::NodeBody;
+use risingwave_pb::common::BatchQueryEpoch;
+use risingwave_pb::plan_common::StorageTableDesc;
+use risingwave_storage::table::batch_table::storage_table::StorageTable;
+use risingwave_storage::table::{collect_data_chunk, KeyedRow, TableDistribution};
+use risingwave_storage::{dispatch_state_store, StateStore};
+
+use super::{BoxedDataChunkStream, BoxedExecutor, BoxedExecutorBuilder, Executor, ExecutorBuilder};
+use crate::error::{BatchError, Result};
+use crate::monitor::BatchMetricsWithTaskLabels;
+use crate::task::BatchTaskContext;
+
+pub struct LogRowSeqScanExecutor<S: StateStore> {
+    chunk_size: usize,
+    identity: String,
+    // It is table schema + op column
+    schema: Schema,
+
+    /// Batch metrics.
+    /// None: Local mode don't record mertics.
+    metrics: Option<BatchMetricsWithTaskLabels>,
+
+    table: StorageTable<S>,
+    old_epoch: BatchQueryEpoch,
+    new_epoch: BatchQueryEpoch,
+}
+
+impl<S: StateStore> LogRowSeqScanExecutor<S> {
+    pub fn new(
+        table: StorageTable<S>,
+        old_epoch: BatchQueryEpoch,
+        new_epoch: BatchQueryEpoch,
+        chunk_size: usize,
+        identity: String,
+        metrics: Option<BatchMetricsWithTaskLabels>,
+    ) -> Self {
+        let mut schema = table.schema().clone();
+        schema.fields.push(Field::with_name(
+            risingwave_common::types::DataType::Int16,
+            "op",
+        ));
+        Self {
+            chunk_size,
+            identity,
+            schema,
+            metrics,
+            table,
+            old_epoch,
+            new_epoch,
+        }
+    }
+}
+
+pub struct LogStoreRowSeqScanExecutorBuilder {}
+
+#[async_trait::async_trait]
+impl BoxedExecutorBuilder for LogStoreRowSeqScanExecutorBuilder {
+    async fn new_boxed_executor<C: BatchTaskContext>(
+        source: &ExecutorBuilder<'_, C>,
+        inputs: Vec<BoxedExecutor>,
+    ) -> Result<BoxedExecutor> {
+        ensure!(
+            inputs.is_empty(),
+            "LogStore row sequential scan should not have input executor!"
+        );
+        let log_store_seq_scan_node = try_match_expand!(
+            source.plan_node().get_node_body().unwrap(),
+            NodeBody::LogRowSeqScan
+        )?;
+
+        let table_desc: &StorageTableDesc = log_store_seq_scan_node.get_table_desc()?;
+        let column_ids = log_store_seq_scan_node
+            .column_ids
+            .iter()
+            .copied()
+            .map(ColumnId::from)
+            .collect();
+
+        let vnodes = match &log_store_seq_scan_node.vnode_bitmap {
+            Some(vnodes) => Some(Bitmap::from(vnodes).into()),
+            // This is possible for dml. vnode_bitmap is not filled by scheduler.
+            // Or it's single distribution, e.g., distinct agg. We scan in a single executor.
+            None => Some(TableDistribution::all_vnodes()),
+        };
+
+        let chunk_size = source.context.get_config().developer.chunk_size as u32;
+        let metrics = source.context().batch_metrics();
+
+        dispatch_state_store!(source.context().state_store(), state_store, {
+            let table = StorageTable::new_partial(state_store, column_ids, vnodes, table_desc);
+            Ok(Box::new(LogRowSeqScanExecutor::new(
+                table,
+                log_store_seq_scan_node.old_epoch.clone().unwrap(),
+                log_store_seq_scan_node.new_epoch.clone().unwrap(),
+                chunk_size as usize,
+                source.plan_node().get_identity().clone(),
+                metrics,
+            )))
+        })
+    }
+}
+impl<S: StateStore> Executor for LogRowSeqScanExecutor<S> {
+    fn schema(&self) -> &Schema {
+        &self.schema
+    }
+
+    fn identity(&self) -> &str {
+        &self.identity
+    }
+
+    fn execute(self: Box<Self>) -> BoxedDataChunkStream {
+        self.do_execute().boxed()
+    }
+}
+
+impl<S: StateStore> LogRowSeqScanExecutor<S> {
+    #[try_stream(ok = DataChunk, error = BatchError)]
+    async fn do_execute(self: Box<Self>) {
+        let Self {
+            chunk_size,
+            identity,
+            metrics,
+            table,
+            old_epoch,
+            new_epoch,
+            schema,
+        } = *self;
+        let table = std::sync::Arc::new(table);
+
+        // Create collector.
+        let histogram = metrics.as_ref().map(|metrics| {
+            metrics
+                .executor_metrics()
+                .row_seq_scan_next_duration
+                .with_guarded_label_values(&metrics.executor_labels(&identity))
+        });
+        // Range Scan
+        // WARN: DO NOT use `select` to execute range scans concurrently
+        //       it can consume too much memory if there're too many ranges.
+        let stream = Self::execute_range(
+            table.clone(),
+            old_epoch.clone(),
+            new_epoch.clone(),
+            chunk_size,
+            histogram.clone(),
+            Arc::new(schema.clone()),
+        );
+        #[for_await]
+        for chunk in stream {
+            let chunk = chunk?;
+            yield chunk;
+        }
+    }
+
+    #[try_stream(ok = DataChunk, error = BatchError)]
+    async fn execute_range(
+        table: Arc<StorageTable<S>>,
+        old_epoch: BatchQueryEpoch,
+        new_epoch: BatchQueryEpoch,
+        chunk_size: usize,
+        histogram: Option<impl Deref<Target = Histogram>>,
+        schema: Arc<Schema>,
+    ) {
+        let pk_prefix = OwnedRow::default();
+
+        let order_type = table.pk_serializer().get_order_types()[pk_prefix.len()];
+        // Range Scan.
+        let iter = table
+            .batch_iter_log_with_pk_bounds(
+                old_epoch.into(),
+                new_epoch.into(),
+                &pk_prefix,
+                (
+                    if order_type.nulls_are_first() {
+                        // `NULL`s are at the start bound side, we should exclude them to meet SQL semantics.
+                        Bound::Excluded(OwnedRow::new(vec![None]))
+                    } else {
+                        // Both start and end are unbounded, so we need to select all rows.
+                        Bound::Unbounded
+                    },
+                    if order_type.nulls_are_last() {
+                        // `NULL`s are at the end bound side, we should exclude them to meet SQL semantics.
+                        Bound::Excluded(OwnedRow::new(vec![None]))
+                    } else {
+                        // Both start and end are unbounded, so we need to select all rows.
+                        Bound::Unbounded
+                    },
+                ),
+            )
+            .await?
+            .map(|r| match r {
+                Ok((op, value)) => {
+                    let (k, row) = value.into_owned_row_key();
+                    // Todo! To avoid create a full row.
+                    let full_row = row
+                        .into_iter()
+                        .chain(vec![Some(ScalarImpl::Int16(op.to_i16()))])
+                        .collect_vec();
+                    let row = OwnedRow::new(full_row);
+                    Ok(KeyedRow::<_>::new(k, row))
+                }
+                Err(e) => Err(e),
+            });
+
+        pin_mut!(iter);
+        loop {
+            let timer = histogram.as_ref().map(|histogram| histogram.start_timer());
+
+            let chunk = collect_data_chunk(&mut iter, &schema, Some(chunk_size))
+                .await
+                .map_err(BatchError::from)?;
+            if let Some(timer) = timer {
+                timer.observe_duration()
+            }
+
+            if let Some(chunk) = chunk {
+                yield chunk
+            } else {
+                break;
+            }
+        }
+    }
+}
diff --git a/src/batch/src/executor/merge_sort_exchange.rs b/src/batch/src/executor/merge_sort_exchange.rs
index 1562f2a8f4557..e2779967dbcbe 100644
--- a/src/batch/src/executor/merge_sort_exchange.rs
+++ b/src/batch/src/executor/merge_sort_exchange.rs
@@ -47,11 +47,13 @@ pub struct MergeSortExchangeExecutorImpl<CS, C> {
     /// Mock-able `CreateSource`.
     source_creators: Vec<CS>,
     schema: Schema,
+    #[expect(dead_code)]
     task_id: TaskId,
     identity: String,
     /// The maximum size of the chunk produced by executor at a time.
     chunk_size: usize,
     mem_ctx: MemoryContext,
+    #[expect(dead_code)]
     alloc: MonitoredGlobalAlloc,
 }
 
@@ -273,10 +275,8 @@ impl BoxedExecutorBuilder for MergeSortExchangeExecutorBuilder {
 
 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
-
     use futures::StreamExt;
-    use risingwave_common::array::{Array, DataChunk};
+    use risingwave_common::array::Array;
     use risingwave_common::test_prelude::DataChunkTestExt;
     use risingwave_common::types::DataType;
     use risingwave_common::util::sort_util::OrderType;
diff --git a/src/batch/src/executor/mod.rs b/src/batch/src/executor/mod.rs
index ded2af7089826..b77027327fe05 100644
--- a/src/batch/src/executor/mod.rs
+++ b/src/batch/src/executor/mod.rs
@@ -24,6 +24,7 @@ mod iceberg_scan;
 mod insert;
 mod join;
 mod limit;
+mod log_row_seq_scan;
 mod managed;
 mod max_one_row;
 mod merge_sort_exchange;
@@ -80,6 +81,7 @@ pub use update::*;
 pub use utils::*;
 pub use values::*;
 
+use self::log_row_seq_scan::LogStoreRowSeqScanExecutorBuilder;
 use self::test_utils::{BlockExecutorBuilder, BusyLoopExecutorBuilder};
 use crate::error::Result;
 use crate::executor::sys_row_seq_scan::SysRowSeqScanExecutorBuilder;
@@ -239,6 +241,7 @@ impl<'a, C: BatchTaskContext> ExecutorBuilder<'a, C> {
             // Follow NodeBody only used for test
             NodeBody::BlockExecutor => BlockExecutorBuilder,
             NodeBody::BusyLoopExecutor => BusyLoopExecutorBuilder,
+            NodeBody::LogRowSeqScan => LogStoreRowSeqScanExecutorBuilder,
         }
         .await?;
 
@@ -257,8 +260,8 @@ mod tests {
     use crate::executor::ExecutorBuilder;
     use crate::task::{ComputeNodeContext, ShutdownToken, TaskId};
 
-    #[test]
-    fn test_clone_for_plan() {
+    #[tokio::test]
+    async fn test_clone_for_plan() {
         let plan_node = PlanNode::default();
         let task_id = &TaskId {
             task_id: 1,
diff --git a/src/batch/src/executor/order_by.rs b/src/batch/src/executor/order_by.rs
index b8be2211c5157..05cd0f8c94fa0 100644
--- a/src/batch/src/executor/order_by.rs
+++ b/src/batch/src/executor/order_by.rs
@@ -19,6 +19,7 @@ use risingwave_common::memory::MemoryContext;
 use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
 use risingwave_common::util::memcmp_encoding::encode_chunk;
 use risingwave_common::util::sort_util::ColumnOrder;
+use risingwave_common_estimate_size::EstimateSize;
 use risingwave_pb::batch_plan::plan_node::NodeBody;
 
 use super::{BoxedDataChunkStream, BoxedExecutor, BoxedExecutorBuilder, Executor, ExecutorBuilder};
@@ -91,7 +92,12 @@ impl SortExecutor {
 
         #[for_await]
         for chunk in self.child.execute() {
-            chunks.push(chunk?.compact());
+            let chunk = chunk?.compact();
+            let chunk_estimated_heap_size = chunk.estimated_heap_size();
+            chunks.push(chunk);
+            if !self.mem_context.add(chunk_estimated_heap_size as i64) {
+                Err(BatchError::OutOfMemory(self.mem_context.mem_limit()))?;
+            }
         }
 
         let mut encoded_rows =
@@ -99,12 +105,19 @@ impl SortExecutor {
 
         for chunk in &chunks {
             let encoded_chunk = encode_chunk(chunk, &self.column_orders)?;
+            let chunk_estimated_heap_size = encoded_chunk
+                .iter()
+                .map(|x| x.estimated_heap_size())
+                .sum::<usize>();
             encoded_rows.extend(
                 encoded_chunk
                     .into_iter()
                     .enumerate()
                     .map(|(row_id, row)| (chunk.row_at_unchecked_vis(row_id), row)),
             );
+            if !self.mem_context.add(chunk_estimated_heap_size as i64) {
+                Err(BatchError::OutOfMemory(self.mem_context.mem_limit()))?;
+            }
         }
 
         encoded_rows.sort_unstable_by(|(_, a), (_, b)| a.cmp(b));
@@ -145,8 +158,7 @@ impl SortExecutor {
 mod tests {
     use futures::StreamExt;
     use risingwave_common::array::*;
-    use risingwave_common::catalog::{Field, Schema};
-    use risingwave_common::test_prelude::DataChunkTestExt;
+    use risingwave_common::catalog::Field;
     use risingwave_common::types::{
         DataType, Date, Interval, Scalar, StructType, Time, Timestamp, F32,
     };
diff --git a/src/batch/src/executor/project.rs b/src/batch/src/executor/project.rs
index fc5cddaec0787..7fbc5540b975b 100644
--- a/src/batch/src/executor/project.rs
+++ b/src/batch/src/executor/project.rs
@@ -108,16 +108,14 @@ impl BoxedExecutorBuilder for ProjectExecutor {
 
 #[cfg(test)]
 mod tests {
-    use futures::stream::StreamExt;
     use risingwave_common::array::{Array, I32Array};
-    use risingwave_common::catalog::{Field, Schema};
     use risingwave_common::test_prelude::*;
     use risingwave_common::types::DataType;
     use risingwave_expr::expr::{InputRefExpression, LiteralExpression};
 
     use super::*;
     use crate::executor::test_utils::MockExecutor;
-    use crate::executor::{Executor, ValuesExecutor};
+    use crate::executor::ValuesExecutor;
     use crate::*;
 
     const CHUNK_SIZE: usize = 1024;
diff --git a/src/batch/src/executor/project_set.rs b/src/batch/src/executor/project_set.rs
index b7291609be586..17eaadab76da2 100644
--- a/src/batch/src/executor/project_set.rs
+++ b/src/batch/src/executor/project_set.rs
@@ -15,13 +15,16 @@
 use either::Either;
 use futures_async_stream::try_stream;
 use itertools::Itertools;
-use risingwave_common::array::DataChunk;
+use risingwave_common::array::{ArrayRef, DataChunk};
 use risingwave_common::catalog::{Field, Schema};
 use risingwave_common::types::{DataType, DatumRef};
 use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
 use risingwave_common::util::iter_util::ZipEqFast;
-use risingwave_expr::table_function::ProjectSetSelectItem;
+use risingwave_expr::expr::{self, BoxedExpression};
+use risingwave_expr::table_function::{self, BoxedTableFunction, TableFunctionOutputIter};
 use risingwave_pb::batch_plan::plan_node::NodeBody;
+use risingwave_pb::expr::project_set_select_item::PbSelectItem;
+use risingwave_pb::expr::PbProjectSetSelectItem;
 
 use crate::error::{BatchError, Result};
 use crate::executor::{
@@ -97,7 +100,7 @@ impl ProjectSetExecutor {
                                     && i == row_idx
                                 {
                                     valid = true;
-                                    value
+                                    value?
                                 } else {
                                     None
                                 }
@@ -170,19 +173,68 @@ impl BoxedExecutorBuilder for ProjectSetExecutor {
     }
 }
 
+/// Either a scalar expression or a set-returning function.
+///
+/// See also [`PbProjectSetSelectItem`]
+#[derive(Debug)]
+pub enum ProjectSetSelectItem {
+    Scalar(BoxedExpression),
+    Set(BoxedTableFunction),
+}
+
+impl From<BoxedTableFunction> for ProjectSetSelectItem {
+    fn from(table_function: BoxedTableFunction) -> Self {
+        ProjectSetSelectItem::Set(table_function)
+    }
+}
+
+impl From<BoxedExpression> for ProjectSetSelectItem {
+    fn from(expr: BoxedExpression) -> Self {
+        ProjectSetSelectItem::Scalar(expr)
+    }
+}
+
+impl ProjectSetSelectItem {
+    pub fn from_prost(prost: &PbProjectSetSelectItem, chunk_size: usize) -> Result<Self> {
+        Ok(match prost.select_item.as_ref().unwrap() {
+            PbSelectItem::Expr(expr) => Self::Scalar(expr::build_from_prost(expr)?),
+            PbSelectItem::TableFunction(tf) => {
+                Self::Set(table_function::build_from_prost(tf, chunk_size)?)
+            }
+        })
+    }
+
+    pub fn return_type(&self) -> DataType {
+        match self {
+            ProjectSetSelectItem::Scalar(expr) => expr.return_type(),
+            ProjectSetSelectItem::Set(tf) => tf.return_type(),
+        }
+    }
+
+    pub async fn eval<'a>(
+        &'a self,
+        input: &'a DataChunk,
+    ) -> Result<Either<TableFunctionOutputIter<'a>, ArrayRef>> {
+        match self {
+            Self::Set(tf) => Ok(Either::Left(
+                TableFunctionOutputIter::new(tf.eval(input).await).await?,
+            )),
+            Self::Scalar(expr) => Ok(Either::Right(expr.eval(input).await?)),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use futures::stream::StreamExt;
     use futures_async_stream::for_await;
-    use risingwave_common::catalog::{Field, Schema};
     use risingwave_common::test_prelude::*;
-    use risingwave_common::types::DataType;
     use risingwave_expr::expr::{ExpressionBoxExt, InputRefExpression, LiteralExpression};
     use risingwave_expr::table_function::repeat;
 
     use super::*;
     use crate::executor::test_utils::MockExecutor;
-    use crate::executor::{Executor, ValuesExecutor};
+    use crate::executor::ValuesExecutor;
     use crate::*;
 
     const CHUNK_SIZE: usize = 1024;
diff --git a/src/batch/src/executor/row_seq_scan.rs b/src/batch/src/executor/row_seq_scan.rs
index 4c1261363a72b..988122218479f 100644
--- a/src/batch/src/executor/row_seq_scan.rs
+++ b/src/batch/src/executor/row_seq_scan.rs
@@ -11,7 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-use std::ops::{Bound, Deref, RangeBounds};
+use std::ops::{Bound, Deref};
 use std::sync::Arc;
 
 use futures::{pin_mut, StreamExt};
@@ -67,11 +67,6 @@ pub struct ScanRange {
 }
 
 impl ScanRange {
-    fn is_full_range<T>(bounds: &impl RangeBounds<T>) -> bool {
-        matches!(bounds.start_bound(), Bound::Unbounded)
-            && matches!(bounds.end_bound(), Bound::Unbounded)
-    }
-
     /// Create a scan range from the prost representation.
     pub fn new(
         scan_range: PbScanRange,
@@ -177,7 +172,6 @@ impl BoxedExecutorBuilder for RowSeqScanExecutorBuilder {
             .copied()
             .map(ColumnId::from)
             .collect();
-
         let vnodes = match &seq_scan_node.vnode_bitmap {
             Some(vnodes) => Some(Bitmap::from(vnodes).into()),
             // This is possible for dml. vnode_bitmap is not filled by scheduler.
diff --git a/src/batch/src/executor/sort_agg.rs b/src/batch/src/executor/sort_agg.rs
index 2f7cea6059367..1cb5968045fa6 100644
--- a/src/batch/src/executor/sort_agg.rs
+++ b/src/batch/src/executor/sort_agg.rs
@@ -109,7 +109,11 @@ impl SortAggExecutor {
     #[try_stream(boxed, ok = DataChunk, error = BatchError)]
     async fn do_execute(mut self: Box<Self>) {
         let mut left_capacity = self.output_size_limit;
-        let mut agg_states = self.aggs.iter().map(|agg| agg.create_state()).collect_vec();
+        let mut agg_states: Vec<_> = self
+            .aggs
+            .iter()
+            .map(|agg| agg.create_state())
+            .try_collect()?;
         let (mut group_builders, mut agg_builders) =
             Self::create_builders(&self.group_key, &self.aggs);
         let mut curr_group = if self.group_key.is_empty() {
@@ -225,7 +229,7 @@ impl SortAggExecutor {
         {
             let result = agg.get_result(state).await?;
             builder.append(result);
-            *state = agg.create_state();
+            *state = agg.create_state()?;
         }
         Ok(())
     }
@@ -360,14 +364,12 @@ mod tests {
     use futures::StreamExt;
     use futures_async_stream::for_await;
     use risingwave_common::array::{Array as _, I64Array};
-    use risingwave_common::catalog::{Field, Schema};
     use risingwave_common::test_prelude::DataChunkTestExt;
     use risingwave_common::types::DataType;
     use risingwave_expr::expr::build_from_pretty;
 
     use super::*;
     use crate::executor::test_utils::MockExecutor;
-    use crate::task::ShutdownToken;
 
     #[tokio::test]
     async fn execute_count_star_int32() -> Result<()> {
diff --git a/src/batch/src/executor/source.rs b/src/batch/src/executor/source.rs
index cbc69444af7f9..c71f94ae36b09 100644
--- a/src/batch/src/executor/source.rs
+++ b/src/batch/src/executor/source.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
 use std::sync::Arc;
 
 use futures::StreamExt;
@@ -49,7 +48,7 @@ pub struct SourceExecutor {
     schema: Schema,
     identity: String,
 
-    source_ctrl_opts: SourceCtrlOpts,
+    chunk_size: usize,
 }
 
 #[async_trait::async_trait]
@@ -65,8 +64,7 @@ impl BoxedExecutorBuilder for SourceExecutor {
         )?;
 
         // prepare connector source
-        let source_props: HashMap<String, String> =
-            HashMap::from_iter(source_node.with_properties.clone());
+        let source_props = source_node.with_properties.clone();
         let config =
             ConnectorProperties::extract(source_props, false).map_err(BatchError::connector)?;
 
@@ -79,11 +77,6 @@ impl BoxedExecutorBuilder for SourceExecutor {
             .map(|c| SourceColumnDesc::from(&ColumnDesc::from(c.column_desc.as_ref().unwrap())))
             .collect();
 
-        let source_ctrl_opts = SourceCtrlOpts {
-            chunk_size: source.context().get_config().developer.chunk_size,
-            rate_limit: None,
-        };
-
         let column_ids: Vec<_> = source_node
             .columns
             .iter()
@@ -144,7 +137,7 @@ impl BoxedExecutorBuilder for SourceExecutor {
                 split_list,
                 schema,
                 identity: source.plan_node().get_identity().clone(),
-                source_ctrl_opts,
+                chunk_size: source.context().get_config().developer.chunk_size,
             }))
         }
     }
@@ -173,12 +166,15 @@ impl SourceExecutor {
             u32::MAX,
             "NA".to_owned(), // source name was not passed in batch plan
             self.metrics,
-            self.source_ctrl_opts.clone(),
+            SourceCtrlOpts {
+                chunk_size: self.chunk_size,
+                rate_limit: None,
+            },
             ConnectorProperties::default(),
         ));
         let stream = self
             .source
-            .to_stream(Some(self.split_list), self.column_ids, source_ctx)
+            .build_stream(Some(self.split_list), self.column_ids, source_ctx)
             .await?;
 
         #[for_await]
diff --git a/src/batch/src/executor/table_function.rs b/src/batch/src/executor/table_function.rs
index 6c0fc7f302eed..8ced12ebbf2ca 100644
--- a/src/batch/src/executor/table_function.rs
+++ b/src/batch/src/executor/table_function.rs
@@ -16,7 +16,7 @@ use futures_async_stream::try_stream;
 use risingwave_common::array::{ArrayImpl, DataChunk};
 use risingwave_common::catalog::{Field, Schema};
 use risingwave_common::types::DataType;
-use risingwave_expr::table_function::{build_from_prost, BoxedTableFunction};
+use risingwave_expr::table_function::{build_from_prost, check_error, BoxedTableFunction};
 use risingwave_pb::batch_plan::plan_node::NodeBody;
 
 use super::{BoxedExecutor, BoxedExecutorBuilder};
@@ -28,6 +28,7 @@ pub struct TableFunctionExecutor {
     schema: Schema,
     identity: String,
     table_function: BoxedTableFunction,
+    #[expect(dead_code)]
     chunk_size: usize,
 }
 
@@ -53,6 +54,7 @@ impl TableFunctionExecutor {
         #[for_await]
         for chunk in self.table_function.eval(&dummy_chunk).await {
             let chunk = chunk?;
+            check_error(&chunk)?;
             // remove the first column and expand the second column if its data type is struct
             yield match chunk.column_at(1).as_ref() {
                 ArrayImpl::Struct(struct_array) => struct_array.into(),
diff --git a/src/batch/src/executor/test_utils.rs b/src/batch/src/executor/test_utils.rs
index 41444d1498875..1d949d6e0e06a 100644
--- a/src/batch/src/executor/test_utils.rs
+++ b/src/batch/src/executor/test_utils.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![cfg_attr(not(test), allow(dead_code))]
+
 use std::collections::VecDeque;
 
 use assert_matches::assert_matches;
@@ -73,7 +75,7 @@ pub fn gen_sorted_data(
         let mut array_builder = DataType::Int64.create_array_builder(batch_size);
 
         for _ in 0..batch_size {
-            array_builder.append(&data_gen.generate_datum(0));
+            array_builder.append(data_gen.generate_datum(0));
         }
 
         let array = array_builder.finish();
@@ -100,7 +102,7 @@ pub fn gen_projected_data(
         let mut array_builder = DataType::Int64.create_array_builder(batch_size);
 
         for j in 0..batch_size {
-            array_builder.append(&data_gen.generate_datum(((i + 1) * (j + 1)) as u64));
+            array_builder.append(data_gen.generate_datum(((i + 1) * (j + 1)) as u64));
         }
 
         let chunk = DataChunk::new(vec![array_builder.finish().into()], batch_size);
diff --git a/src/batch/src/executor/top_n.rs b/src/batch/src/executor/top_n.rs
index c02b84ec5b7ca..a4450a76c17a8 100644
--- a/src/batch/src/executor/top_n.rs
+++ b/src/batch/src/executor/top_n.rs
@@ -14,7 +14,6 @@
 
 use std::cmp::Ordering;
 use std::sync::Arc;
-use std::vec::Vec;
 
 use futures_async_stream::try_stream;
 use risingwave_common::array::DataChunk;
@@ -244,10 +243,6 @@ impl HeapElem {
         }
     }
 
-    pub fn encoded_row(&self) -> &[u8] {
-        &self.encoded_row
-    }
-
     pub fn row(&self) -> impl Row + '_ {
         &self.row
     }
@@ -296,8 +291,8 @@ impl TopNExecutor {
 mod tests {
     use futures::stream::StreamExt;
     use itertools::Itertools;
-    use risingwave_common::array::{Array, DataChunk};
-    use risingwave_common::catalog::{Field, Schema};
+    use risingwave_common::array::Array;
+    use risingwave_common::catalog::Field;
     use risingwave_common::test_prelude::DataChunkTestExt;
     use risingwave_common::types::DataType;
     use risingwave_common::util::sort_util::OrderType;
diff --git a/src/batch/src/executor/update.rs b/src/batch/src/executor/update.rs
index 9e9c2fa8f543d..a753aef840f52 100644
--- a/src/batch/src/executor/update.rs
+++ b/src/batch/src/executor/update.rs
@@ -259,7 +259,6 @@ mod tests {
     use std::sync::Arc;
 
     use futures::StreamExt;
-    use risingwave_common::array::Array;
     use risingwave_common::catalog::{
         schema_test_utils, ColumnDesc, ColumnId, INITIAL_TABLE_VERSION_ID,
     };
diff --git a/src/batch/src/executor/utils.rs b/src/batch/src/executor/utils.rs
index 9c6f162f02268..4f724ec5416c8 100644
--- a/src/batch/src/executor/utils.rs
+++ b/src/batch/src/executor/utils.rs
@@ -99,3 +99,28 @@ impl DummyExecutor {
     #[try_stream(boxed, ok = DataChunk, error = BatchError)]
     async fn do_nothing() {}
 }
+
+pub struct WrapStreamExecutor {
+    schema: Schema,
+    stream: BoxedDataChunkStream,
+}
+
+impl WrapStreamExecutor {
+    pub fn new(schema: Schema, stream: BoxedDataChunkStream) -> Self {
+        Self { schema, stream }
+    }
+}
+
+impl Executor for WrapStreamExecutor {
+    fn schema(&self) -> &Schema {
+        &self.schema
+    }
+
+    fn identity(&self) -> &str {
+        "WrapStreamExecutor"
+    }
+
+    fn execute(self: Box<Self>) -> BoxedDataChunkStream {
+        self.stream
+    }
+}
diff --git a/src/batch/src/executor/values.rs b/src/batch/src/executor/values.rs
index d401cd41dd195..7eb1a6e9893b5 100644
--- a/src/batch/src/executor/values.rs
+++ b/src/batch/src/executor/values.rs
@@ -37,6 +37,7 @@ pub struct ValuesExecutor {
 }
 
 impl ValuesExecutor {
+    #[cfg(test)]
     pub(crate) fn new(
         rows: Vec<Vec<BoxedExpression>>,
         schema: Schema,
diff --git a/src/batch/src/lib.rs b/src/batch/src/lib.rs
index d937c64826550..3a29e2a90b27e 100644
--- a/src/batch/src/lib.rs
+++ b/src/batch/src/lib.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![expect(dead_code)]
 #![allow(clippy::derive_partial_eq_without_eq)]
 #![feature(trait_alias)]
 #![feature(exact_size_is_empty)]
@@ -25,13 +24,11 @@
 #![feature(is_sorted)]
 #![recursion_limit = "256"]
 #![feature(let_chains)]
-#![feature(bound_map)]
 #![feature(int_roundings)]
 #![feature(allocator_api)]
 #![feature(impl_trait_in_assoc_type)]
 #![feature(assert_matches)]
 #![feature(lazy_cell)]
-#![feature(array_methods)]
 #![feature(error_generic_member_access)]
 #![feature(map_try_insert)]
 
@@ -41,6 +38,7 @@ pub mod execution;
 pub mod executor;
 pub mod monitor;
 pub mod rpc;
+pub mod spill;
 pub mod task;
 pub mod worker_manager;
 
diff --git a/src/batch/src/rpc/service/task_service.rs b/src/batch/src/rpc/service/task_service.rs
index 816b62a02fc4d..11ae64e7b3918 100644
--- a/src/batch/src/rpc/service/task_service.rs
+++ b/src/batch/src/rpc/service/task_service.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::convert::Into;
 use std::sync::Arc;
 
 use risingwave_common::util::tracing::TracingContext;
diff --git a/src/storage/src/hummock/file_cache/mod.rs b/src/batch/src/spill/mod.rs
similarity index 88%
rename from src/storage/src/hummock/file_cache/mod.rs
rename to src/batch/src/spill/mod.rs
index 2f2ac826410b0..6af1eae7429c6 100644
--- a/src/storage/src/hummock/file_cache/mod.rs
+++ b/src/batch/src/spill/mod.rs
@@ -12,8 +12,4 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod recent_filter;
-mod store;
-
-pub use recent_filter::*;
-pub use store::*;
+pub mod spill_op;
diff --git a/src/batch/src/spill/spill_op.rs b/src/batch/src/spill/spill_op.rs
new file mode 100644
index 0000000000000..d04c5ac26019a
--- /dev/null
+++ b/src/batch/src/spill/spill_op.rs
@@ -0,0 +1,168 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::hash::BuildHasher;
+use std::ops::{Deref, DerefMut};
+use std::sync::LazyLock;
+
+use anyhow::anyhow;
+use futures_async_stream::try_stream;
+use futures_util::AsyncReadExt;
+use opendal::layers::RetryLayer;
+use opendal::services::Fs;
+use opendal::Operator;
+use prost::Message;
+use risingwave_common::array::DataChunk;
+use risingwave_pb::data::DataChunk as PbDataChunk;
+use thiserror_ext::AsReport;
+use tokio::sync::Mutex;
+use twox_hash::XxHash64;
+
+use crate::error::{BatchError, Result};
+
+const RW_BATCH_SPILL_DIR_ENV: &str = "RW_BATCH_SPILL_DIR";
+pub const DEFAULT_SPILL_PARTITION_NUM: usize = 20;
+const DEFAULT_SPILL_DIR: &str = "/tmp/";
+const RW_MANAGED_SPILL_DIR: &str = "/rw_batch_spill/";
+const DEFAULT_IO_BUFFER_SIZE: usize = 256 * 1024;
+const DEFAULT_IO_CONCURRENT_TASK: usize = 8;
+
+/// `SpillOp` is used to manage the spill directory of the spilling executor and it will drop the directory with a RAII style.
+pub struct SpillOp {
+    pub op: Operator,
+}
+
+impl SpillOp {
+    pub fn create(path: String) -> Result<SpillOp> {
+        assert!(path.ends_with('/'));
+
+        let spill_dir =
+            std::env::var(RW_BATCH_SPILL_DIR_ENV).unwrap_or_else(|_| DEFAULT_SPILL_DIR.to_string());
+        let root = format!("/{}/{}/{}/", spill_dir, RW_MANAGED_SPILL_DIR, path);
+
+        let mut builder = Fs::default();
+        builder.root(&root);
+
+        let op: Operator = Operator::new(builder)?
+            .layer(RetryLayer::default())
+            .finish();
+        Ok(SpillOp { op })
+    }
+
+    pub async fn clean_spill_directory() -> opendal::Result<()> {
+        static LOCK: LazyLock<Mutex<usize>> = LazyLock::new(|| Mutex::new(0));
+        let _guard = LOCK.lock().await;
+
+        let spill_dir =
+            std::env::var(RW_BATCH_SPILL_DIR_ENV).unwrap_or_else(|_| DEFAULT_SPILL_DIR.to_string());
+        let root = format!("/{}/{}/", spill_dir, RW_MANAGED_SPILL_DIR);
+
+        let mut builder = Fs::default();
+        builder.root(&root);
+
+        let op: Operator = Operator::new(builder)?
+            .layer(RetryLayer::default())
+            .finish();
+
+        op.remove_all("/").await
+    }
+
+    pub async fn writer_with(&self, name: &str) -> Result<opendal::Writer> {
+        Ok(self
+            .op
+            .writer_with(name)
+            .buffer(DEFAULT_IO_BUFFER_SIZE)
+            .concurrent(DEFAULT_IO_CONCURRENT_TASK)
+            .await?)
+    }
+
+    pub async fn reader_with(&self, name: &str) -> Result<opendal::Reader> {
+        Ok(self
+            .op
+            .reader_with(name)
+            .buffer(DEFAULT_IO_BUFFER_SIZE)
+            .await?)
+    }
+
+    /// spill file content will look like the below.
+    ///
+    /// ```text
+    /// [proto_len]
+    /// [proto_bytes]
+    /// ...
+    /// [proto_len]
+    /// [proto_bytes]
+    /// ```
+    #[try_stream(boxed, ok = DataChunk, error = BatchError)]
+    pub async fn read_stream(mut reader: opendal::Reader) {
+        let mut buf = [0u8; 4];
+        loop {
+            if let Err(err) = reader.read_exact(&mut buf).await {
+                if err.kind() == std::io::ErrorKind::UnexpectedEof {
+                    break;
+                } else {
+                    return Err(anyhow!(err).into());
+                }
+            }
+            let len = u32::from_le_bytes(buf) as usize;
+            let mut buf = vec![0u8; len];
+            reader.read_exact(&mut buf).await.map_err(|e| anyhow!(e))?;
+            let chunk_pb: PbDataChunk = Message::decode(buf.as_slice()).map_err(|e| anyhow!(e))?;
+            let chunk = DataChunk::from_protobuf(&chunk_pb)?;
+            yield chunk;
+        }
+    }
+}
+
+impl Drop for SpillOp {
+    fn drop(&mut self) {
+        let op = self.op.clone();
+        tokio::task::spawn(async move {
+            let result = op.remove_all("/").await;
+            if let Err(error) = result {
+                error!(
+                    error = %error.as_report(),
+                    "Failed to remove spill directory"
+                );
+            }
+        });
+    }
+}
+
+impl DerefMut for SpillOp {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.op
+    }
+}
+
+impl Deref for SpillOp {
+    type Target = Operator;
+
+    fn deref(&self) -> &Self::Target {
+        &self.op
+    }
+}
+
+#[derive(Default, Clone, Copy)]
+pub struct SpillBuildHasher(pub u64);
+
+impl BuildHasher for SpillBuildHasher {
+    type Hasher = XxHash64;
+
+    fn build_hasher(&self) -> Self::Hasher {
+        XxHash64::with_seed(self.0)
+    }
+}
+
+pub const SPILL_AT_LEAST_MEMORY: u64 = 1024 * 1024;
diff --git a/src/batch/src/task/consistent_hash_shuffle_channel.rs b/src/batch/src/task/consistent_hash_shuffle_channel.rs
index 3e42186fcf587..1181654f41249 100644
--- a/src/batch/src/task/consistent_hash_shuffle_channel.rs
+++ b/src/batch/src/task/consistent_hash_shuffle_channel.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::fmt::{Debug, Formatter};
-use std::option::Option;
 use std::sync::Arc;
 
 use anyhow::anyhow;
diff --git a/src/batch/src/task/context.rs b/src/batch/src/task/context.rs
index 886eeb7d9753d..f633439675eb1 100644
--- a/src/batch/src/task/context.rs
+++ b/src/batch/src/task/context.rs
@@ -11,7 +11,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 
 use prometheus::core::Atomic;
@@ -63,10 +62,6 @@ pub trait BatchTaskContext: Clone + Send + Sync + 'static {
 
     fn source_metrics(&self) -> Arc<SourceMetrics>;
 
-    fn store_mem_usage(&self, val: usize);
-
-    fn mem_usage(&self) -> usize;
-
     fn create_executor_mem_context(&self, executor_id: &str) -> MemoryContext;
 
     fn worker_node_manager(&self) -> Option<WorkerNodeManagerRef>;
@@ -80,12 +75,6 @@ pub struct ComputeNodeContext {
     batch_metrics: Option<BatchMetricsWithTaskLabels>,
 
     mem_context: MemoryContext,
-
-    // Last mem usage value. Init to be 0. Should be the last value of `cur_mem_val`.
-    last_mem_val: Arc<AtomicUsize>,
-    // How many memory bytes have been used in this task for the latest report value. Will be moved
-    // to `last_mem_val` if new value comes in.
-    cur_mem_val: Arc<AtomicUsize>,
 }
 
 impl BatchTaskContext for ComputeNodeContext {
@@ -127,22 +116,6 @@ impl BatchTaskContext for ComputeNodeContext {
         self.env.source_metrics()
     }
 
-    fn store_mem_usage(&self, val: usize) {
-        // Record the last mem val.
-        // Calculate the difference between old val and new value, and apply the diff to total
-        // memory usage value.
-        let old_value = self.cur_mem_val.load(Ordering::Relaxed);
-        self.last_mem_val.store(old_value, Ordering::Relaxed);
-        let diff = val as i64 - old_value as i64;
-        self.env.task_manager().apply_mem_diff(diff);
-
-        self.cur_mem_val.store(val, Ordering::Relaxed);
-    }
-
-    fn mem_usage(&self) -> usize {
-        self.cur_mem_val.load(Ordering::Relaxed)
-    }
-
     fn create_executor_mem_context(&self, executor_id: &str) -> MemoryContext {
         if let Some(metrics) = &self.batch_metrics {
             let executor_mem_usage = metrics
@@ -167,8 +140,6 @@ impl ComputeNodeContext {
         Self {
             env: BatchEnvironment::for_test(),
             batch_metrics: None,
-            cur_mem_val: Arc::new(0.into()),
-            last_mem_val: Arc::new(0.into()),
             mem_context: MemoryContext::none(),
         }
     }
@@ -189,8 +160,6 @@ impl ComputeNodeContext {
             Self {
                 env,
                 batch_metrics: Some(batch_metrics),
-                cur_mem_val: Arc::new(0.into()),
-                last_mem_val: Arc::new(0.into()),
                 mem_context,
             }
         } else {
@@ -198,8 +167,6 @@ impl ComputeNodeContext {
             Self {
                 env,
                 batch_metrics: None,
-                cur_mem_val: Arc::new(0.into()),
-                last_mem_val: Arc::new(0.into()),
                 mem_context: batch_mem_context,
             }
         }
@@ -210,13 +177,7 @@ impl ComputeNodeContext {
         Self {
             env,
             batch_metrics: None,
-            cur_mem_val: Arc::new(0.into()),
-            last_mem_val: Arc::new(0.into()),
             mem_context: batch_mem_context,
         }
     }
-
-    pub fn mem_usage(&self) -> usize {
-        self.cur_mem_val.load(Ordering::Relaxed)
-    }
 }
diff --git a/src/batch/src/task/env.rs b/src/batch/src/task/env.rs
index 58631cb9563a0..3e91551a6db4a 100644
--- a/src/batch/src/task/env.rs
+++ b/src/batch/src/task/env.rs
@@ -103,6 +103,7 @@ impl BatchEnvironment {
             task_manager: Arc::new(BatchManager::new(
                 BatchConfig::default(),
                 BatchManagerMetrics::for_test(),
+                u64::MAX,
             )),
             server_addr: "127.0.0.1:5688".parse().unwrap(),
             config: Arc::new(BatchConfig::default()),
diff --git a/src/batch/src/task/hash_shuffle_channel.rs b/src/batch/src/task/hash_shuffle_channel.rs
index 3965161030a32..626324e6f3456 100644
--- a/src/batch/src/task/hash_shuffle_channel.rs
+++ b/src/batch/src/task/hash_shuffle_channel.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::fmt::{Debug, Formatter};
-use std::option::Option;
 use std::sync::Arc;
 
 use anyhow::anyhow;
diff --git a/src/batch/src/task/task_execution.rs b/src/batch/src/task/task_execution.rs
index 7b5e00df18c03..4536dad1c031f 100644
--- a/src/batch/src/task/task_execution.rs
+++ b/src/batch/src/task/task_execution.rs
@@ -79,7 +79,7 @@ impl StateReporter {
 
 #[derive(PartialEq, Eq, Hash, Clone, Debug, Default)]
 pub struct TaskId {
-    pub task_id: u32,
+    pub task_id: u64,
     pub stage_id: u32,
     pub query_id: String,
 }
@@ -87,7 +87,7 @@ pub struct TaskId {
 #[derive(PartialEq, Eq, Hash, Clone, Default)]
 pub struct TaskOutputId {
     pub task_id: TaskId,
-    pub output_id: u32,
+    pub output_id: u64,
 }
 
 /// More compact formatter compared to derived `fmt::Debug`.
@@ -559,8 +559,8 @@ impl<C: BatchTaskContext> BatchTaskExecution<C> {
                         }
                     }
                 }
-                date_chunk = data_chunk_stream.next()=> {
-                    match date_chunk {
+                data_chunk = data_chunk_stream.next()=> {
+                    match data_chunk {
                         Some(Ok(data_chunk)) => {
                             if let Err(e) = sender.send(data_chunk).await {
                                 match e {
@@ -691,10 +691,6 @@ impl<C: BatchTaskContext> BatchTaskExecution<C> {
         }
     }
 
-    pub fn mem_usage(&self) -> usize {
-        self.context.mem_usage()
-    }
-
     /// Check the task status: whether has ended.
     pub fn is_end(&self) -> bool {
         let guard = self.state.lock();
diff --git a/src/batch/src/task/task_manager.rs b/src/batch/src/task/task_manager.rs
index e24fb7b09defd..f25ce88379905 100644
--- a/src/batch/src/task/task_manager.rs
+++ b/src/batch/src/task/task_manager.rs
@@ -17,7 +17,6 @@ use std::net::SocketAddr;
 use std::sync::Arc;
 
 use anyhow::Context;
-use hytra::TrAdder;
 use parking_lot::Mutex;
 use risingwave_common::config::BatchConfig;
 use risingwave_common::memory::MemoryContext;
@@ -50,11 +49,6 @@ pub struct BatchManager {
     /// Batch configuration
     config: BatchConfig,
 
-    /// Total batch memory usage in this CN.
-    /// When each task context report their own usage, it will apply the diff into this total mem
-    /// value for all tasks.
-    total_mem_val: Arc<TrAdder<i64>>,
-
     /// Memory context used for batch tasks in cn.
     mem_context: MemoryContext,
 
@@ -63,7 +57,7 @@ pub struct BatchManager {
 }
 
 impl BatchManager {
-    pub fn new(config: BatchConfig, metrics: Arc<BatchManagerMetrics>) -> Self {
+    pub fn new(config: BatchConfig, metrics: Arc<BatchManagerMetrics>, mem_limit: u64) -> Self {
         let runtime = {
             let mut builder = tokio::runtime::Builder::new_multi_thread();
             if let Some(worker_threads_num) = config.worker_threads_num {
@@ -76,12 +70,11 @@ impl BatchManager {
                 .unwrap()
         };
 
-        let mem_context = MemoryContext::root(metrics.batch_total_mem.clone());
+        let mem_context = MemoryContext::root(metrics.batch_total_mem.clone(), mem_limit);
         BatchManager {
             tasks: Arc::new(Mutex::new(HashMap::new())),
             runtime: Arc::new(runtime.into()),
             config,
-            total_mem_val: TrAdder::new().into(),
             metrics,
             mem_context,
         }
@@ -288,45 +281,6 @@ impl BatchManager {
     pub fn config(&self) -> &BatchConfig {
         &self.config
     }
-
-    /// Kill batch queries with larges memory consumption per task. Required to maintain task level
-    /// memory usage in the struct. Will be called by global memory manager.
-    pub fn kill_queries(&self, reason: String) {
-        let mut max_mem_task_id = None;
-        let mut max_mem = usize::MIN;
-        let guard = self.tasks.lock();
-        for (t_id, t) in &*guard {
-            // If the task has been stopped, we should not count this.
-            if t.is_end() {
-                continue;
-            }
-            // Alternatively, we can use a bool flag to indicate end of execution.
-            // Now we use only store 0 bytes in Context after execution ends.
-            let mem_usage = t.mem_usage();
-            if mem_usage > max_mem {
-                max_mem = mem_usage;
-                max_mem_task_id = Some(t_id.clone());
-            }
-        }
-        if let Some(id) = max_mem_task_id {
-            let t = guard.get(&id).unwrap();
-            // FIXME: `Abort` will not report error but truncated results to user. We should
-            // consider throw error.
-            t.abort(reason);
-        }
-    }
-
-    /// Called by global memory manager for total usage of batch tasks. This op is designed to be
-    /// light-weight
-    pub fn total_mem_usage(&self) -> usize {
-        self.total_mem_val.get() as usize
-    }
-
-    /// Calculate the diff between this time and last time memory usage report, apply the diff for
-    /// the global counter. Due to the limitation of hytra, we need to use i64 type here.
-    pub fn apply_mem_diff(&self, diff: i64) {
-        self.total_mem_val.inc(diff)
-    }
 }
 
 #[cfg(test)]
@@ -343,11 +297,12 @@ mod tests {
     use crate::monitor::BatchManagerMetrics;
     use crate::task::{BatchManager, TaskId};
 
-    #[test]
-    fn test_task_not_found() {
+    #[tokio::test]
+    async fn test_task_not_found() {
         let manager = Arc::new(BatchManager::new(
             BatchConfig::default(),
             BatchManagerMetrics::for_test(),
+            u64::MAX,
         ));
         let task_id = TaskId {
             task_id: 0,
@@ -375,6 +330,7 @@ mod tests {
         let manager = Arc::new(BatchManager::new(
             BatchConfig::default(),
             BatchManagerMetrics::for_test(),
+            u64::MAX,
         ));
         let plan = PlanFragment {
             root: Some(PlanNode {
@@ -415,6 +371,7 @@ mod tests {
         let manager = Arc::new(BatchManager::new(
             BatchConfig::default(),
             BatchManagerMetrics::for_test(),
+            u64::MAX,
         ));
         let plan = PlanFragment {
             root: Some(PlanNode {
@@ -445,6 +402,7 @@ mod tests {
         let manager = Arc::new(BatchManager::new(
             BatchConfig::default(),
             BatchManagerMetrics::for_test(),
+            u64::MAX,
         ));
         let plan = PlanFragment {
             root: Some(PlanNode {
diff --git a/src/batch/src/worker_manager/worker_node_manager.rs b/src/batch/src/worker_manager/worker_node_manager.rs
index 5b0813186fd1c..8b7dcc42b565a 100644
--- a/src/batch/src/worker_manager/worker_node_manager.rs
+++ b/src/batch/src/worker_manager/worker_node_manager.rs
@@ -18,8 +18,7 @@ use std::time::Duration;
 
 use rand::seq::SliceRandom;
 use risingwave_common::bail;
-use risingwave_common::hash::{ParallelUnitId, ParallelUnitMapping};
-use risingwave_common::util::worker_util::get_pu_to_worker_mapping;
+use risingwave_common::hash::{WorkerSlotId, WorkerSlotMapping};
 use risingwave_common::vnode_mapping::vnode_placement::place_vnode;
 use risingwave_pb::common::{WorkerNode, WorkerType};
 
@@ -36,12 +35,10 @@ pub struct WorkerNodeManager {
 
 struct WorkerNodeManagerInner {
     worker_nodes: Vec<WorkerNode>,
-    /// A cache for parallel units to worker nodes. It should be consistent with `worker_nodes`.
-    pu_to_worker: HashMap<ParallelUnitId, WorkerNode>,
     /// fragment vnode mapping info for streaming
-    streaming_fragment_vnode_mapping: HashMap<FragmentId, ParallelUnitMapping>,
+    streaming_fragment_vnode_mapping: HashMap<FragmentId, WorkerSlotMapping>,
     /// fragment vnode mapping info for serving
-    serving_fragment_vnode_mapping: HashMap<FragmentId, ParallelUnitMapping>,
+    serving_fragment_vnode_mapping: HashMap<FragmentId, WorkerSlotMapping>,
 }
 
 pub type WorkerNodeManagerRef = Arc<WorkerNodeManager>;
@@ -57,7 +54,6 @@ impl WorkerNodeManager {
         Self {
             inner: RwLock::new(WorkerNodeManagerInner {
                 worker_nodes: Default::default(),
-                pu_to_worker: Default::default(),
                 streaming_fragment_vnode_mapping: Default::default(),
                 serving_fragment_vnode_mapping: Default::default(),
             }),
@@ -68,7 +64,6 @@ impl WorkerNodeManager {
     /// Used in tests.
     pub fn mock(worker_nodes: Vec<WorkerNode>) -> Self {
         let inner = RwLock::new(WorkerNodeManagerInner {
-            pu_to_worker: get_pu_to_worker_mapping(&worker_nodes),
             worker_nodes,
             streaming_fragment_vnode_mapping: HashMap::new(),
             serving_fragment_vnode_mapping: HashMap::new(),
@@ -120,23 +115,18 @@ impl WorkerNodeManager {
                 *w = node;
             }
         }
-        // Update `pu_to_worker`
-        write_guard.pu_to_worker = get_pu_to_worker_mapping(&write_guard.worker_nodes);
     }
 
     pub fn remove_worker_node(&self, node: WorkerNode) {
         let mut write_guard = self.inner.write().unwrap();
         write_guard.worker_nodes.retain(|x| x.id != node.id);
-
-        // Update `pu_to_worker`
-        write_guard.pu_to_worker = get_pu_to_worker_mapping(&write_guard.worker_nodes);
     }
 
     pub fn refresh(
         &self,
         nodes: Vec<WorkerNode>,
-        streaming_mapping: HashMap<FragmentId, ParallelUnitMapping>,
-        serving_mapping: HashMap<FragmentId, ParallelUnitMapping>,
+        streaming_mapping: HashMap<FragmentId, WorkerSlotMapping>,
+        serving_mapping: HashMap<FragmentId, WorkerSlotMapping>,
     ) {
         let mut write_guard = self.inner.write().unwrap();
         tracing::debug!("Refresh worker nodes {:?}.", nodes);
@@ -149,42 +139,51 @@ impl WorkerNodeManager {
             serving_mapping.keys()
         );
         write_guard.worker_nodes = nodes;
-        // Update `pu_to_worker`
-        write_guard.pu_to_worker = get_pu_to_worker_mapping(&write_guard.worker_nodes);
         write_guard.streaming_fragment_vnode_mapping = streaming_mapping;
         write_guard.serving_fragment_vnode_mapping = serving_mapping;
     }
 
-    /// If parallel unit ids is empty, the scheduler may fail to schedule any task and stuck at
+    /// If worker slot ids is empty, the scheduler may fail to schedule any task and stuck at
     /// schedule next stage. If we do not return error in this case, needs more complex control
     /// logic above. Report in this function makes the schedule root fail reason more clear.
-    pub fn get_workers_by_parallel_unit_ids(
+    pub fn get_workers_by_worker_slot_ids(
         &self,
-        parallel_unit_ids: &[ParallelUnitId],
+        worker_slot_ids: &[WorkerSlotId],
     ) -> Result<Vec<WorkerNode>> {
-        if parallel_unit_ids.is_empty() {
+        if worker_slot_ids.is_empty() {
             return Err(BatchError::EmptyWorkerNodes);
         }
 
         let guard = self.inner.read().unwrap();
 
-        let mut workers = Vec::with_capacity(parallel_unit_ids.len());
-        for parallel_unit_id in parallel_unit_ids {
-            match guard.pu_to_worker.get(parallel_unit_id) {
-                Some(worker) => workers.push(worker.clone()),
+        let worker_slot_index: HashMap<_, _> = guard
+            .worker_nodes
+            .iter()
+            .flat_map(|worker| {
+                (0..worker.parallel_units.len())
+                    .map(move |i| (WorkerSlotId::new(worker.id, i), worker))
+            })
+            .collect();
+
+        let mut workers = Vec::with_capacity(worker_slot_ids.len());
+
+        for worker_slot_id in worker_slot_ids {
+            match worker_slot_index.get(worker_slot_id) {
+                Some(worker) => workers.push((*worker).clone()),
                 None => bail!(
-                    "No worker node found for parallel unit id: {}",
-                    parallel_unit_id
+                    "No worker node found for worker slot id: {}",
+                    worker_slot_id
                 ),
             }
         }
+
         Ok(workers)
     }
 
     pub fn get_streaming_fragment_mapping(
         &self,
         fragment_id: &FragmentId,
-    ) -> Result<ParallelUnitMapping> {
+    ) -> Result<WorkerSlotMapping> {
         self.inner
             .read()
             .unwrap()
@@ -197,7 +196,7 @@ impl WorkerNodeManager {
     pub fn insert_streaming_fragment_mapping(
         &self,
         fragment_id: FragmentId,
-        vnode_mapping: ParallelUnitMapping,
+        vnode_mapping: WorkerSlotMapping,
     ) {
         self.inner
             .write()
@@ -210,7 +209,7 @@ impl WorkerNodeManager {
     pub fn update_streaming_fragment_mapping(
         &self,
         fragment_id: FragmentId,
-        vnode_mapping: ParallelUnitMapping,
+        vnode_mapping: WorkerSlotMapping,
     ) {
         let mut guard = self.inner.write().unwrap();
         guard
@@ -228,7 +227,7 @@ impl WorkerNodeManager {
     }
 
     /// Returns fragment's vnode mapping for serving.
-    fn serving_fragment_mapping(&self, fragment_id: FragmentId) -> Result<ParallelUnitMapping> {
+    fn serving_fragment_mapping(&self, fragment_id: FragmentId) -> Result<WorkerSlotMapping> {
         self.inner
             .read()
             .unwrap()
@@ -236,7 +235,7 @@ impl WorkerNodeManager {
             .ok_or_else(|| BatchError::ServingVnodeMappingNotFound(fragment_id))
     }
 
-    pub fn set_serving_fragment_mapping(&self, mappings: HashMap<FragmentId, ParallelUnitMapping>) {
+    pub fn set_serving_fragment_mapping(&self, mappings: HashMap<FragmentId, WorkerSlotMapping>) {
         let mut guard = self.inner.write().unwrap();
         tracing::debug!(
             "Set serving vnode mapping for fragments {:?}",
@@ -247,7 +246,7 @@ impl WorkerNodeManager {
 
     pub fn upsert_serving_fragment_mapping(
         &self,
-        mappings: HashMap<FragmentId, ParallelUnitMapping>,
+        mappings: HashMap<FragmentId, WorkerSlotMapping>,
     ) {
         let mut guard = self.inner.write().unwrap();
         tracing::debug!(
@@ -299,7 +298,7 @@ impl WorkerNodeManager {
 }
 
 impl WorkerNodeManagerInner {
-    fn get_serving_fragment_mapping(&self, fragment_id: FragmentId) -> Option<ParallelUnitMapping> {
+    fn get_serving_fragment_mapping(&self, fragment_id: FragmentId) -> Option<WorkerSlotMapping> {
         self.serving_fragment_vnode_mapping
             .get(&fragment_id)
             .cloned()
@@ -342,7 +341,7 @@ impl WorkerNodeSelector {
             .sum()
     }
 
-    pub fn fragment_mapping(&self, fragment_id: FragmentId) -> Result<ParallelUnitMapping> {
+    pub fn fragment_mapping(&self, fragment_id: FragmentId) -> Result<WorkerSlotMapping> {
         if self.enable_barrier_read {
             self.manager.get_streaming_fragment_mapping(&fragment_id)
         } else {
diff --git a/src/bench/Cargo.toml b/src/bench/Cargo.toml
index f5c36c6b9adc2..c32b42e4b7354 100644
--- a/src/bench/Cargo.toml
+++ b/src/bench/Cargo.toml
@@ -54,7 +54,7 @@ tracing-subscriber = "0.3.17"
 workspace-hack = { path = "../workspace-hack" }
 
 [target.'cfg(target_os = "linux")'.dependencies]
-nix = { version = "0.28", features = ["fs", "mman"] }
+nix = { version = "0.29", features = ["fs", "mman"] }
 
 [[bin]]
 name = "s3-bench"
diff --git a/src/bench/sink_bench/main.rs b/src/bench/sink_bench/main.rs
index 91ebe6bd44d0f..1637b5df6c659 100644
--- a/src/bench/sink_bench/main.rs
+++ b/src/bench/sink_bench/main.rs
@@ -17,7 +17,7 @@
 #![feature(let_chains)]
 
 use core::str::FromStr;
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 
 use anyhow::anyhow;
 use clap::Parser;
@@ -49,7 +49,6 @@ use risingwave_connector::source::datagen::{
 use risingwave_connector::source::{
     Column, DataType, SourceContext, SourceEnumeratorContext, SplitEnumerator, SplitReader,
 };
-use risingwave_pb::connector_service::SinkPayloadFormat;
 use risingwave_stream::executor::test_utils::prelude::ColumnDesc;
 use risingwave_stream::executor::{Barrier, Message, MessageStreamItem, StreamExecutorError};
 use serde::{Deserialize, Deserializer};
@@ -116,7 +115,7 @@ impl LogReader for MockRangeLogReader {
         }
     }
 
-    async fn truncate(&mut self, _offset: TruncateOffset) -> LogStoreResult<()> {
+    fn truncate(&mut self, _offset: TruncateOffset) -> LogStoreResult<()> {
         Ok(())
     }
 
@@ -358,9 +357,9 @@ fn read_table_schema_from_yml(path: &str) -> TableSchemaFromYml {
     table
 }
 
-fn read_sink_option_from_yml(path: &str) -> HashMap<String, HashMap<String, String>> {
+fn read_sink_option_from_yml(path: &str) -> HashMap<String, BTreeMap<String, String>> {
     let data = std::fs::read_to_string(path).unwrap();
-    let sink_option: HashMap<String, HashMap<String, String>> =
+    let sink_option: HashMap<String, BTreeMap<String, String>> =
         serde_yaml::from_str(&data).unwrap();
     sink_option
 }
@@ -404,6 +403,7 @@ fn mock_from_legacy_type(
             format,
             encode: SinkEncode::Json,
             options: Default::default(),
+            key_encode: None,
         }))
     } else {
         SinkFormatDesc::from_legacy_type(connector, r#type)
@@ -471,9 +471,8 @@ async fn main() {
             sink_from_name: "not_need_set".to_string(),
         };
         let sink = build_sink(sink_param).unwrap();
-        let mut sink_writer_param = SinkWriterParam::for_test();
+        let sink_writer_param = SinkWriterParam::for_test();
         println!("Start Sink Bench!, Wait {:?}s", BENCH_TIME);
-        sink_writer_param.connector_params.sink_payload_format = SinkPayloadFormat::StreamChunk;
         tokio::spawn(async move {
             dispatch_sink!(sink, sink, {
                 consume_log_stream(sink, mock_range_log_reader, sink_writer_param).boxed()
diff --git a/src/cmd_all/Cargo.toml b/src/cmd_all/Cargo.toml
index 5ed92b65609cc..d3341b3137d43 100644
--- a/src/cmd_all/Cargo.toml
+++ b/src/cmd_all/Cargo.toml
@@ -8,11 +8,15 @@ license = { workspace = true }
 repository = { workspace = true }
 
 [features]
+default = ["rw-static-link"]
 rw-static-link = ["workspace-config/rw-static-link"]
 rw-dynamic-link = ["workspace-config/rw-dynamic-link"]
-embedded-deno-udf = ["risingwave_expr/embedded-deno-udf"]
-embedded-python-udf = ["risingwave_expr/embedded-python-udf"]
-default = ["rw-static-link"]
+all-udf = ["external-udf", "wasm-udf", "js-udf", "deno-udf", "python-udf"]
+external-udf = ["risingwave_expr_impl/external-udf"]
+wasm-udf = ["risingwave_expr_impl/wasm-udf"]
+js-udf = ["risingwave_expr_impl/js-udf"]
+deno-udf = ["risingwave_expr_impl/deno-udf"]
+python-udf = ["risingwave_expr_impl/python-udf"]
 
 [package.metadata.cargo-machete]
 ignored = ["workspace-hack", "workspace-config", "task_stats_alloc"]
@@ -32,7 +36,6 @@ risingwave_common = { workspace = true }
 risingwave_compactor = { workspace = true }
 risingwave_compute = { workspace = true }
 risingwave_ctl = { workspace = true }
-risingwave_expr = { workspace = true }
 risingwave_expr_impl = { workspace = true }
 risingwave_frontend = { workspace = true }
 risingwave_meta_node = { workspace = true }
diff --git a/src/cmd_all/scripts/e2e-full-standalone-demo.sh b/src/cmd_all/scripts/e2e-full-standalone-demo.sh
index 28469aaddbe70..409c543e03afa 100755
--- a/src/cmd_all/scripts/e2e-full-standalone-demo.sh
+++ b/src/cmd_all/scripts/e2e-full-standalone-demo.sh
@@ -20,17 +20,14 @@
 set -euo pipefail
 
 insert_json_kafka() {
-  echo $1 | \
-    $KAFKA_PATH/bin/kafka-console-producer.sh \
-      --topic source_kafka \
-      --bootstrap-server localhost:29092
+  echo $1 |
+    RPK_BROKERS=localhost:29092 \
+      rpk topic produce source_kafka -f "%v"
 }
 
 create_topic_kafka() {
-  "$KAFKA_PATH"/bin/kafka-topics.sh \
-    --create \
-    --topic source_kafka \
-    --bootstrap-server localhost:29092
+  RPK_BROKERS=localhost:29092 \
+    rpk topic create source_kafka
 }
 
 # Make sure we start on clean cluster
diff --git a/src/cmd_all/src/single_node.rs b/src/cmd_all/src/single_node.rs
index 63099c1ff640c..2340841a13c31 100644
--- a/src/cmd_all/src/single_node.rs
+++ b/src/cmd_all/src/single_node.rs
@@ -186,7 +186,7 @@ pub fn map_single_node_opts_to_standalone_opts(opts: SingleNodeOpts) -> ParsedSt
             std::fs::create_dir_all(&meta_store_dir).unwrap();
             let meta_store_endpoint =
                 format!("sqlite://{}/single_node.db?mode=rwc", &meta_store_dir);
-            meta_opts.sql_endpoint = Some(meta_store_endpoint);
+            meta_opts.sql_endpoint = Some(meta_store_endpoint.into());
         }
     }
 
diff --git a/src/cmd_all/src/standalone.rs b/src/cmd_all/src/standalone.rs
index 90ba35b99d626..819aba03d865e 100644
--- a/src/cmd_all/src/standalone.rs
+++ b/src/cmd_all/src/standalone.rs
@@ -356,9 +356,9 @@ mod test {
                                     http://127.0.0.1:5690/,
                                 ],
                             ),
-                            connector_rpc_sink_payload_format: None,
                             config_path: "src/config/test.toml",
                             total_memory_bytes: 34359738368,
+                            reserved_memory_bytes: None,
                             parallelism: 10,
                             role: Both,
                             metrics_level: None,
@@ -371,7 +371,7 @@ mod test {
                     ),
                     frontend_opts: Some(
                         FrontendOpts {
-                            listen_addr: "127.0.0.1:4566",
+                            listen_addr: "0.0.0.0:4566",
                             advertise_addr: None,
                             meta_addr: List(
                                 [
diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml
index d21e276089c2c..e5b4a2feaa2a0 100644
--- a/src/common/Cargo.toml
+++ b/src/common/Cargo.toml
@@ -14,6 +14,7 @@ ignored = ["workspace-hack"]
 normal = ["workspace-hack"]
 
 [dependencies]
+ahash = "0.8"
 anyhow = "1"
 arc-swap = "1"
 arrow-array = { workspace = true }
@@ -34,12 +35,12 @@ chrono = { version = "0.4", default-features = false, features = [
     "clock",
     "std",
 ] }
-chrono-tz = { version = "0.8", features = ["case-insensitive"] }
+chrono-tz = { version = "0.9", features = ["case-insensitive"] }
 clap = { workspace = true }
 comfy-table = "7"
 crc32fast = "1"
 easy-ext = "1"
-educe = "0.5"
+educe = "0.6"
 either = "1"
 enum-as-inner = "0.6"
 enumflags2 = { version = "0.7.8" }
@@ -48,13 +49,14 @@ fixedbitset = "0.5"
 foyer = { workspace = true }
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 governor = { version = "0.6", default-features = false, features = ["std"] }
+hashbrown = "0.14"
 hex = "0.4.3"
 http = "0.2"
 humantime = "2.1"
 hytra = { workspace = true }
 itertools = { workspace = true }
 itoa = "1.0"
-jsonbb = "0.1.2"
+jsonbb = { workspace = true }
 lru = { workspace = true }
 memcomparable = { version = "0.2", features = ["decimal"] }
 num-integer = "0.1"
@@ -135,6 +137,7 @@ libc = "0.2"
 mach2 = "0.4"
 
 [dev-dependencies]
+coarsetime = "0.1"
 criterion = { workspace = true }
 expect-test = "1"
 more-asserts = "0.3"
@@ -167,5 +170,13 @@ harness = false
 name = "bench_array"
 harness = false
 
+[[bench]]
+name = "bench_sequencer"
+harness = false
+
+[[bench]]
+name = "bench_lru"
+harness = false
+
 [lints]
 workspace = true
diff --git a/src/common/benches/bench_lru.rs b/src/common/benches/bench_lru.rs
new file mode 100644
index 0000000000000..84f49a405a106
--- /dev/null
+++ b/src/common/benches/bench_lru.rs
@@ -0,0 +1,88 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::hint::black_box;
+use std::sync::atomic::Ordering;
+use std::time::{Duration, Instant};
+
+use itertools::Itertools;
+use lru::LruCache;
+use risingwave_common::lru::LruCache as RwLruCache;
+use risingwave_common::sequence::SEQUENCE_GLOBAL;
+
+fn lru(loops: usize, evict_ratio: u64) -> (usize, Duration) {
+    let mut lru = LruCache::unbounded();
+    let mut evicted = 0;
+    let now = Instant::now();
+    for i in 0..loops as u64 {
+        if i % evict_ratio == 0 && i != 0 {
+            lru.update_epoch(i);
+            while lru.pop_lru_by_epoch(i).is_some() {
+                evicted += 1;
+            }
+        }
+        lru.put(i, i);
+    }
+
+    (evicted, now.elapsed())
+}
+
+fn rw_lru(loops: usize, evict_ratio: u64) -> (usize, Duration) {
+    let mut lru = RwLruCache::unbounded();
+    let mut evicted = 0;
+    let now = Instant::now();
+    for i in 0..loops as u64 {
+        if i % evict_ratio == 0 {
+            let sequence = SEQUENCE_GLOBAL.load(Ordering::Relaxed);
+            while lru.pop_with_sequence(sequence).is_some() {
+                evicted += 1;
+            }
+        }
+        lru.put(i, i);
+    }
+
+    (evicted, now.elapsed())
+}
+
+fn benchmark<F>(name: &str, threads: usize, loops: usize, f: F)
+where
+    F: Fn() -> (usize, Duration) + Clone + Send + 'static,
+{
+    let handles = (0..threads)
+        .map(|_| std::thread::spawn(black_box(f.clone())))
+        .collect_vec();
+    let mut dur = Duration::from_nanos(0);
+    let mut evicted = 0;
+    for handle in handles {
+        let (e, d) = handle.join().unwrap();
+        evicted += e;
+        dur += d;
+    }
+    println!(
+        "{:20} {} threads {} loops: {:?} per iter, total evicted: {}",
+        name,
+        threads,
+        loops,
+        Duration::from_nanos((dur.as_nanos() / threads as u128 / loops as u128) as u64),
+        evicted,
+    );
+}
+
+fn main() {
+    for threads in [1, 4, 8, 16, 32, 64] {
+        println!();
+        benchmark("lru - 1024", threads, 1000000, || lru(1000000, 1024));
+        benchmark("rw  - 1024", threads, 1000000, || rw_lru(1000000, 1024));
+    }
+}
diff --git a/src/common/benches/bench_sequencer.rs b/src/common/benches/bench_sequencer.rs
new file mode 100644
index 0000000000000..12e92f1f3332d
--- /dev/null
+++ b/src/common/benches/bench_sequencer.rs
@@ -0,0 +1,170 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(lint_reasons)]
+
+use std::cell::RefCell;
+use std::hint::black_box;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use itertools::Itertools;
+use risingwave_common::sequence::*;
+
+thread_local! {
+    pub static SEQUENCER_64_8: RefCell<Sequencer> = const { RefCell::new(Sequencer::new(64, 64 * 8)) };
+    pub static SEQUENCER_64_16: RefCell<Sequencer> = const { RefCell::new(Sequencer::new(64, 64 * 16)) };
+    pub static SEQUENCER_64_32: RefCell<Sequencer> = const { RefCell::new(Sequencer::new(64, 64 * 32)) };
+    pub static SEQUENCER_128_8: RefCell<Sequencer> = const { RefCell::new(Sequencer::new(128, 128 * 8)) };
+    pub static SEQUENCER_128_16: RefCell<Sequencer> = const { RefCell::new(Sequencer::new(128, 128 * 16)) };
+    pub static SEQUENCER_128_32: RefCell<Sequencer> = const { RefCell::new(Sequencer::new(128, 128 * 32)) };
+}
+
+fn coarse(loops: usize) -> Duration {
+    let now = Instant::now();
+    for _ in 0..loops {
+        let _ = coarsetime::Instant::now();
+    }
+    now.elapsed()
+}
+
+#[expect(clippy::explicit_counter_loop)]
+fn primitive(loops: usize) -> Duration {
+    let mut cnt = 0usize;
+    let now = Instant::now();
+    for _ in 0..loops {
+        cnt += 1;
+        let _ = cnt;
+    }
+    now.elapsed()
+}
+
+fn atomic(loops: usize, atomic: Arc<AtomicUsize>) -> Duration {
+    let now = Instant::now();
+    for _ in 0..loops {
+        let _ = atomic.fetch_add(1, Ordering::Relaxed);
+    }
+    now.elapsed()
+}
+
+fn atomic_skip(loops: usize, atomic: Arc<AtomicUsize>, skip: usize) -> Duration {
+    let mut cnt = 0usize;
+    let now = Instant::now();
+    for _ in 0..loops {
+        cnt += 1;
+        let _ = cnt;
+        if cnt % skip == 0 {
+            let _ = atomic.fetch_add(skip, Ordering::Relaxed);
+        } else {
+            let _ = atomic.load(Ordering::Relaxed);
+        }
+    }
+    now.elapsed()
+}
+
+fn sequencer(loops: usize, step: Sequence, lag_amp: Sequence) -> Duration {
+    let sequencer = match (step, lag_amp) {
+        (64, 8) => &SEQUENCER_64_8,
+        (64, 16) => &SEQUENCER_64_16,
+        (64, 32) => &SEQUENCER_64_32,
+        (128, 8) => &SEQUENCER_128_8,
+        (128, 16) => &SEQUENCER_128_16,
+        (128, 32) => &SEQUENCER_128_32,
+        _ => unimplemented!(),
+    };
+    let now = Instant::now();
+    for _ in 0..loops {
+        let _ = sequencer.with(|s| s.borrow_mut().alloc());
+    }
+    now.elapsed()
+}
+
+fn benchmark<F>(name: &str, threads: usize, loops: usize, f: F)
+where
+    F: Fn() -> Duration + Clone + Send + 'static,
+{
+    let handles = (0..threads)
+        .map(|_| std::thread::spawn(black_box(f.clone())))
+        .collect_vec();
+    let mut dur = Duration::from_nanos(0);
+    for handle in handles {
+        dur += handle.join().unwrap();
+    }
+    println!(
+        "{:20} {} threads {} loops: {:?} per iter",
+        name,
+        threads,
+        loops,
+        Duration::from_nanos((dur.as_nanos() / threads as u128 / loops as u128) as u64)
+    );
+}
+
+fn main() {
+    for (threads, loops) in [
+        (1, 10_000_000),
+        (4, 10_000_000),
+        (8, 10_000_000),
+        (16, 10_000_000),
+        (32, 10_000_000),
+    ] {
+        println!();
+
+        benchmark("primitive", threads, loops, move || primitive(loops));
+
+        let a = Arc::new(AtomicUsize::new(0));
+        benchmark("atomic", threads, loops, move || atomic(loops, a.clone()));
+
+        let a = Arc::new(AtomicUsize::new(0));
+        benchmark("atomic skip 8", threads, loops, move || {
+            atomic_skip(loops, a.clone(), 8)
+        });
+
+        let a = Arc::new(AtomicUsize::new(0));
+        benchmark("atomic skip 16", threads, loops, move || {
+            atomic_skip(loops, a.clone(), 16)
+        });
+
+        let a = Arc::new(AtomicUsize::new(0));
+        benchmark("atomic skip 32", threads, loops, move || {
+            atomic_skip(loops, a.clone(), 32)
+        });
+
+        let a = Arc::new(AtomicUsize::new(0));
+        benchmark("atomic skip 64", threads, loops, move || {
+            atomic_skip(loops, a.clone(), 64)
+        });
+
+        benchmark("sequencer(64,8)", threads, loops, move || {
+            sequencer(loops, 64, 8)
+        });
+        benchmark("sequencer(64,16)", threads, loops, move || {
+            sequencer(loops, 64, 16)
+        });
+        benchmark("sequencer(64,32)", threads, loops, move || {
+            sequencer(loops, 64, 32)
+        });
+        benchmark("sequencer(128,8)", threads, loops, move || {
+            sequencer(loops, 128, 8)
+        });
+        benchmark("sequencer(128,16)", threads, loops, move || {
+            sequencer(loops, 128, 16)
+        });
+        benchmark("sequencer(128,32)", threads, loops, move || {
+            sequencer(loops, 128, 32)
+        });
+
+        benchmark("coarse", threads, loops, move || coarse(loops));
+    }
+}
diff --git a/src/common/common_service/src/observer_manager.rs b/src/common/common_service/src/observer_manager.rs
index 3f18be7697520..e760a0e16866c 100644
--- a/src/common/common_service/src/observer_manager.rs
+++ b/src/common/common_service/src/observer_manager.rs
@@ -27,6 +27,7 @@ pub trait SubscribeTypeEnum {
 }
 
 pub struct SubscribeFrontend {}
+
 impl SubscribeTypeEnum for SubscribeFrontend {
     fn subscribe_type() -> SubscribeType {
         SubscribeType::Frontend
@@ -34,6 +35,7 @@ impl SubscribeTypeEnum for SubscribeFrontend {
 }
 
 pub struct SubscribeHummock {}
+
 impl SubscribeTypeEnum for SubscribeHummock {
     fn subscribe_type() -> SubscribeType {
         SubscribeType::Hummock
@@ -41,6 +43,7 @@ impl SubscribeTypeEnum for SubscribeHummock {
 }
 
 pub struct SubscribeCompactor {}
+
 impl SubscribeTypeEnum for SubscribeCompactor {
     fn subscribe_type() -> SubscribeType {
         SubscribeType::Compactor
@@ -48,6 +51,7 @@ impl SubscribeTypeEnum for SubscribeCompactor {
 }
 
 pub struct SubscribeCompute {}
+
 impl SubscribeTypeEnum for SubscribeCompute {
     fn subscribe_type() -> SubscribeType {
         SubscribeType::Compute
@@ -142,12 +146,10 @@ where
             | Info::RelationGroup(_)
             | Info::User(_)
             | Info::Connection(_)
+            | Info::Secret(_)
             | Info::Function(_) => {
                 notification.version > info.version.as_ref().unwrap().catalog_version
             }
-            Info::ParallelUnitMapping(_) => {
-                notification.version > info.version.as_ref().unwrap().parallel_unit_mapping_version
-            }
             Info::Node(_) => {
                 notification.version > info.version.as_ref().unwrap().worker_node_version
             }
@@ -157,10 +159,18 @@ where
             Info::HummockSnapshot(_) => true,
             Info::MetaBackupManifestId(_) => true,
             Info::SystemParams(_) | Info::SessionParam(_) => true,
-            Info::ServingParallelUnitMappings(_) => true,
             Info::Snapshot(_) | Info::HummockWriteLimits(_) => unreachable!(),
             Info::HummockStats(_) => true,
             Info::Recovery(_) => true,
+            Info::StreamingWorkerSlotMapping(_) => {
+                notification.version
+                    > info
+                        .version
+                        .as_ref()
+                        .unwrap()
+                        .streaming_worker_slot_mapping_version
+            }
+            Info::ServingWorkerSlotMappings(_) => true,
         });
 
         self.observer_states
@@ -226,6 +236,7 @@ where
         }
     }
 }
+
 const RE_SUBSCRIBE_RETRY_INTERVAL: Duration = Duration::from_millis(100);
 
 #[async_trait::async_trait]
diff --git a/src/common/estimate_size/Cargo.toml b/src/common/estimate_size/Cargo.toml
index 04bcf369cc588..77e4203f9c7cb 100644
--- a/src/common/estimate_size/Cargo.toml
+++ b/src/common/estimate_size/Cargo.toml
@@ -16,10 +16,10 @@ normal = ["workspace-hack"]
 
 [dependencies]
 bytes = "1"
-educe = "0.5"
+educe = "0.6"
 ethnum = { version = "1", features = ["serde"] }
 fixedbitset = "0.5"
-jsonbb = "0.1.2"
+jsonbb = { workspace = true }
 lru = { workspace = true }
 risingwave_common_proc_macro = { workspace = true }
 rust_decimal = "1"
diff --git a/src/common/estimate_size/src/collections/btreemap.rs b/src/common/estimate_size/src/collections/btreemap.rs
index cf4398eb191df..84ecd687d8df1 100644
--- a/src/common/estimate_size/src/collections/btreemap.rs
+++ b/src/common/estimate_size/src/collections/btreemap.rs
@@ -92,7 +92,11 @@ where
 
         // [ left, [mid], right ]
         let mut mid_right = self.inner.split_off(start);
-        let mid_right_split_key = mid_right.lower_bound(Bound::Excluded(end)).key().cloned();
+        let mid_right_split_key = mid_right
+            .lower_bound(Bound::Excluded(end))
+            .peek_next()
+            .map(|(k, _)| k)
+            .cloned();
         let right = if let Some(ref mid_right_split_key) = mid_right_split_key {
             mid_right.split_off(mid_right_split_key)
         } else {
diff --git a/src/common/estimate_size/src/collections/lru.rs b/src/common/estimate_size/src/collections/lru.rs
deleted file mode 100644
index 4add8df754673..0000000000000
--- a/src/common/estimate_size/src/collections/lru.rs
+++ /dev/null
@@ -1,159 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::alloc::{Allocator, Global};
-use std::borrow::Borrow;
-use std::hash::{BuildHasher, Hash};
-
-use lru::{DefaultHasher, LruCache};
-
-use super::{AtomicMutGuard, MutGuard};
-use crate::{EstimateSize, KvSize};
-
-/// The managed cache is a lru cache that bounds the memory usage by epoch.
-/// Should be used with `MemoryManager`.
-pub struct EstimatedLruCache<K, V, S = DefaultHasher, A: Clone + Allocator = Global> {
-    inner: LruCache<K, V, S, A>,
-    kv_heap_size: KvSize,
-}
-
-impl<K: Hash + Eq + EstimateSize, V: EstimateSize, S: BuildHasher, A: Clone + Allocator>
-    EstimatedLruCache<K, V, S, A>
-{
-    pub fn with_hasher_in(hasher: S, alloc: A) -> Self {
-        Self {
-            inner: LruCache::unbounded_with_hasher_in(hasher, alloc),
-            kv_heap_size: KvSize::new(),
-        }
-    }
-
-    /// Evict epochs lower than the watermark
-    pub fn evict_by_epoch(&mut self, epoch: u64) {
-        while let Some((key, value, _)) = self.inner.pop_lru_by_epoch(epoch) {
-            self.kv_heap_size.sub(&key, &value);
-        }
-    }
-
-    pub fn update_epoch(&mut self, epoch: u64) {
-        self.inner.update_epoch(epoch);
-    }
-
-    pub fn current_epoch(&mut self) -> u64 {
-        self.inner.current_epoch()
-    }
-
-    /// An iterator visiting all values in most-recently used order. The iterator element type is
-    /// &V.
-    pub fn values(&self) -> impl Iterator<Item = &V> {
-        self.inner.iter().map(|(_k, v)| v)
-    }
-
-    pub fn get_mut(&mut self, k: &K) -> Option<MutGuard<'_, V>> {
-        let v = self.inner.get_mut(k);
-        v.map(|inner| MutGuard::new(inner, &mut self.kv_heap_size))
-    }
-
-    pub fn get<Q>(&mut self, k: &Q) -> Option<&V>
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        self.inner.get(k)
-    }
-
-    pub fn iter_mut(
-        &mut self,
-    ) -> impl ExactSizeIterator<Item = (&'_ K, AtomicMutGuard<'_, V>)> + '_ {
-        let kv_heap_size = &self.kv_heap_size;
-        self.inner.iter_mut().map(move |(k, v)| {
-            let guard = AtomicMutGuard::new(v, kv_heap_size);
-            (k, guard)
-        })
-    }
-
-    pub fn peek_mut(&mut self, k: &K) -> Option<MutGuard<'_, V>> {
-        let v = self.inner.peek_mut(k);
-        v.map(|inner| MutGuard::new(inner, &mut self.kv_heap_size))
-    }
-
-    pub fn push(&mut self, k: K, v: V) -> Option<(K, V)> {
-        self.kv_heap_size.add(&k, &v);
-
-        let old_kv = self.inner.push(k, v);
-
-        if let Some((old_key, old_val)) = &old_kv {
-            self.kv_heap_size.sub(old_key, old_val);
-        }
-        old_kv
-    }
-
-    pub fn contains<Q>(&self, k: &Q) -> bool
-    where
-        K: Borrow<Q>,
-        Q: Hash + Eq + ?Sized,
-    {
-        self.inner.contains(k)
-    }
-
-    pub fn len(&self) -> usize {
-        self.inner.len()
-    }
-
-    pub fn is_empty(&self) -> bool {
-        self.inner.len() == 0
-    }
-
-    pub fn clear(&mut self) {
-        self.inner.clear();
-    }
-}
-
-impl<K: Hash + Eq + EstimateSize, V: EstimateSize> EstimatedLruCache<K, V> {
-    pub fn unbounded() -> Self {
-        Self {
-            inner: LruCache::unbounded(),
-            kv_heap_size: KvSize::new(),
-        }
-    }
-}
-
-impl<K: Hash + Eq + EstimateSize, V: EstimateSize, S: BuildHasher> EstimatedLruCache<K, V, S> {
-    pub fn unbounded_with_hasher(hasher: S) -> Self {
-        Self {
-            inner: LruCache::unbounded_with_hasher(hasher),
-            kv_heap_size: KvSize::new(),
-        }
-    }
-}
-
-impl<K: Hash + Eq + EstimateSize, V: EstimateSize, S: BuildHasher, A: Allocator + Clone>
-    EstimatedLruCache<K, V, S, A>
-{
-    pub fn unbounded_with_hasher_in(hasher: S, allocator: A) -> Self {
-        Self {
-            inner: LruCache::unbounded_with_hasher_in(hasher, allocator),
-            kv_heap_size: KvSize::new(),
-        }
-    }
-}
-
-impl<K: Hash + Eq + EstimateSize, V: EstimateSize, S: BuildHasher, A: Clone + Allocator>
-    EstimateSize for EstimatedLruCache<K, V, S, A>
-{
-    fn estimated_heap_size(&self) -> usize {
-        // TODO: Add lru cache internal size
-        // https://github.com/risingwavelabs/risingwave/issues/9713
-        self.kv_heap_size.size()
-    }
-}
diff --git a/src/common/estimate_size/src/collections/mod.rs b/src/common/estimate_size/src/collections/mod.rs
index f7cdff490a88d..5bffd5133eddc 100644
--- a/src/common/estimate_size/src/collections/mod.rs
+++ b/src/common/estimate_size/src/collections/mod.rs
@@ -16,7 +16,6 @@ use std::ops::{Deref, DerefMut};
 
 use super::{EstimateSize, KvSize};
 
-pub mod lru;
 pub mod vecdeque;
 pub use vecdeque::EstimatedVecDeque;
 pub mod hashmap;
diff --git a/src/common/heap_profiling/src/jeprof.rs b/src/common/heap_profiling/src/jeprof.rs
index b0ae0f1658c9e..6c17fed7045a5 100644
--- a/src/common/heap_profiling/src/jeprof.rs
+++ b/src/common/heap_profiling/src/jeprof.rs
@@ -14,7 +14,6 @@
 
 use std::path::Path;
 use std::process::Command;
-use std::result::Result;
 use std::{env, fs};
 
 /// Error type for running `jeprof`.
diff --git a/src/common/metrics/src/guarded_metrics.rs b/src/common/metrics/src/guarded_metrics.rs
index 9c0919d7a420b..27710748ae359 100644
--- a/src/common/metrics/src/guarded_metrics.rs
+++ b/src/common/metrics/src/guarded_metrics.rs
@@ -21,31 +21,15 @@ use std::sync::Arc;
 use itertools::Itertools;
 use parking_lot::Mutex;
 use prometheus::core::{
-    Atomic, AtomicF64, AtomicI64, AtomicU64, Collector, Desc, GenericCounter, GenericCounterVec,
-    GenericGauge, GenericGaugeVec, GenericLocalCounter, MetricVec, MetricVecBuilder,
+    Atomic, AtomicF64, AtomicI64, AtomicU64, Collector, Desc, GenericCounter, GenericLocalCounter,
+    MetricVec, MetricVecBuilder,
 };
 use prometheus::local::{LocalHistogram, LocalIntCounter};
 use prometheus::proto::MetricFamily;
-use prometheus::{Gauge, Histogram, HistogramVec, IntCounter, IntGauge};
+use prometheus::{Gauge, Histogram, IntCounter, IntGauge};
 use thiserror_ext::AsReport;
 use tracing::warn;
 
-pub fn __extract_counter_builder<P: Atomic>(
-    vec: GenericCounterVec<P>,
-) -> MetricVec<VecBuilderOfCounter<P>> {
-    vec
-}
-
-pub fn __extract_gauge_builder<P: Atomic>(
-    vec: GenericGaugeVec<P>,
-) -> MetricVec<VecBuilderOfGauge<P>> {
-    vec
-}
-
-pub fn __extract_histogram_builder(vec: HistogramVec) -> MetricVec<VecBuilderOfHistogram> {
-    vec
-}
-
 #[macro_export]
 macro_rules! register_guarded_histogram_vec_with_registry {
     ($NAME:expr, $HELP:expr, $LABELS_NAMES:expr, $REGISTRY:expr $(,)?) => {{
@@ -55,6 +39,13 @@ macro_rules! register_guarded_histogram_vec_with_registry {
             $REGISTRY
         }
     }};
+    ($NAME:expr, $HELP:expr, $LABELS_NAMES:expr, $BUCKETS:expr, $REGISTRY:expr $(,)?) => {{
+        $crate::register_guarded_histogram_vec_with_registry! {
+            {prometheus::histogram_opts!($NAME, $HELP, $BUCKETS)},
+            $LABELS_NAMES,
+            $REGISTRY
+        }
+    }};
     ($HOPTS:expr, $LABELS_NAMES:expr, $REGISTRY:expr $(,)?) => {{
         let inner = prometheus::HistogramVec::new($HOPTS, $LABELS_NAMES);
         inner.and_then(|inner| {
@@ -105,9 +96,35 @@ macro_rules! register_guarded_int_counter_vec_with_registry {
     }};
 }
 
-pub type VecBuilderOfCounter<P: Atomic> = impl MetricVecBuilder<M = GenericCounter<P>>;
-pub type VecBuilderOfGauge<P: Atomic> = impl MetricVecBuilder<M = GenericGauge<P>>;
-pub type VecBuilderOfHistogram = impl MetricVecBuilder<M = Histogram>;
+// put TAITs in a separate module to avoid "non-defining opaque type use in defining scope"
+mod tait {
+    use prometheus::core::{
+        Atomic, GenericCounter, GenericCounterVec, GenericGauge, GenericGaugeVec, MetricVec,
+        MetricVecBuilder,
+    };
+    use prometheus::{Histogram, HistogramVec};
+
+    pub type VecBuilderOfCounter<P: Atomic> = impl MetricVecBuilder<M = GenericCounter<P>>;
+    pub type VecBuilderOfGauge<P: Atomic> = impl MetricVecBuilder<M = GenericGauge<P>>;
+    pub type VecBuilderOfHistogram = impl MetricVecBuilder<M = Histogram>;
+
+    pub fn __extract_counter_builder<P: Atomic>(
+        vec: GenericCounterVec<P>,
+    ) -> MetricVec<VecBuilderOfCounter<P>> {
+        vec
+    }
+
+    pub fn __extract_gauge_builder<P: Atomic>(
+        vec: GenericGaugeVec<P>,
+    ) -> MetricVec<VecBuilderOfGauge<P>> {
+        vec
+    }
+
+    pub fn __extract_histogram_builder(vec: HistogramVec) -> MetricVec<VecBuilderOfHistogram> {
+        vec
+    }
+}
+pub use tait::*;
 
 pub type LabelGuardedHistogramVec<const N: usize> = LabelGuardedMetricVec<VecBuilderOfHistogram, N>;
 pub type LabelGuardedIntCounterVec<const N: usize> =
@@ -320,7 +337,7 @@ pub struct LabelGuardedMetric<T, const N: usize> {
     _guard: Arc<LabelGuard<N>>,
 }
 
-impl<T: MetricVecBuilder, const N: usize> Debug for LabelGuardedMetric<T, N> {
+impl<T, const N: usize> Debug for LabelGuardedMetric<T, N> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("LabelGuardedMetric").finish()
     }
diff --git a/src/common/metrics/src/lib.rs b/src/common/metrics/src/lib.rs
index a2e4156b525d1..574e684c6703b 100644
--- a/src/common/metrics/src/lib.rs
+++ b/src/common/metrics/src/lib.rs
@@ -15,7 +15,6 @@
 #![feature(lazy_cell)]
 #![feature(type_alias_impl_trait)]
 #![feature(impl_trait_in_assoc_type)]
-#![feature(array_methods)]
 use std::ops::Deref;
 use std::sync::LazyLock;
 
diff --git a/src/common/metrics/src/monitor/connection.rs b/src/common/metrics/src/monitor/connection.rs
index f5570ff97f60f..295fb6399ba4b 100644
--- a/src/common/metrics/src/monitor/connection.rs
+++ b/src/common/metrics/src/monitor/connection.rs
@@ -744,7 +744,7 @@ impl MonitorAsyncReadWrite for MonitorAsyncReadWriteImpl {
     }
 
     fn on_read_err(&mut self, err: &Error) {
-        // No need to store the value returned from with_label_values
+        // No need to store the value returned from `with_guarded_label_values`
         // because it is reporting a single error.
         GLOBAL_CONNECTION_METRICS
             .io_err_rate
@@ -775,7 +775,7 @@ impl MonitorAsyncReadWrite for MonitorAsyncReadWriteImpl {
     }
 
     fn on_write_err(&mut self, err: &Error) {
-        // No need to store the value returned from with_label_values
+        // No need to store the value returned from `with_guarded_label_values`
         // because it is reporting a single error.
         GLOBAL_CONNECTION_METRICS
             .io_err_rate
diff --git a/src/common/metrics/src/monitor/rwlock.rs b/src/common/metrics/src/monitor/rwlock.rs
index 46f9d5edea91c..4d65c53801106 100644
--- a/src/common/metrics/src/monitor/rwlock.rs
+++ b/src/common/metrics/src/monitor/rwlock.rs
@@ -16,31 +16,34 @@ use prometheus::HistogramVec;
 use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
 
 pub struct MonitoredRwLock<T> {
+    // labels: [lock_name, lock_type]
     metrics: HistogramVec,
     inner: RwLock<T>,
+    lock_name: &'static str,
 }
 
 impl<T> MonitoredRwLock<T> {
-    pub fn new(metrics: HistogramVec, val: T) -> Self {
+    pub fn new(metrics: HistogramVec, val: T, lock_name: &'static str) -> Self {
         Self {
             metrics,
             inner: RwLock::new(val),
+            lock_name,
         }
     }
 
-    pub async fn read<'a, 'b>(
-        &'a self,
-        label_values: &'b [&'static str],
-    ) -> RwLockReadGuard<'a, T> {
-        let _timer = self.metrics.with_label_values(label_values).start_timer();
+    pub async fn read(&self) -> RwLockReadGuard<'_, T> {
+        let _timer = self
+            .metrics
+            .with_label_values(&[self.lock_name, "read"])
+            .start_timer();
         self.inner.read().await
     }
 
-    pub async fn write<'a, 'b>(
-        &'a self,
-        label_values: &'b [&'static str],
-    ) -> RwLockWriteGuard<'a, T> {
-        let _timer = self.metrics.with_label_values(label_values).start_timer();
+    pub async fn write(&self) -> RwLockWriteGuard<'_, T> {
+        let _timer = self
+            .metrics
+            .with_label_values(&[self.lock_name, "write"])
+            .start_timer();
         self.inner.write().await
     }
 }
diff --git a/src/common/metrics/src/relabeled_metric.rs b/src/common/metrics/src/relabeled_metric.rs
index 1b4b4e8aaa038..8c0582f15f875 100644
--- a/src/common/metrics/src/relabeled_metric.rs
+++ b/src/common/metrics/src/relabeled_metric.rs
@@ -16,16 +16,17 @@ use prometheus::core::{MetricVec, MetricVecBuilder};
 use prometheus::{HistogramVec, IntCounterVec};
 
 use crate::{
-    LabelGuardedHistogramVec, LabelGuardedIntCounterVec, LabelGuardedMetric, LabelGuardedMetricVec,
-    MetricLevel,
+    LabelGuardedHistogramVec, LabelGuardedIntCounterVec, LabelGuardedIntGaugeVec,
+    LabelGuardedMetric, LabelGuardedMetricVec, MetricLevel,
 };
 
 /// For all `Relabeled*Vec` below,
 /// - when `metric_level` <= `relabel_threshold`, they behaves exactly the same as their inner
 ///   metric.
 /// - when `metric_level` > `relabel_threshold`, all their input label values are rewrite to "" when
-/// calling `with_label_values`. That's means the metric vec is aggregated into a single metric.
-
+///   calling `with_label_values`. That's means the metric vec is aggregated into a single metric.
+///
+///
 /// These wrapper classes add a `metric_level` field to corresponding metric.
 /// We could have use one single struct to represent all `MetricVec<T: MetricVecBuilder>`, rather
 /// than specializing them one by one. However, that's undoable because prometheus crate doesn't
@@ -108,3 +109,5 @@ pub type RelabeledGuardedHistogramVec<const N: usize> =
     RelabeledMetricVec<LabelGuardedHistogramVec<N>>;
 pub type RelabeledGuardedIntCounterVec<const N: usize> =
     RelabeledMetricVec<LabelGuardedIntCounterVec<N>>;
+pub type RelabeledGuardedIntGaugeVec<const N: usize> =
+    RelabeledMetricVec<LabelGuardedIntGaugeVec<N>>;
diff --git a/src/common/src/acl/mod.rs b/src/common/src/acl/mod.rs
index 91f69fea51310..b1c41bc8ec6ad 100644
--- a/src/common/src/acl/mod.rs
+++ b/src/common/src/acl/mod.rs
@@ -14,7 +14,6 @@
 
 //! `Acl` defines all grantable privileges.
 
-use std::convert::Into;
 use std::fmt::Formatter;
 use std::sync::LazyLock;
 
@@ -100,11 +99,11 @@ pub static ALL_AVAILABLE_DATABASE_MODES: LazyLock<AclModeSet> =
     LazyLock::new(|| make_bitflags!(AclMode::{Create | Connect}).into());
 pub static ALL_AVAILABLE_SCHEMA_MODES: LazyLock<AclModeSet> =
     LazyLock::new(|| make_bitflags!(AclMode::{Create | Usage}).into());
+// Including TABLES and VIEWS
 pub static ALL_AVAILABLE_TABLE_MODES: LazyLock<AclModeSet> =
     LazyLock::new(|| make_bitflags!(AclMode::{Select | Insert | Update | Delete}).into());
 pub static ALL_AVAILABLE_SOURCE_MODES: LazyLock<AclModeSet> = LazyLock::new(AclModeSet::readonly);
 pub static ALL_AVAILABLE_MVIEW_MODES: LazyLock<AclModeSet> = LazyLock::new(AclModeSet::readonly);
-pub static ALL_AVAILABLE_VIEW_MODES: LazyLock<AclModeSet> = LazyLock::new(AclModeSet::readonly);
 pub static ALL_AVAILABLE_SINK_MODES: LazyLock<AclModeSet> = LazyLock::new(AclModeSet::empty);
 pub static ALL_AVAILABLE_SUBSCRIPTION_MODES: LazyLock<AclModeSet> =
     LazyLock::new(AclModeSet::empty);
diff --git a/src/common/src/array/arrow/arrow_default.rs b/src/common/src/array/arrow/arrow_default.rs
deleted file mode 100644
index 5d04527b354ba..0000000000000
--- a/src/common/src/array/arrow/arrow_default.rs
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! This is for arrow dependency named `arrow-xxx` such as `arrow-array` in the cargo workspace.
-//!
-//! This should the default arrow version to be used in our system.
-//!
-//! The corresponding version of arrow is currently used by `udf` and `iceberg` sink.
-
-#![allow(unused_imports)]
-pub use arrow_impl::{
-    to_record_batch_with_schema, ToArrowArrayConvert, ToArrowArrayWithTypeConvert,
-    ToArrowTypeConvert,
-};
-use {arrow_array, arrow_buffer, arrow_cast, arrow_schema};
-
-#[expect(clippy::duplicate_mod)]
-#[path = "./arrow_impl.rs"]
-mod arrow_impl;
diff --git a/src/common/src/array/arrow/arrow_deltalake.rs b/src/common/src/array/arrow/arrow_deltalake.rs
index c55cae305b07f..c9f4052e2036d 100644
--- a/src/common/src/array/arrow/arrow_deltalake.rs
+++ b/src/common/src/array/arrow/arrow_deltalake.rs
@@ -21,25 +21,29 @@ use std::ops::{Div, Mul};
 use std::sync::Arc;
 
 use arrow_array::ArrayRef;
-use arrow_schema::DataType;
-use itertools::Itertools;
 use num_traits::abs;
 use {
     arrow_array_deltalake as arrow_array, arrow_buffer_deltalake as arrow_buffer,
     arrow_cast_deltalake as arrow_cast, arrow_schema_deltalake as arrow_schema,
 };
 
-use self::arrow_impl::ToArrowArrayWithTypeConvert;
-use crate::array::arrow::arrow_deltalake::arrow_impl::FromIntoArrow;
-use crate::array::{Array, ArrayError, ArrayImpl, DataChunk, Decimal, DecimalArray, ListArray};
-use crate::util::iter_util::ZipEqFast;
+use self::arrow_impl::ToArrow;
+use crate::array::{Array, ArrayError, DataChunk, Decimal, DecimalArray};
 #[expect(clippy::duplicate_mod)]
 #[path = "./arrow_impl.rs"]
 mod arrow_impl;
 
-struct DeltaLakeConvert;
+pub struct DeltaLakeConvert;
 
 impl DeltaLakeConvert {
+    pub fn to_record_batch(
+        &self,
+        schema: arrow_schema::SchemaRef,
+        chunk: &DataChunk,
+    ) -> Result<arrow_array::RecordBatch, ArrayError> {
+        ToArrow::to_record_batch(self, schema, chunk)
+    }
+
     fn decimal_to_i128(decimal: Decimal, precision: u8, max_scale: i8) -> Option<i128> {
         match decimal {
             crate::array::Decimal::Normalized(e) => {
@@ -67,7 +71,7 @@ impl DeltaLakeConvert {
     }
 }
 
-impl arrow_impl::ToArrowArrayWithTypeConvert for DeltaLakeConvert {
+impl ToArrow for DeltaLakeConvert {
     fn decimal_to_arrow(
         &self,
         data_type: &arrow_schema::DataType,
@@ -89,189 +93,6 @@ impl arrow_impl::ToArrowArrayWithTypeConvert for DeltaLakeConvert {
             .map_err(ArrayError::from_arrow)?;
         Ok(Arc::new(array) as ArrayRef)
     }
-
-    #[inline]
-    fn list_to_arrow(
-        &self,
-        data_type: &arrow_schema::DataType,
-        array: &ListArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        use arrow_array::builder::*;
-        fn build<A, B, F>(
-            array: &ListArray,
-            a: &A,
-            builder: B,
-            mut append: F,
-        ) -> arrow_array::ListArray
-        where
-            A: Array,
-            B: arrow_array::builder::ArrayBuilder,
-            F: FnMut(&mut B, Option<A::RefItem<'_>>),
-        {
-            let mut builder = ListBuilder::with_capacity(builder, a.len());
-            for i in 0..array.len() {
-                for j in array.offsets[i]..array.offsets[i + 1] {
-                    append(builder.values(), a.value_at(j as usize));
-                }
-                builder.append(!array.is_null(i));
-            }
-            builder.finish()
-        }
-        let inner_type = match data_type {
-            arrow_schema::DataType::List(inner) => inner.data_type(),
-            _ => return Err(ArrayError::to_arrow("Invalid list type")),
-        };
-        let arr: arrow_array::ListArray = match &*array.value {
-            ArrayImpl::Int16(a) => build(array, a, Int16Builder::with_capacity(a.len()), |b, v| {
-                b.append_option(v)
-            }),
-            ArrayImpl::Int32(a) => build(array, a, Int32Builder::with_capacity(a.len()), |b, v| {
-                b.append_option(v)
-            }),
-            ArrayImpl::Int64(a) => build(array, a, Int64Builder::with_capacity(a.len()), |b, v| {
-                b.append_option(v)
-            }),
-
-            ArrayImpl::Float32(a) => {
-                build(array, a, Float32Builder::with_capacity(a.len()), |b, v| {
-                    b.append_option(v.map(|f| f.0))
-                })
-            }
-            ArrayImpl::Float64(a) => {
-                build(array, a, Float64Builder::with_capacity(a.len()), |b, v| {
-                    b.append_option(v.map(|f| f.0))
-                })
-            }
-            ArrayImpl::Utf8(a) => build(
-                array,
-                a,
-                StringBuilder::with_capacity(a.len(), a.data().len()),
-                |b, v| b.append_option(v),
-            ),
-            ArrayImpl::Int256(a) => build(
-                array,
-                a,
-                Decimal256Builder::with_capacity(a.len()).with_data_type(
-                    arrow_schema::DataType::Decimal256(arrow_schema::DECIMAL256_MAX_PRECISION, 0),
-                ),
-                |b, v| b.append_option(v.map(Into::into)),
-            ),
-            ArrayImpl::Bool(a) => {
-                build(array, a, BooleanBuilder::with_capacity(a.len()), |b, v| {
-                    b.append_option(v)
-                })
-            }
-            ArrayImpl::Decimal(a) => {
-                let (precision, max_scale) = match inner_type {
-                    arrow_schema::DataType::Decimal128(precision, scale) => (*precision, *scale),
-                    _ => return Err(ArrayError::to_arrow("Invalid decimal type")),
-                };
-                build(
-                    array,
-                    a,
-                    Decimal128Builder::with_capacity(a.len())
-                        .with_data_type(DataType::Decimal128(precision, max_scale)),
-                    |b, v| {
-                        let v = v.and_then(|v| {
-                            DeltaLakeConvert::decimal_to_i128(v, precision, max_scale)
-                        });
-                        b.append_option(v);
-                    },
-                )
-            }
-            ArrayImpl::Interval(a) => build(
-                array,
-                a,
-                IntervalMonthDayNanoBuilder::with_capacity(a.len()),
-                |b, v| b.append_option(v.map(|d| d.into_arrow())),
-            ),
-            ArrayImpl::Date(a) => build(array, a, Date32Builder::with_capacity(a.len()), |b, v| {
-                b.append_option(v.map(|d| d.into_arrow()))
-            }),
-            ArrayImpl::Timestamp(a) => build(
-                array,
-                a,
-                TimestampMicrosecondBuilder::with_capacity(a.len()),
-                |b, v| b.append_option(v.map(|d| d.into_arrow())),
-            ),
-            ArrayImpl::Timestamptz(a) => build(
-                array,
-                a,
-                TimestampMicrosecondBuilder::with_capacity(a.len()),
-                |b, v| b.append_option(v.map(|d| d.into_arrow())),
-            ),
-            ArrayImpl::Time(a) => build(
-                array,
-                a,
-                Time64MicrosecondBuilder::with_capacity(a.len()),
-                |b, v| b.append_option(v.map(|d| d.into_arrow())),
-            ),
-            ArrayImpl::Jsonb(a) => build(
-                array,
-                a,
-                LargeStringBuilder::with_capacity(a.len(), a.len() * 16),
-                |b, v| b.append_option(v.map(|j| j.to_string())),
-            ),
-            ArrayImpl::Serial(_) => todo!("list of serial"),
-            ArrayImpl::Struct(a) => {
-                let values = Arc::new(arrow_array::StructArray::try_from(a)?);
-                arrow_array::ListArray::new(
-                    Arc::new(arrow_schema::Field::new(
-                        "item",
-                        a.data_type().try_into()?,
-                        true,
-                    )),
-                    arrow_buffer::OffsetBuffer::new(arrow_buffer::ScalarBuffer::from(
-                        array
-                            .offsets()
-                            .iter()
-                            .map(|o| *o as i32)
-                            .collect::<Vec<i32>>(),
-                    )),
-                    values,
-                    Some(array.null_bitmap().into()),
-                )
-            }
-            ArrayImpl::List(_) => todo!("list of list"),
-            ArrayImpl::Bytea(a) => build(
-                array,
-                a,
-                BinaryBuilder::with_capacity(a.len(), a.data().len()),
-                |b, v| b.append_option(v),
-            ),
-        };
-        Ok(Arc::new(arr))
-    }
-}
-
-/// Converts RisingWave array to Arrow array with the schema.
-/// This function will try to convert the array if the type is not same with the schema.
-pub fn to_deltalake_record_batch_with_schema(
-    schema: arrow_schema::SchemaRef,
-    chunk: &DataChunk,
-) -> Result<arrow_array::RecordBatch, ArrayError> {
-    if !chunk.is_compacted() {
-        let c = chunk.clone();
-        return to_deltalake_record_batch_with_schema(schema, &c.compact());
-    }
-    let columns: Vec<_> = chunk
-        .columns()
-        .iter()
-        .zip_eq_fast(schema.fields().iter())
-        .map(|(column, field)| {
-            let column: arrow_array::ArrayRef =
-                DeltaLakeConvert.to_arrow_with_type(field.data_type(), column)?;
-            if column.data_type() == field.data_type() {
-                Ok(column)
-            } else {
-                arrow_cast::cast(&column, field.data_type()).map_err(ArrayError::from_arrow)
-            }
-        })
-        .try_collect::<_, _, ArrayError>()?;
-
-    let opts = arrow_array::RecordBatchOptions::default().with_row_count(Some(chunk.capacity()));
-    arrow_array::RecordBatch::try_new_with_options(schema, columns, &opts)
-        .map_err(ArrayError::to_arrow)
 }
 
 #[cfg(test)]
@@ -283,6 +104,7 @@ mod test {
     use arrow_schema::Field;
     use {arrow_array_deltalake as arrow_array, arrow_schema_deltalake as arrow_schema};
 
+    use crate::array::arrow::arrow_deltalake::DeltaLakeConvert;
     use crate::array::{ArrayImpl, Decimal, DecimalArray, ListArray, ListValue};
     use crate::buffer::Bitmap;
 
@@ -304,13 +126,14 @@ mod test {
             arrow_schema::DataType::List(Arc::new(Field::new(
                 "test",
                 arrow_schema::DataType::Decimal128(10, 0),
-                false,
+                true,
             ))),
             false,
         )]);
 
-        let record_batch =
-            super::to_deltalake_record_batch_with_schema(Arc::new(schema), &chunk).unwrap();
+        let record_batch = DeltaLakeConvert
+            .to_record_batch(Arc::new(schema), &chunk)
+            .unwrap();
         let expect_array = Arc::new(
             arrow_array::Decimal128Array::from(vec![
                 None,
diff --git a/src/common/src/array/arrow/arrow_iceberg.rs b/src/common/src/array/arrow/arrow_iceberg.rs
index 2dd7900da5da1..72a49cb349370 100644
--- a/src/common/src/array/arrow/arrow_iceberg.rs
+++ b/src/common/src/array/arrow/arrow_iceberg.rs
@@ -15,25 +15,22 @@
 use std::ops::{Div, Mul};
 use std::sync::Arc;
 
-use arrow_array::{ArrayRef, StructArray};
-use arrow_schema::DataType;
-use itertools::Itertools;
+use arrow_array::ArrayRef;
 use num_traits::abs;
 
-use super::{ToArrowArrayWithTypeConvert, ToArrowTypeConvert};
-use crate::array::{Array, ArrayError, DataChunk, DecimalArray};
-use crate::util::iter_util::ZipEqFast;
+use super::{FromArrow, ToArrow};
+use crate::array::{Array, ArrayError, DecimalArray};
 
-struct IcebergArrowConvert;
+pub struct IcebergArrowConvert;
 
-impl ToArrowTypeConvert for IcebergArrowConvert {
+impl ToArrow for IcebergArrowConvert {
     #[inline]
-    fn decimal_type_to_arrow(&self) -> arrow_schema::DataType {
-        arrow_schema::DataType::Decimal128(arrow_schema::DECIMAL128_MAX_PRECISION, 0)
+    fn decimal_type_to_arrow(&self, name: &str) -> arrow_schema::Field {
+        let data_type =
+            arrow_schema::DataType::Decimal128(arrow_schema::DECIMAL128_MAX_PRECISION, 0);
+        arrow_schema::Field::new(name, data_type, true)
     }
-}
 
-impl ToArrowArrayWithTypeConvert for IcebergArrowConvert {
     fn decimal_to_arrow(
         &self,
         data_type: &arrow_schema::DataType,
@@ -85,63 +82,7 @@ impl ToArrowArrayWithTypeConvert for IcebergArrowConvert {
     }
 }
 
-/// Converts RisingWave array to Arrow array with the schema.
-/// The behavior is specified for iceberg:
-/// For different struct type, try to use fields in schema to cast.
-pub fn to_iceberg_record_batch_with_schema(
-    schema: arrow_schema::SchemaRef,
-    chunk: &DataChunk,
-) -> Result<arrow_array::RecordBatch, ArrayError> {
-    if !chunk.is_compacted() {
-        let c = chunk.clone();
-        return to_iceberg_record_batch_with_schema(schema, &c.compact());
-    }
-    let columns: Vec<_> = chunk
-        .columns()
-        .iter()
-        .zip_eq_fast(schema.fields().iter())
-        .map(|(column, field)| {
-            let column: arrow_array::ArrayRef =
-                IcebergArrowConvert {}.to_arrow_with_type(field.data_type(), column)?;
-            if column.data_type() == field.data_type() {
-                Ok(column)
-            } else if let DataType::Struct(actual) = column.data_type()
-                && let DataType::Struct(expect) = field.data_type()
-            {
-                // Special case for iceberg
-                if actual.len() != expect.len() {
-                    return Err(ArrayError::to_arrow(format!(
-                        "Struct field count mismatch, expect {}, actual {}",
-                        expect.len(),
-                        actual.len()
-                    )));
-                }
-                let column = column
-                    .as_any()
-                    .downcast_ref::<arrow_array::StructArray>()
-                    .unwrap()
-                    .clone();
-                let (_, struct_columns, nullable) = column.into_parts();
-                Ok(Arc::new(
-                    StructArray::try_new(expect.clone(), struct_columns, nullable)
-                        .map_err(ArrayError::from_arrow)?,
-                ) as ArrayRef)
-            } else {
-                arrow_cast::cast(&column, field.data_type()).map_err(ArrayError::from_arrow)
-            }
-        })
-        .try_collect::<_, _, ArrayError>()?;
-
-    let opts = arrow_array::RecordBatchOptions::default().with_row_count(Some(chunk.capacity()));
-    arrow_array::RecordBatch::try_new_with_options(schema, columns, &opts)
-        .map_err(ArrayError::to_arrow)
-}
-
-pub fn iceberg_to_arrow_type(
-    data_type: &crate::array::DataType,
-) -> Result<arrow_schema::DataType, ArrayError> {
-    IcebergArrowConvert {}.to_arrow_type(data_type)
-}
+impl FromArrow for IcebergArrowConvert {}
 
 #[cfg(test)]
 mod test {
@@ -150,7 +91,7 @@ mod test {
     use arrow_array::ArrayRef;
 
     use crate::array::arrow::arrow_iceberg::IcebergArrowConvert;
-    use crate::array::arrow::ToArrowArrayWithTypeConvert;
+    use crate::array::arrow::ToArrow;
     use crate::array::{Decimal, DecimalArray};
 
     #[test]
diff --git a/src/common/src/array/arrow/arrow_impl.rs b/src/common/src/array/arrow/arrow_impl.rs
index e426caa306d55..773efdd088e3d 100644
--- a/src/common/src/array/arrow/arrow_impl.rs
+++ b/src/common/src/array/arrow/arrow_impl.rs
@@ -36,9 +36,13 @@
 //! mod arrow_impl;
 //! ```
 
+// Is this a bug? Why do we have these lints?
+#![allow(unused_imports)]
+#![allow(dead_code)]
+
 use std::fmt::Write;
-use std::sync::Arc;
 
+use arrow_buffer::OffsetBuffer;
 use chrono::{NaiveDateTime, NaiveTime};
 use itertools::Itertools;
 
@@ -46,321 +50,79 @@ use itertools::Itertools;
 use super::{arrow_array, arrow_buffer, arrow_cast, arrow_schema};
 // Other import should always use the absolute path.
 use crate::array::*;
-use crate::buffer::Bitmap;
 use crate::types::*;
 use crate::util::iter_util::ZipEqFast;
 
-/// Converts RisingWave array to Arrow array with the schema.
-/// This function will try to convert the array if the type is not same with the schema.
-#[allow(dead_code)]
-pub fn to_record_batch_with_schema(
-    schema: arrow_schema::SchemaRef,
-    chunk: &DataChunk,
-) -> Result<arrow_array::RecordBatch, ArrayError> {
-    if !chunk.is_compacted() {
-        let c = chunk.clone();
-        return to_record_batch_with_schema(schema, &c.compact());
-    }
-    let columns: Vec<_> = chunk
-        .columns()
-        .iter()
-        .zip_eq_fast(schema.fields().iter())
-        .map(|(column, field)| {
-            let column: arrow_array::ArrayRef = column.as_ref().try_into()?;
-            if column.data_type() == field.data_type() {
-                Ok(column)
-            } else {
-                arrow_cast::cast(&column, field.data_type()).map_err(ArrayError::from_arrow)
-            }
-        })
-        .try_collect::<_, _, ArrayError>()?;
-
-    let opts = arrow_array::RecordBatchOptions::default().with_row_count(Some(chunk.capacity()));
-    arrow_array::RecordBatch::try_new_with_options(schema, columns, &opts)
-        .map_err(ArrayError::to_arrow)
-}
-
-// Implement bi-directional `From` between `DataChunk` and `arrow_array::RecordBatch`.
-impl TryFrom<&DataChunk> for arrow_array::RecordBatch {
-    type Error = ArrayError;
-
-    fn try_from(chunk: &DataChunk) -> Result<Self, Self::Error> {
+/// Defines how to convert RisingWave arrays to Arrow arrays.
+pub trait ToArrow {
+    /// Converts RisingWave `DataChunk` to Arrow `RecordBatch` with specified schema.
+    ///
+    /// This function will try to convert the array if the type is not same with the schema.
+    fn to_record_batch(
+        &self,
+        schema: arrow_schema::SchemaRef,
+        chunk: &DataChunk,
+    ) -> Result<arrow_array::RecordBatch, ArrayError> {
+        // compact the chunk if it's not compacted
         if !chunk.is_compacted() {
             let c = chunk.clone();
-            return Self::try_from(&c.compact());
+            return self.to_record_batch(schema, &c.compact());
         }
+
+        // convert each column to arrow array
         let columns: Vec<_> = chunk
             .columns()
             .iter()
-            .map(|column| column.as_ref().try_into())
-            .try_collect::<_, _, Self::Error>()?;
-
-        let fields: Vec<_> = columns
-            .iter()
-            .map(|array: &Arc<dyn arrow_array::Array>| {
-                let nullable = array.null_count() > 0;
-                let data_type = array.data_type().clone();
-                arrow_schema::Field::new("", data_type, nullable)
-            })
-            .collect();
+            .zip_eq_fast(schema.fields().iter())
+            .map(|(column, field)| self.to_array(field.data_type(), column))
+            .try_collect()?;
 
-        let schema = Arc::new(arrow_schema::Schema::new(fields));
+        // create record batch
         let opts =
             arrow_array::RecordBatchOptions::default().with_row_count(Some(chunk.capacity()));
         arrow_array::RecordBatch::try_new_with_options(schema, columns, &opts)
             .map_err(ArrayError::to_arrow)
     }
-}
-
-impl TryFrom<&arrow_array::RecordBatch> for DataChunk {
-    type Error = ArrayError;
 
-    fn try_from(batch: &arrow_array::RecordBatch) -> Result<Self, Self::Error> {
-        let mut columns = Vec::with_capacity(batch.num_columns());
-        for array in batch.columns() {
-            let column = Arc::new(array.try_into()?);
-            columns.push(column);
-        }
-        Ok(DataChunk::new(columns, batch.num_rows()))
-    }
-}
-
-/// Provides the default conversion logic for RisingWave array to Arrow array with type info.
-pub trait ToArrowArrayWithTypeConvert {
-    fn to_arrow_with_type(
+    /// Converts RisingWave array to Arrow array.
+    fn to_array(
         &self,
         data_type: &arrow_schema::DataType,
         array: &ArrayImpl,
     ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        match array {
-            ArrayImpl::Int16(array) => self.int16_to_arrow(data_type, array),
-            ArrayImpl::Int32(array) => self.int32_to_arrow(data_type, array),
-            ArrayImpl::Int64(array) => self.int64_to_arrow(data_type, array),
-            ArrayImpl::Float32(array) => self.float32_to_arrow(data_type, array),
-            ArrayImpl::Float64(array) => self.float64_to_arrow(data_type, array),
-            ArrayImpl::Utf8(array) => self.utf8_to_arrow(data_type, array),
-            ArrayImpl::Bool(array) => self.bool_to_arrow(data_type, array),
-            ArrayImpl::Decimal(array) => self.decimal_to_arrow(data_type, array),
-            ArrayImpl::Int256(array) => self.int256_to_arrow(data_type, array),
-            ArrayImpl::Date(array) => self.date_to_arrow(data_type, array),
-            ArrayImpl::Timestamp(array) => self.timestamp_to_arrow(data_type, array),
-            ArrayImpl::Timestamptz(array) => self.timestamptz_to_arrow(data_type, array),
-            ArrayImpl::Time(array) => self.time_to_arrow(data_type, array),
-            ArrayImpl::Interval(array) => self.interval_to_arrow(data_type, array),
-            ArrayImpl::Struct(array) => self.struct_to_arrow(data_type, array),
-            ArrayImpl::List(array) => self.list_to_arrow(data_type, array),
-            ArrayImpl::Bytea(array) => self.bytea_to_arrow(data_type, array),
-            ArrayImpl::Jsonb(array) => self.jsonb_to_arrow(data_type, array),
-            ArrayImpl::Serial(array) => self.serial_to_arrow(data_type, array),
-        }
-    }
-
-    #[inline]
-    fn int16_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &I16Array,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::Int16Array::from(array)))
-    }
-
-    #[inline]
-    fn int32_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &I32Array,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::Int32Array::from(array)))
-    }
-
-    #[inline]
-    fn int64_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &I64Array,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::Int64Array::from(array)))
-    }
-
-    #[inline]
-    fn float32_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &F32Array,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::Float32Array::from(array)))
-    }
-
-    #[inline]
-    fn float64_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &F64Array,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::Float64Array::from(array)))
-    }
-
-    #[inline]
-    fn utf8_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &Utf8Array,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::StringArray::from(array)))
-    }
-
-    #[inline]
-    fn bool_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &BoolArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::BooleanArray::from(array)))
-    }
-
-    // Decimal values are stored as ASCII text representation in a large binary array.
-    #[inline]
-    fn decimal_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &DecimalArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::LargeBinaryArray::from(array)))
-    }
-
-    #[inline]
-    fn int256_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &Int256Array,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::Decimal256Array::from(array)))
-    }
-
-    #[inline]
-    fn date_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &DateArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::Date32Array::from(array)))
-    }
-
-    #[inline]
-    fn timestamp_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &TimestampArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::TimestampMicrosecondArray::from(
-            array,
-        )))
-    }
-
-    #[inline]
-    fn timestamptz_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &TimestamptzArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(
-            arrow_array::TimestampMicrosecondArray::from(array).with_timezone_utc(),
-        ))
-    }
-
-    #[inline]
-    fn time_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &TimeArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::Time64MicrosecondArray::from(array)))
-    }
-
-    #[inline]
-    fn interval_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &IntervalArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::IntervalMonthDayNanoArray::from(
-            array,
-        )))
-    }
-
-    #[inline]
-    fn struct_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &StructArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::StructArray::try_from(array)?))
-    }
-
-    #[inline]
-    fn list_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &ListArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::ListArray::try_from(array)?))
-    }
-
-    #[inline]
-    fn bytea_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &BytesArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::BinaryArray::from(array)))
-    }
-
-    // JSON values are stored as text representation in a large string array.
-    #[inline]
-    fn jsonb_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        array: &JsonbArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::LargeStringArray::from(array)))
-    }
-
-    #[inline]
-    fn serial_to_arrow(
-        &self,
-        _data_type: &arrow_schema::DataType,
-        _array: &SerialArray,
-    ) -> Result<arrow_array::ArrayRef, ArrayError> {
-        todo!("serial type is not supported to convert to arrow")
-    }
-}
-
-/// Provides the default conversion logic for RisingWave array to Arrow array with type info.
-pub trait ToArrowArrayConvert {
-    fn to_arrow(&self, array: &ArrayImpl) -> Result<arrow_array::ArrayRef, ArrayError> {
-        match array {
+        let arrow_array = match array {
+            ArrayImpl::Bool(array) => self.bool_to_arrow(array),
             ArrayImpl::Int16(array) => self.int16_to_arrow(array),
             ArrayImpl::Int32(array) => self.int32_to_arrow(array),
             ArrayImpl::Int64(array) => self.int64_to_arrow(array),
+            ArrayImpl::Int256(array) => self.int256_to_arrow(array),
             ArrayImpl::Float32(array) => self.float32_to_arrow(array),
             ArrayImpl::Float64(array) => self.float64_to_arrow(array),
-            ArrayImpl::Utf8(array) => self.utf8_to_arrow(array),
-            ArrayImpl::Bool(array) => self.bool_to_arrow(array),
-            ArrayImpl::Decimal(array) => self.decimal_to_arrow(array),
-            ArrayImpl::Int256(array) => self.int256_to_arrow(array),
             ArrayImpl::Date(array) => self.date_to_arrow(array),
+            ArrayImpl::Time(array) => self.time_to_arrow(array),
             ArrayImpl::Timestamp(array) => self.timestamp_to_arrow(array),
             ArrayImpl::Timestamptz(array) => self.timestamptz_to_arrow(array),
-            ArrayImpl::Time(array) => self.time_to_arrow(array),
             ArrayImpl::Interval(array) => self.interval_to_arrow(array),
-            ArrayImpl::Struct(array) => self.struct_to_arrow(array),
-            ArrayImpl::List(array) => self.list_to_arrow(array),
+            ArrayImpl::Utf8(array) => self.utf8_to_arrow(array),
             ArrayImpl::Bytea(array) => self.bytea_to_arrow(array),
+            ArrayImpl::Decimal(array) => self.decimal_to_arrow(data_type, array),
             ArrayImpl::Jsonb(array) => self.jsonb_to_arrow(array),
             ArrayImpl::Serial(array) => self.serial_to_arrow(array),
+            ArrayImpl::List(array) => self.list_to_arrow(data_type, array),
+            ArrayImpl::Struct(array) => self.struct_to_arrow(data_type, array),
+        }?;
+        if arrow_array.data_type() != data_type {
+            arrow_cast::cast(&arrow_array, data_type).map_err(ArrayError::to_arrow)
+        } else {
+            Ok(arrow_array)
         }
     }
 
+    #[inline]
+    fn bool_to_arrow(&self, array: &BoolArray) -> Result<arrow_array::ArrayRef, ArrayError> {
+        Ok(Arc::new(arrow_array::BooleanArray::from(array)))
+    }
+
     #[inline]
     fn int16_to_arrow(&self, array: &I16Array) -> Result<arrow_array::ArrayRef, ArrayError> {
         Ok(Arc::new(arrow_array::Int16Array::from(array)))
@@ -391,17 +153,6 @@ pub trait ToArrowArrayConvert {
         Ok(Arc::new(arrow_array::StringArray::from(array)))
     }
 
-    #[inline]
-    fn bool_to_arrow(&self, array: &BoolArray) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::BooleanArray::from(array)))
-    }
-
-    // Decimal values are stored as ASCII text representation in a large binary array.
-    #[inline]
-    fn decimal_to_arrow(&self, array: &DecimalArray) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::LargeBinaryArray::from(array)))
-    }
-
     #[inline]
     fn int256_to_arrow(&self, array: &Int256Array) -> Result<arrow_array::ArrayRef, ArrayError> {
         Ok(Arc::new(arrow_array::Decimal256Array::from(array)))
@@ -448,56 +199,103 @@ pub trait ToArrowArrayConvert {
     }
 
     #[inline]
-    fn struct_to_arrow(&self, array: &StructArray) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::StructArray::try_from(array)?))
+    fn bytea_to_arrow(&self, array: &BytesArray) -> Result<arrow_array::ArrayRef, ArrayError> {
+        Ok(Arc::new(arrow_array::BinaryArray::from(array)))
     }
 
+    // Decimal values are stored as ASCII text representation in a string array.
     #[inline]
-    fn list_to_arrow(&self, array: &ListArray) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::ListArray::try_from(array)?))
+    fn decimal_to_arrow(
+        &self,
+        _data_type: &arrow_schema::DataType,
+        array: &DecimalArray,
+    ) -> Result<arrow_array::ArrayRef, ArrayError> {
+        Ok(Arc::new(arrow_array::StringArray::from(array)))
     }
 
+    // JSON values are stored as text representation in a string array.
     #[inline]
-    fn bytea_to_arrow(&self, array: &BytesArray) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::BinaryArray::from(array)))
+    fn jsonb_to_arrow(&self, array: &JsonbArray) -> Result<arrow_array::ArrayRef, ArrayError> {
+        Ok(Arc::new(arrow_array::StringArray::from(array)))
     }
 
-    // JSON values are stored as text representation in a large string array.
     #[inline]
-    fn jsonb_to_arrow(&self, array: &JsonbArray) -> Result<arrow_array::ArrayRef, ArrayError> {
-        Ok(Arc::new(arrow_array::LargeStringArray::from(array)))
+    fn serial_to_arrow(&self, array: &SerialArray) -> Result<arrow_array::ArrayRef, ArrayError> {
+        Ok(Arc::new(arrow_array::Int64Array::from(array)))
     }
 
     #[inline]
-    fn serial_to_arrow(&self, _array: &SerialArray) -> Result<arrow_array::ArrayRef, ArrayError> {
-        todo!("serial type is not supported to convert to arrow")
+    fn list_to_arrow(
+        &self,
+        data_type: &arrow_schema::DataType,
+        array: &ListArray,
+    ) -> Result<arrow_array::ArrayRef, ArrayError> {
+        let arrow_schema::DataType::List(field) = data_type else {
+            return Err(ArrayError::to_arrow("Invalid list type"));
+        };
+        let values = self.to_array(field.data_type(), array.values())?;
+        let offsets = OffsetBuffer::new(array.offsets().iter().map(|&o| o as i32).collect());
+        let nulls = (!array.null_bitmap().all()).then(|| array.null_bitmap().into());
+        Ok(Arc::new(arrow_array::ListArray::new(
+            field.clone(),
+            offsets,
+            values,
+            nulls,
+        )))
     }
-}
 
-pub trait ToArrowTypeConvert {
-    fn to_arrow_type(&self, value: &DataType) -> Result<arrow_schema::DataType, ArrayError> {
-        match value {
+    #[inline]
+    fn struct_to_arrow(
+        &self,
+        data_type: &arrow_schema::DataType,
+        array: &StructArray,
+    ) -> Result<arrow_array::ArrayRef, ArrayError> {
+        let arrow_schema::DataType::Struct(fields) = data_type else {
+            return Err(ArrayError::to_arrow("Invalid struct type"));
+        };
+        Ok(Arc::new(arrow_array::StructArray::new(
+            fields.clone(),
+            array
+                .fields()
+                .zip_eq_fast(fields)
+                .map(|(arr, field)| self.to_array(field.data_type(), arr))
+                .try_collect::<_, _, ArrayError>()?,
+            Some(array.null_bitmap().into()),
+        )))
+    }
+
+    /// Convert RisingWave data type to Arrow data type.
+    ///
+    /// This function returns a `Field` instead of `DataType` because some may be converted to
+    /// extension types which require additional metadata in the field.
+    fn to_arrow_field(
+        &self,
+        name: &str,
+        value: &DataType,
+    ) -> Result<arrow_schema::Field, ArrayError> {
+        let data_type = match value {
             // using the inline function
-            DataType::Boolean => Ok(self.bool_type_to_arrow()),
-            DataType::Int16 => Ok(self.int16_type_to_arrow()),
-            DataType::Int32 => Ok(self.int32_type_to_arrow()),
-            DataType::Int64 => Ok(self.int64_type_to_arrow()),
-            DataType::Int256 => Ok(self.int256_type_to_arrow()),
-            DataType::Float32 => Ok(self.float32_type_to_arrow()),
-            DataType::Float64 => Ok(self.float64_type_to_arrow()),
-            DataType::Date => Ok(self.date_type_to_arrow()),
-            DataType::Timestamp => Ok(self.timestamp_type_to_arrow()),
-            DataType::Timestamptz => Ok(self.timestamptz_type_to_arrow()),
-            DataType::Time => Ok(self.time_type_to_arrow()),
-            DataType::Interval => Ok(self.interval_type_to_arrow()),
-            DataType::Varchar => Ok(self.varchar_type_to_arrow()),
-            DataType::Jsonb => Ok(self.jsonb_type_to_arrow()),
-            DataType::Bytea => Ok(self.bytea_type_to_arrow()),
-            DataType::Decimal => Ok(self.decimal_type_to_arrow()),
-            DataType::Serial => Ok(self.serial_type_to_arrow()),
-            DataType::Struct(fields) => self.struct_type_to_arrow(fields),
-            DataType::List(datatype) => self.list_type_to_arrow(datatype),
-        }
+            DataType::Boolean => self.bool_type_to_arrow(),
+            DataType::Int16 => self.int16_type_to_arrow(),
+            DataType::Int32 => self.int32_type_to_arrow(),
+            DataType::Int64 => self.int64_type_to_arrow(),
+            DataType::Int256 => self.int256_type_to_arrow(),
+            DataType::Float32 => self.float32_type_to_arrow(),
+            DataType::Float64 => self.float64_type_to_arrow(),
+            DataType::Date => self.date_type_to_arrow(),
+            DataType::Time => self.time_type_to_arrow(),
+            DataType::Timestamp => self.timestamp_type_to_arrow(),
+            DataType::Timestamptz => self.timestamptz_type_to_arrow(),
+            DataType::Interval => self.interval_type_to_arrow(),
+            DataType::Varchar => self.varchar_type_to_arrow(),
+            DataType::Bytea => self.bytea_type_to_arrow(),
+            DataType::Serial => self.serial_type_to_arrow(),
+            DataType::Decimal => return Ok(self.decimal_type_to_arrow(name)),
+            DataType::Jsonb => return Ok(self.jsonb_type_to_arrow(name)),
+            DataType::Struct(fields) => self.struct_type_to_arrow(fields)?,
+            DataType::List(datatype) => self.list_type_to_arrow(datatype)?,
+        };
+        Ok(arrow_schema::Field::new(name, data_type, true))
     }
 
     #[inline]
@@ -505,6 +303,11 @@ pub trait ToArrowTypeConvert {
         arrow_schema::DataType::Boolean
     }
 
+    #[inline]
+    fn int16_type_to_arrow(&self) -> arrow_schema::DataType {
+        arrow_schema::DataType::Int16
+    }
+
     #[inline]
     fn int32_type_to_arrow(&self) -> arrow_schema::DataType {
         arrow_schema::DataType::Int32
@@ -515,12 +318,6 @@ pub trait ToArrowTypeConvert {
         arrow_schema::DataType::Int64
     }
 
-    // generate function for each type for me using inline
-    #[inline]
-    fn int16_type_to_arrow(&self) -> arrow_schema::DataType {
-        arrow_schema::DataType::Int16
-    }
-
     #[inline]
     fn int256_type_to_arrow(&self) -> arrow_schema::DataType {
         arrow_schema::DataType::Decimal256(arrow_schema::DECIMAL256_MAX_PRECISION, 0)
@@ -541,6 +338,11 @@ pub trait ToArrowTypeConvert {
         arrow_schema::DataType::Date32
     }
 
+    #[inline]
+    fn time_type_to_arrow(&self) -> arrow_schema::DataType {
+        arrow_schema::DataType::Time64(arrow_schema::TimeUnit::Microsecond)
+    }
+
     #[inline]
     fn timestamp_type_to_arrow(&self) -> arrow_schema::DataType {
         arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Microsecond, None)
@@ -550,13 +352,8 @@ pub trait ToArrowTypeConvert {
     fn timestamptz_type_to_arrow(&self) -> arrow_schema::DataType {
         arrow_schema::DataType::Timestamp(
             arrow_schema::TimeUnit::Microsecond,
-            Some("+00:00".into()),
-        )
-    }
-
-    #[inline]
-    fn time_type_to_arrow(&self) -> arrow_schema::DataType {
-        arrow_schema::DataType::Time64(arrow_schema::TimeUnit::Microsecond)
+            Some("+00:00".into()),
+        )
     }
 
     #[inline]
@@ -570,8 +367,9 @@ pub trait ToArrowTypeConvert {
     }
 
     #[inline]
-    fn jsonb_type_to_arrow(&self) -> arrow_schema::DataType {
-        arrow_schema::DataType::LargeUtf8
+    fn jsonb_type_to_arrow(&self, name: &str) -> arrow_schema::Field {
+        arrow_schema::Field::new(name, arrow_schema::DataType::Utf8, true)
+            .with_metadata([("ARROW:extension:name".into(), "arrowudf.json".into())].into())
     }
 
     #[inline]
@@ -580,22 +378,23 @@ pub trait ToArrowTypeConvert {
     }
 
     #[inline]
-    fn decimal_type_to_arrow(&self) -> arrow_schema::DataType {
-        arrow_schema::DataType::LargeBinary
+    fn decimal_type_to_arrow(&self, name: &str) -> arrow_schema::Field {
+        arrow_schema::Field::new(name, arrow_schema::DataType::Utf8, true)
+            .with_metadata([("ARROW:extension:name".into(), "arrowudf.decimal".into())].into())
     }
 
     #[inline]
     fn serial_type_to_arrow(&self) -> arrow_schema::DataType {
-        todo!("serial type is not supported to convert to arrow")
+        arrow_schema::DataType::Int64
     }
 
     #[inline]
     fn list_type_to_arrow(
         &self,
-        datatype: &DataType,
+        elem_type: &DataType,
     ) -> Result<arrow_schema::DataType, ArrayError> {
         Ok(arrow_schema::DataType::List(Arc::new(
-            arrow_schema::Field::new("item", datatype.try_into()?, true),
+            self.to_arrow_field("item", elem_type)?,
         )))
     }
 
@@ -607,160 +406,287 @@ pub trait ToArrowTypeConvert {
         Ok(arrow_schema::DataType::Struct(
             fields
                 .iter()
-                .map(|(name, ty)| Ok(arrow_schema::Field::new(name, ty.try_into()?, true)))
+                .map(|(name, ty)| self.to_arrow_field(name, ty))
                 .try_collect::<_, _, ArrayError>()?,
         ))
     }
 }
 
-struct DefaultArrowConvert;
-impl ToArrowArrayConvert for DefaultArrowConvert {}
-
-/// Implement bi-directional `From` between `ArrayImpl` and `arrow_array::ArrayRef`.
-macro_rules! converts_generic {
-    ($({ $ArrowType:ty, $ArrowPattern:pat, $ArrayImplPattern:path }),*) => {
-        // RisingWave array -> Arrow array
-        impl TryFrom<&ArrayImpl> for arrow_array::ArrayRef {
-            type Error = ArrayError;
-            fn try_from(array: &ArrayImpl) -> Result<Self, Self::Error> {
-                DefaultArrowConvert{}.to_arrow(array)
-            }
+/// Defines how to convert Arrow arrays to RisingWave arrays.
+#[allow(clippy::wrong_self_convention)]
+pub trait FromArrow {
+    /// Converts Arrow `RecordBatch` to RisingWave `DataChunk`.
+    fn from_record_batch(&self, batch: &arrow_array::RecordBatch) -> Result<DataChunk, ArrayError> {
+        let mut columns = Vec::with_capacity(batch.num_columns());
+        for (array, field) in batch.columns().iter().zip_eq_fast(batch.schema().fields()) {
+            let column = Arc::new(self.from_array(field, array)?);
+            columns.push(column);
         }
-        // Arrow array -> RisingWave array
-        impl TryFrom<&arrow_array::ArrayRef> for ArrayImpl {
-            type Error = ArrayError;
-            fn try_from(array: &arrow_array::ArrayRef) -> Result<Self, Self::Error> {
-                use arrow_schema::DataType::*;
-                use arrow_schema::IntervalUnit::*;
-                use arrow_schema::TimeUnit::*;
-                match array.data_type() {
-                    $($ArrowPattern => Ok($ArrayImplPattern(
-                        array
-                            .as_any()
-                            .downcast_ref::<$ArrowType>()
-                            .unwrap()
-                            .try_into()?,
-                    )),)*
-                    Timestamp(Microsecond, Some(_)) => Ok(ArrayImpl::Timestamptz(
-                        array
-                            .as_any()
-                            .downcast_ref::<arrow_array::TimestampMicrosecondArray>()
-                            .unwrap()
-                            .try_into()?,
-                    )),
-                    // This arrow decimal type is used by iceberg source to read iceberg decimal into RW decimal.
-                    Decimal128(_, _) => Ok(ArrayImpl::Decimal(
-                        array
-                            .as_any()
-                            .downcast_ref::<arrow_array::Decimal128Array>()
-                            .unwrap()
-                            .try_into()?,
-                    )),
-                    t => Err(ArrayError::from_arrow(format!("unsupported data type: {t:?}"))),
-                }
+        Ok(DataChunk::new(columns, batch.num_rows()))
+    }
+
+    /// Converts Arrow `Fields` to RisingWave `StructType`.
+    fn from_fields(&self, fields: &arrow_schema::Fields) -> Result<StructType, ArrayError> {
+        Ok(StructType::new(
+            fields
+                .iter()
+                .map(|f| Ok((f.name().clone(), self.from_field(f)?)))
+                .try_collect::<_, _, ArrayError>()?,
+        ))
+    }
+
+    /// Converts Arrow `Field` to RisingWave `DataType`.
+    fn from_field(&self, field: &arrow_schema::Field) -> Result<DataType, ArrayError> {
+        use arrow_schema::DataType::*;
+        use arrow_schema::IntervalUnit::*;
+        use arrow_schema::TimeUnit::*;
+
+        // extension type
+        if let Some(type_name) = field.metadata().get("ARROW:extension:name") {
+            return self.from_extension_type(type_name, field.data_type());
+        }
+
+        Ok(match field.data_type() {
+            Boolean => DataType::Boolean,
+            Int16 => DataType::Int16,
+            Int32 => DataType::Int32,
+            Int64 => DataType::Int64,
+            Float32 => DataType::Float32,
+            Float64 => DataType::Float64,
+            Decimal128(_, _) => DataType::Decimal,
+            Decimal256(_, _) => DataType::Int256,
+            Date32 => DataType::Date,
+            Time64(Microsecond) => DataType::Time,
+            Timestamp(Microsecond, None) => DataType::Timestamp,
+            Timestamp(Microsecond, Some(_)) => DataType::Timestamptz,
+            Interval(MonthDayNano) => DataType::Interval,
+            Utf8 => DataType::Varchar,
+            Binary => DataType::Bytea,
+            LargeUtf8 => self.from_large_utf8()?,
+            LargeBinary => self.from_large_binary()?,
+            List(field) => DataType::List(Box::new(self.from_field(field)?)),
+            Struct(fields) => DataType::Struct(self.from_fields(fields)?),
+            t => {
+                return Err(ArrayError::from_arrow(format!(
+                    "unsupported arrow data type: {t:?}"
+                )))
             }
+        })
+    }
+
+    /// Converts Arrow `LargeUtf8` type to RisingWave data type.
+    fn from_large_utf8(&self) -> Result<DataType, ArrayError> {
+        Ok(DataType::Varchar)
+    }
+
+    /// Converts Arrow `LargeBinary` type to RisingWave data type.
+    fn from_large_binary(&self) -> Result<DataType, ArrayError> {
+        Ok(DataType::Bytea)
+    }
+
+    /// Converts Arrow extension type to RisingWave `DataType`.
+    fn from_extension_type(
+        &self,
+        type_name: &str,
+        physical_type: &arrow_schema::DataType,
+    ) -> Result<DataType, ArrayError> {
+        match (type_name, physical_type) {
+            ("arrowudf.decimal", arrow_schema::DataType::Utf8) => Ok(DataType::Decimal),
+            ("arrowudf.json", arrow_schema::DataType::Utf8) => Ok(DataType::Jsonb),
+            _ => Err(ArrayError::from_arrow(format!(
+                "unsupported extension type: {type_name:?}"
+            ))),
         }
-    };
-}
-converts_generic! {
-    { arrow_array::Int16Array, Int16, ArrayImpl::Int16 },
-    { arrow_array::Int32Array, Int32, ArrayImpl::Int32 },
-    { arrow_array::Int64Array, Int64, ArrayImpl::Int64 },
-    { arrow_array::Float32Array, Float32, ArrayImpl::Float32 },
-    { arrow_array::Float64Array, Float64, ArrayImpl::Float64 },
-    { arrow_array::StringArray, Utf8, ArrayImpl::Utf8 },
-    { arrow_array::BooleanArray, Boolean, ArrayImpl::Bool },
-    // Arrow doesn't have a data type to represent unconstrained numeric (`DECIMAL` in RisingWave and
-    // Postgres). So we pick a special type `LargeBinary` for it.
-    // Values stored in the array are the string representation of the decimal. e.g. b"1.234", b"+inf"
-    { arrow_array::LargeBinaryArray, LargeBinary, ArrayImpl::Decimal },
-    { arrow_array::Decimal256Array, Decimal256(_, _), ArrayImpl::Int256 },
-    { arrow_array::Date32Array, Date32, ArrayImpl::Date },
-    { arrow_array::TimestampMicrosecondArray, Timestamp(Microsecond, None), ArrayImpl::Timestamp },
-    { arrow_array::Time64MicrosecondArray, Time64(Microsecond), ArrayImpl::Time },
-    { arrow_array::IntervalMonthDayNanoArray, Interval(MonthDayNano), ArrayImpl::Interval },
-    { arrow_array::StructArray, Struct(_), ArrayImpl::Struct },
-    { arrow_array::ListArray, List(_), ArrayImpl::List },
-    { arrow_array::BinaryArray, Binary, ArrayImpl::Bytea },
-    { arrow_array::LargeStringArray, LargeUtf8, ArrayImpl::Jsonb }    // we use LargeUtf8 to represent Jsonb in arrow
-}
+    }
 
-// Arrow Datatype -> Risingwave Datatype
-impl From<&arrow_schema::DataType> for DataType {
-    fn from(value: &arrow_schema::DataType) -> Self {
+    /// Converts Arrow `Array` to RisingWave `ArrayImpl`.
+    fn from_array(
+        &self,
+        field: &arrow_schema::Field,
+        array: &arrow_array::ArrayRef,
+    ) -> Result<ArrayImpl, ArrayError> {
         use arrow_schema::DataType::*;
         use arrow_schema::IntervalUnit::*;
         use arrow_schema::TimeUnit::*;
-        match value {
-            Boolean => Self::Boolean,
-            Int16 => Self::Int16,
-            Int32 => Self::Int32,
-            Int64 => Self::Int64,
-            Float32 => Self::Float32,
-            Float64 => Self::Float64,
-            LargeBinary => Self::Decimal,
-            Decimal256(_, _) => Self::Int256,
-            Date32 => Self::Date,
-            Time64(Microsecond) => Self::Time,
-            Timestamp(Microsecond, None) => Self::Timestamp,
-            Timestamp(Microsecond, Some(_)) => Self::Timestamptz,
-            Interval(MonthDayNano) => Self::Interval,
-            Binary => Self::Bytea,
-            Utf8 => Self::Varchar,
-            LargeUtf8 => Self::Jsonb,
-            Struct(fields) => Self::Struct(fields.into()),
-            List(field) => Self::List(Box::new(field.data_type().into())),
-            Decimal128(_, _) => Self::Decimal,
-            _ => todo!("Unsupported arrow data type: {value:?}"),
+
+        // extension type
+        if let Some(type_name) = field.metadata().get("ARROW:extension:name") {
+            return self.from_extension_array(type_name, array);
+        }
+
+        match array.data_type() {
+            Boolean => self.from_bool_array(array.as_any().downcast_ref().unwrap()),
+            Int16 => self.from_int16_array(array.as_any().downcast_ref().unwrap()),
+            Int32 => self.from_int32_array(array.as_any().downcast_ref().unwrap()),
+            Int64 => self.from_int64_array(array.as_any().downcast_ref().unwrap()),
+            Decimal256(_, _) => self.from_int256_array(array.as_any().downcast_ref().unwrap()),
+            Float32 => self.from_float32_array(array.as_any().downcast_ref().unwrap()),
+            Float64 => self.from_float64_array(array.as_any().downcast_ref().unwrap()),
+            Date32 => self.from_date32_array(array.as_any().downcast_ref().unwrap()),
+            Time64(Microsecond) => self.from_time64us_array(array.as_any().downcast_ref().unwrap()),
+            Timestamp(Microsecond, _) => {
+                self.from_timestampus_array(array.as_any().downcast_ref().unwrap())
+            }
+            Interval(MonthDayNano) => {
+                self.from_interval_array(array.as_any().downcast_ref().unwrap())
+            }
+            Utf8 => self.from_utf8_array(array.as_any().downcast_ref().unwrap()),
+            Binary => self.from_binary_array(array.as_any().downcast_ref().unwrap()),
+            LargeUtf8 => self.from_large_utf8_array(array.as_any().downcast_ref().unwrap()),
+            LargeBinary => self.from_large_binary_array(array.as_any().downcast_ref().unwrap()),
+            List(_) => self.from_list_array(array.as_any().downcast_ref().unwrap()),
+            Struct(_) => self.from_struct_array(array.as_any().downcast_ref().unwrap()),
+            t => Err(ArrayError::from_arrow(format!(
+                "unsupported arrow data type: {t:?}",
+            ))),
         }
     }
-}
 
-impl From<&arrow_schema::Fields> for StructType {
-    fn from(fields: &arrow_schema::Fields) -> Self {
-        Self::new(
-            fields
-                .iter()
-                .map(|f| (f.name().clone(), f.data_type().into()))
-                .collect(),
-        )
+    /// Converts Arrow extension array to RisingWave `ArrayImpl`.
+    fn from_extension_array(
+        &self,
+        type_name: &str,
+        array: &arrow_array::ArrayRef,
+    ) -> Result<ArrayImpl, ArrayError> {
+        match type_name {
+            "arrowudf.decimal" => {
+                let array: &arrow_array::StringArray =
+                    array.as_any().downcast_ref().ok_or_else(|| {
+                        ArrayError::from_arrow(
+                            "expected string array for `arrowudf.decimal`".to_string(),
+                        )
+                    })?;
+                Ok(ArrayImpl::Decimal(array.try_into()?))
+            }
+            "arrowudf.json" => {
+                let array: &arrow_array::StringArray =
+                    array.as_any().downcast_ref().ok_or_else(|| {
+                        ArrayError::from_arrow(
+                            "expected string array for `arrowudf.json`".to_string(),
+                        )
+                    })?;
+                Ok(ArrayImpl::Jsonb(array.try_into()?))
+            }
+            _ => Err(ArrayError::from_arrow(format!(
+                "unsupported extension type: {type_name:?}"
+            ))),
+        }
     }
-}
 
-impl TryFrom<&StructType> for arrow_schema::Fields {
-    type Error = ArrayError;
+    fn from_bool_array(&self, array: &arrow_array::BooleanArray) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Bool(array.into()))
+    }
 
-    fn try_from(struct_type: &StructType) -> Result<Self, Self::Error> {
-        struct_type
-            .iter()
-            .map(|(name, ty)| Ok(arrow_schema::Field::new(name, ty.try_into()?, true)))
-            .try_collect()
+    fn from_int16_array(&self, array: &arrow_array::Int16Array) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Int16(array.into()))
     }
-}
 
-impl From<arrow_schema::DataType> for DataType {
-    fn from(value: arrow_schema::DataType) -> Self {
-        (&value).into()
+    fn from_int32_array(&self, array: &arrow_array::Int32Array) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Int32(array.into()))
     }
-}
 
-struct DefaultArrowTypeConvert;
+    fn from_int64_array(&self, array: &arrow_array::Int64Array) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Int64(array.into()))
+    }
 
-impl ToArrowTypeConvert for DefaultArrowTypeConvert {}
+    fn from_int256_array(
+        &self,
+        array: &arrow_array::Decimal256Array,
+    ) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Int256(array.into()))
+    }
 
-impl TryFrom<&DataType> for arrow_schema::DataType {
-    type Error = ArrayError;
+    fn from_float32_array(
+        &self,
+        array: &arrow_array::Float32Array,
+    ) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Float32(array.into()))
+    }
 
-    fn try_from(value: &DataType) -> Result<Self, Self::Error> {
-        DefaultArrowTypeConvert {}.to_arrow_type(value)
+    fn from_float64_array(
+        &self,
+        array: &arrow_array::Float64Array,
+    ) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Float64(array.into()))
     }
-}
 
-impl TryFrom<DataType> for arrow_schema::DataType {
-    type Error = ArrayError;
+    fn from_date32_array(&self, array: &arrow_array::Date32Array) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Date(array.into()))
+    }
+
+    fn from_time64us_array(
+        &self,
+        array: &arrow_array::Time64MicrosecondArray,
+    ) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Time(array.into()))
+    }
+
+    fn from_timestampus_array(
+        &self,
+        array: &arrow_array::TimestampMicrosecondArray,
+    ) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Timestamp(array.into()))
+    }
+
+    fn from_interval_array(
+        &self,
+        array: &arrow_array::IntervalMonthDayNanoArray,
+    ) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Interval(array.into()))
+    }
+
+    fn from_utf8_array(&self, array: &arrow_array::StringArray) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Utf8(array.into()))
+    }
+
+    fn from_binary_array(&self, array: &arrow_array::BinaryArray) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Bytea(array.into()))
+    }
+
+    fn from_large_utf8_array(
+        &self,
+        array: &arrow_array::LargeStringArray,
+    ) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Utf8(array.into()))
+    }
+
+    fn from_large_binary_array(
+        &self,
+        array: &arrow_array::LargeBinaryArray,
+    ) -> Result<ArrayImpl, ArrayError> {
+        Ok(ArrayImpl::Bytea(array.into()))
+    }
+
+    fn from_list_array(&self, array: &arrow_array::ListArray) -> Result<ArrayImpl, ArrayError> {
+        use arrow_array::Array;
+        let arrow_schema::DataType::List(field) = array.data_type() else {
+            panic!("nested field types cannot be determined.");
+        };
+        Ok(ArrayImpl::List(ListArray {
+            value: Box::new(self.from_array(field, array.values())?),
+            bitmap: match array.nulls() {
+                Some(nulls) => nulls.iter().collect(),
+                None => Bitmap::ones(array.len()),
+            },
+            offsets: array.offsets().iter().map(|o| *o as u32).collect(),
+        }))
+    }
 
-    fn try_from(value: DataType) -> Result<Self, Self::Error> {
-        (&value).try_into()
+    fn from_struct_array(&self, array: &arrow_array::StructArray) -> Result<ArrayImpl, ArrayError> {
+        use arrow_array::Array;
+        let arrow_schema::DataType::Struct(fields) = array.data_type() else {
+            panic!("nested field types cannot be determined.");
+        };
+        Ok(ArrayImpl::Struct(StructArray::new(
+            self.from_fields(fields)?,
+            array
+                .columns()
+                .iter()
+                .zip_eq_fast(fields)
+                .map(|(array, field)| self.from_array(field, array).map(Arc::new))
+                .try_collect()?,
+            (0..array.len()).map(|i| array.is_valid(i)).collect(),
+        )))
     }
 }
 
@@ -830,12 +756,15 @@ converts!(I64Array, arrow_array::Int64Array);
 converts!(F32Array, arrow_array::Float32Array, @map);
 converts!(F64Array, arrow_array::Float64Array, @map);
 converts!(BytesArray, arrow_array::BinaryArray);
+converts!(BytesArray, arrow_array::LargeBinaryArray);
 converts!(Utf8Array, arrow_array::StringArray);
+converts!(Utf8Array, arrow_array::LargeStringArray);
 converts!(DateArray, arrow_array::Date32Array, @map);
 converts!(TimeArray, arrow_array::Time64MicrosecondArray, @map);
 converts!(TimestampArray, arrow_array::TimestampMicrosecondArray, @map);
 converts!(TimestamptzArray, arrow_array::TimestampMicrosecondArray, @map);
 converts!(IntervalArray, arrow_array::IntervalMonthDayNanoArray, @map);
+converts!(SerialArray, arrow_array::Int64Array, @map);
 
 /// Converts RisingWave value from and into Arrow value.
 pub trait FromIntoArrow {
@@ -845,6 +774,18 @@ pub trait FromIntoArrow {
     fn into_arrow(self) -> Self::ArrowType;
 }
 
+impl FromIntoArrow for Serial {
+    type ArrowType = i64;
+
+    fn from_arrow(value: Self::ArrowType) -> Self {
+        value.into()
+    }
+
+    fn into_arrow(self) -> Self::ArrowType {
+        self.into()
+    }
+}
+
 impl FromIntoArrow for F32 {
     type ArrowType = f32;
 
@@ -973,6 +914,17 @@ impl From<&DecimalArray> for arrow_array::LargeBinaryArray {
     }
 }
 
+impl From<&DecimalArray> for arrow_array::StringArray {
+    fn from(array: &DecimalArray) -> Self {
+        let mut builder =
+            arrow_array::builder::StringBuilder::with_capacity(array.len(), array.len() * 8);
+        for value in array.iter() {
+            builder.append_option(value.map(|d| d.to_string()));
+        }
+        builder.finish()
+    }
+}
+
 // This arrow decimal type is used by iceberg source to read iceberg decimal into RW decimal.
 impl TryFrom<&arrow_array::Decimal128Array> for DecimalArray {
     type Error = ArrayError;
@@ -1020,6 +972,57 @@ impl TryFrom<&arrow_array::LargeBinaryArray> for DecimalArray {
     }
 }
 
+impl TryFrom<&arrow_array::StringArray> for DecimalArray {
+    type Error = ArrayError;
+
+    fn try_from(array: &arrow_array::StringArray) -> Result<Self, Self::Error> {
+        array
+            .iter()
+            .map(|o| {
+                o.map(|s| {
+                    s.parse()
+                        .map_err(|_| ArrayError::from_arrow(format!("invalid decimal: {s:?}")))
+                })
+                .transpose()
+            })
+            .try_collect()
+    }
+}
+
+impl From<&JsonbArray> for arrow_array::StringArray {
+    fn from(array: &JsonbArray) -> Self {
+        let mut builder =
+            arrow_array::builder::StringBuilder::with_capacity(array.len(), array.len() * 16);
+        for value in array.iter() {
+            match value {
+                Some(jsonb) => {
+                    write!(&mut builder, "{}", jsonb).unwrap();
+                    builder.append_value("");
+                }
+                None => builder.append_null(),
+            }
+        }
+        builder.finish()
+    }
+}
+
+impl TryFrom<&arrow_array::StringArray> for JsonbArray {
+    type Error = ArrayError;
+
+    fn try_from(array: &arrow_array::StringArray) -> Result<Self, Self::Error> {
+        array
+            .iter()
+            .map(|o| {
+                o.map(|s| {
+                    s.parse()
+                        .map_err(|_| ArrayError::from_arrow(format!("invalid json: {s}")))
+                })
+                .transpose()
+            })
+            .try_collect()
+    }
+}
+
 impl From<&JsonbArray> for arrow_array::LargeStringArray {
     fn from(array: &JsonbArray) -> Self {
         let mut builder =
@@ -1088,195 +1091,8 @@ impl From<&arrow_array::Decimal256Array> for Int256Array {
     }
 }
 
-impl TryFrom<&ListArray> for arrow_array::ListArray {
-    type Error = ArrayError;
-
-    fn try_from(array: &ListArray) -> Result<Self, Self::Error> {
-        use arrow_array::builder::*;
-        fn build<A, B, F>(
-            array: &ListArray,
-            a: &A,
-            builder: B,
-            mut append: F,
-        ) -> arrow_array::ListArray
-        where
-            A: Array,
-            B: arrow_array::builder::ArrayBuilder,
-            F: FnMut(&mut B, Option<A::RefItem<'_>>),
-        {
-            let mut builder = ListBuilder::with_capacity(builder, a.len());
-            for i in 0..array.len() {
-                for j in array.offsets[i]..array.offsets[i + 1] {
-                    append(builder.values(), a.value_at(j as usize));
-                }
-                builder.append(!array.is_null(i));
-            }
-            builder.finish()
-        }
-        Ok(match &*array.value {
-            ArrayImpl::Int16(a) => build(array, a, Int16Builder::with_capacity(a.len()), |b, v| {
-                b.append_option(v)
-            }),
-            ArrayImpl::Int32(a) => build(array, a, Int32Builder::with_capacity(a.len()), |b, v| {
-                b.append_option(v)
-            }),
-            ArrayImpl::Int64(a) => build(array, a, Int64Builder::with_capacity(a.len()), |b, v| {
-                b.append_option(v)
-            }),
-
-            ArrayImpl::Float32(a) => {
-                build(array, a, Float32Builder::with_capacity(a.len()), |b, v| {
-                    b.append_option(v.map(|f| f.0))
-                })
-            }
-            ArrayImpl::Float64(a) => {
-                build(array, a, Float64Builder::with_capacity(a.len()), |b, v| {
-                    b.append_option(v.map(|f| f.0))
-                })
-            }
-            ArrayImpl::Utf8(a) => build(
-                array,
-                a,
-                StringBuilder::with_capacity(a.len(), a.data().len()),
-                |b, v| b.append_option(v),
-            ),
-            ArrayImpl::Int256(a) => build(
-                array,
-                a,
-                Decimal256Builder::with_capacity(a.len()).with_data_type(
-                    arrow_schema::DataType::Decimal256(arrow_schema::DECIMAL256_MAX_PRECISION, 0),
-                ),
-                |b, v| b.append_option(v.map(Into::into)),
-            ),
-            ArrayImpl::Bool(a) => {
-                build(array, a, BooleanBuilder::with_capacity(a.len()), |b, v| {
-                    b.append_option(v)
-                })
-            }
-            ArrayImpl::Decimal(a) => build(
-                array,
-                a,
-                LargeBinaryBuilder::with_capacity(a.len(), a.len() * 8),
-                |b, v| b.append_option(v.map(|d| d.to_string())),
-            ),
-            ArrayImpl::Interval(a) => build(
-                array,
-                a,
-                IntervalMonthDayNanoBuilder::with_capacity(a.len()),
-                |b, v| b.append_option(v.map(|d| d.into_arrow())),
-            ),
-            ArrayImpl::Date(a) => build(array, a, Date32Builder::with_capacity(a.len()), |b, v| {
-                b.append_option(v.map(|d| d.into_arrow()))
-            }),
-            ArrayImpl::Timestamp(a) => build(
-                array,
-                a,
-                TimestampMicrosecondBuilder::with_capacity(a.len()),
-                |b, v| b.append_option(v.map(|d| d.into_arrow())),
-            ),
-            ArrayImpl::Timestamptz(a) => build(
-                array,
-                a,
-                TimestampMicrosecondBuilder::with_capacity(a.len()),
-                |b, v| b.append_option(v.map(|d| d.into_arrow())),
-            ),
-            ArrayImpl::Time(a) => build(
-                array,
-                a,
-                Time64MicrosecondBuilder::with_capacity(a.len()),
-                |b, v| b.append_option(v.map(|d| d.into_arrow())),
-            ),
-            ArrayImpl::Jsonb(a) => build(
-                array,
-                a,
-                LargeStringBuilder::with_capacity(a.len(), a.len() * 16),
-                |b, v| b.append_option(v.map(|j| j.to_string())),
-            ),
-            ArrayImpl::Serial(_) => todo!("list of serial"),
-            ArrayImpl::Struct(a) => {
-                let values = Arc::new(arrow_array::StructArray::try_from(a)?);
-                arrow_array::ListArray::new(
-                    Arc::new(arrow_schema::Field::new(
-                        "item",
-                        a.data_type().try_into()?,
-                        true,
-                    )),
-                    arrow_buffer::OffsetBuffer::new(arrow_buffer::ScalarBuffer::from(
-                        array
-                            .offsets()
-                            .iter()
-                            .map(|o| *o as i32)
-                            .collect::<Vec<i32>>(),
-                    )),
-                    values,
-                    Some(array.null_bitmap().into()),
-                )
-            }
-            ArrayImpl::List(_) => todo!("list of list"),
-            ArrayImpl::Bytea(a) => build(
-                array,
-                a,
-                BinaryBuilder::with_capacity(a.len(), a.data().len()),
-                |b, v| b.append_option(v),
-            ),
-        })
-    }
-}
-
-impl TryFrom<&arrow_array::ListArray> for ListArray {
-    type Error = ArrayError;
-
-    fn try_from(array: &arrow_array::ListArray) -> Result<Self, Self::Error> {
-        use arrow_array::Array;
-        Ok(ListArray {
-            value: Box::new(ArrayImpl::try_from(array.values())?),
-            bitmap: match array.nulls() {
-                Some(nulls) => nulls.iter().collect(),
-                None => Bitmap::ones(array.len()),
-            },
-            offsets: array.offsets().iter().map(|o| *o as u32).collect(),
-        })
-    }
-}
-
-impl TryFrom<&StructArray> for arrow_array::StructArray {
-    type Error = ArrayError;
-
-    fn try_from(array: &StructArray) -> Result<Self, Self::Error> {
-        Ok(arrow_array::StructArray::new(
-            array.data_type().as_struct().try_into()?,
-            array
-                .fields()
-                .map(|arr| arr.as_ref().try_into())
-                .try_collect::<_, _, ArrayError>()?,
-            Some(array.null_bitmap().into()),
-        ))
-    }
-}
-
-impl TryFrom<&arrow_array::StructArray> for StructArray {
-    type Error = ArrayError;
-
-    fn try_from(array: &arrow_array::StructArray) -> Result<Self, Self::Error> {
-        use arrow_array::Array;
-        let arrow_schema::DataType::Struct(fields) = array.data_type() else {
-            panic!("nested field types cannot be determined.");
-        };
-        Ok(StructArray::new(
-            fields.into(),
-            array
-                .columns()
-                .iter()
-                .map(|a| ArrayImpl::try_from(a).map(Arc::new))
-                .try_collect()?,
-            (0..array.len()).map(|i| !array.is_null(i)).collect(),
-        ))
-    }
-}
-
 #[cfg(test)]
 mod tests {
-    use super::arrow_array::Array as _;
     use super::*;
 
     #[test]
@@ -1293,6 +1109,20 @@ mod tests {
         assert_eq!(I16Array::from(&arrow), array);
     }
 
+    #[test]
+    fn i32() {
+        let array = I32Array::from_iter([None, Some(-7), Some(25)]);
+        let arrow = arrow_array::Int32Array::from(&array);
+        assert_eq!(I32Array::from(&arrow), array);
+    }
+
+    #[test]
+    fn i64() {
+        let array = I64Array::from_iter([None, Some(-7), Some(25)]);
+        let arrow = arrow_array::Int64Array::from(&array);
+        assert_eq!(I64Array::from(&arrow), array);
+    }
+
     #[test]
     fn f32() {
         let array = F32Array::from_iter([None, Some(-7.0), Some(25.0)]);
@@ -1300,6 +1130,13 @@ mod tests {
         assert_eq!(F32Array::from(&arrow), array);
     }
 
+    #[test]
+    fn f64() {
+        let array = F64Array::from_iter([None, Some(-7.0), Some(25.0)]);
+        let arrow = arrow_array::Float64Array::from(&array);
+        assert_eq!(F64Array::from(&arrow), array);
+    }
+
     #[test]
     fn date() {
         let array = DateArray::from_iter([
@@ -1352,6 +1189,13 @@ mod tests {
         assert_eq!(Utf8Array::from(&arrow), array);
     }
 
+    #[test]
+    fn binary() {
+        let array = BytesArray::from_iter([None, Some("array".as_bytes())]);
+        let arrow = arrow_array::BinaryArray::from(&array);
+        assert_eq!(BytesArray::from(&arrow), array);
+    }
+
     #[test]
     fn decimal() {
         let array = DecimalArray::from_iter([
@@ -1364,6 +1208,9 @@ mod tests {
         ]);
         let arrow = arrow_array::LargeBinaryArray::from(&array);
         assert_eq!(DecimalArray::try_from(&arrow).unwrap(), array);
+
+        let arrow = arrow_array::StringArray::from(&array);
+        assert_eq!(DecimalArray::try_from(&arrow).unwrap(), array);
     }
 
     #[test]
@@ -1378,6 +1225,9 @@ mod tests {
         ]);
         let arrow = arrow_array::LargeStringArray::from(&array);
         assert_eq!(JsonbArray::try_from(&arrow).unwrap(), array);
+
+        let arrow = arrow_array::StringArray::from(&array);
+        assert_eq!(JsonbArray::try_from(&arrow).unwrap(), array);
     }
 
     #[test]
@@ -1403,62 +1253,4 @@ mod tests {
         let arrow = arrow_array::Decimal256Array::from(&array);
         assert_eq!(Int256Array::from(&arrow), array);
     }
-
-    #[test]
-    fn struct_array() {
-        // Empty array - risingwave to arrow conversion.
-        let test_arr = StructArray::new(StructType::empty(), vec![], Bitmap::ones(0));
-        assert_eq!(
-            arrow_array::StructArray::try_from(&test_arr).unwrap().len(),
-            0
-        );
-
-        // Empty array - arrow to risingwave conversion.
-        let test_arr_2 = arrow_array::StructArray::from(vec![]);
-        assert_eq!(StructArray::try_from(&test_arr_2).unwrap().len(), 0);
-
-        // Struct array with primitive types. arrow to risingwave conversion.
-        let test_arrow_struct_array = arrow_array::StructArray::try_from(vec![
-            (
-                "a",
-                Arc::new(arrow_array::BooleanArray::from(vec![
-                    Some(false),
-                    Some(false),
-                    Some(true),
-                    None,
-                ])) as arrow_array::ArrayRef,
-            ),
-            (
-                "b",
-                Arc::new(arrow_array::Int32Array::from(vec![
-                    Some(42),
-                    Some(28),
-                    Some(19),
-                    None,
-                ])) as arrow_array::ArrayRef,
-            ),
-        ])
-        .unwrap();
-        let actual_risingwave_struct_array =
-            StructArray::try_from(&test_arrow_struct_array).unwrap();
-        let expected_risingwave_struct_array = StructArray::new(
-            StructType::new(vec![("a", DataType::Boolean), ("b", DataType::Int32)]),
-            vec![
-                BoolArray::from_iter([Some(false), Some(false), Some(true), None]).into_ref(),
-                I32Array::from_iter([Some(42), Some(28), Some(19), None]).into_ref(),
-            ],
-            [true, true, true, true].into_iter().collect(),
-        );
-        assert_eq!(
-            expected_risingwave_struct_array,
-            actual_risingwave_struct_array
-        );
-    }
-
-    #[test]
-    fn list() {
-        let array = ListArray::from_iter([None, Some(vec![0, -127, 127, 50]), Some(vec![0; 0])]);
-        let arrow = arrow_array::ListArray::try_from(&array).unwrap();
-        assert_eq!(ListArray::try_from(&arrow).unwrap(), array);
-    }
 }
diff --git a/src/common/src/array/arrow/arrow_udf.rs b/src/common/src/array/arrow/arrow_udf.rs
new file mode 100644
index 0000000000000..83383044f506d
--- /dev/null
+++ b/src/common/src/array/arrow/arrow_udf.rs
@@ -0,0 +1,207 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! This is for arrow dependency named `arrow-xxx` such as `arrow-array` in the cargo workspace.
+//!
+//! This should the default arrow version to be used in our system.
+//!
+//! The corresponding version of arrow is currently used by `udf` and `iceberg` sink.
+
+use std::sync::Arc;
+
+pub use arrow_impl::{FromArrow, ToArrow};
+use {arrow_array, arrow_buffer, arrow_cast, arrow_schema};
+
+use crate::array::{ArrayError, ArrayImpl, DataType, DecimalArray, JsonbArray};
+
+#[expect(clippy::duplicate_mod)]
+#[path = "./arrow_impl.rs"]
+mod arrow_impl;
+
+/// Arrow conversion for UDF.
+#[derive(Default, Debug)]
+pub struct UdfArrowConvert {
+    /// Whether the UDF talks in legacy mode.
+    ///
+    /// If true, decimal and jsonb types are mapped to Arrow `LargeBinary` and `LargeUtf8` types.
+    /// Otherwise, they are mapped to Arrow extension types.
+    /// See <https://github.com/risingwavelabs/arrow-udf/tree/main#extension-types>.
+    pub legacy: bool,
+}
+
+impl ToArrow for UdfArrowConvert {
+    fn decimal_to_arrow(
+        &self,
+        _data_type: &arrow_schema::DataType,
+        array: &DecimalArray,
+    ) -> Result<arrow_array::ArrayRef, ArrayError> {
+        if self.legacy {
+            // Decimal values are stored as ASCII text representation in a large binary array.
+            Ok(Arc::new(arrow_array::LargeBinaryArray::from(array)))
+        } else {
+            Ok(Arc::new(arrow_array::StringArray::from(array)))
+        }
+    }
+
+    fn jsonb_to_arrow(&self, array: &JsonbArray) -> Result<arrow_array::ArrayRef, ArrayError> {
+        if self.legacy {
+            // JSON values are stored as text representation in a large string array.
+            Ok(Arc::new(arrow_array::LargeStringArray::from(array)))
+        } else {
+            Ok(Arc::new(arrow_array::StringArray::from(array)))
+        }
+    }
+
+    fn jsonb_type_to_arrow(&self, name: &str) -> arrow_schema::Field {
+        if self.legacy {
+            arrow_schema::Field::new(name, arrow_schema::DataType::LargeUtf8, true)
+        } else {
+            arrow_schema::Field::new(name, arrow_schema::DataType::Utf8, true)
+                .with_metadata([("ARROW:extension:name".into(), "arrowudf.json".into())].into())
+        }
+    }
+
+    fn decimal_type_to_arrow(&self, name: &str) -> arrow_schema::Field {
+        if self.legacy {
+            arrow_schema::Field::new(name, arrow_schema::DataType::LargeBinary, true)
+        } else {
+            arrow_schema::Field::new(name, arrow_schema::DataType::Utf8, true)
+                .with_metadata([("ARROW:extension:name".into(), "arrowudf.decimal".into())].into())
+        }
+    }
+}
+
+impl FromArrow for UdfArrowConvert {
+    fn from_large_utf8(&self) -> Result<DataType, ArrayError> {
+        if self.legacy {
+            Ok(DataType::Jsonb)
+        } else {
+            Ok(DataType::Varchar)
+        }
+    }
+
+    fn from_large_binary(&self) -> Result<DataType, ArrayError> {
+        if self.legacy {
+            Ok(DataType::Decimal)
+        } else {
+            Ok(DataType::Bytea)
+        }
+    }
+
+    fn from_large_utf8_array(
+        &self,
+        array: &arrow_array::LargeStringArray,
+    ) -> Result<ArrayImpl, ArrayError> {
+        if self.legacy {
+            Ok(ArrayImpl::Jsonb(array.try_into()?))
+        } else {
+            Ok(ArrayImpl::Utf8(array.into()))
+        }
+    }
+
+    fn from_large_binary_array(
+        &self,
+        array: &arrow_array::LargeBinaryArray,
+    ) -> Result<ArrayImpl, ArrayError> {
+        if self.legacy {
+            Ok(ArrayImpl::Decimal(array.try_into()?))
+        } else {
+            Ok(ArrayImpl::Bytea(array.into()))
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::array::*;
+
+    #[test]
+    fn struct_array() {
+        // Empty array - risingwave to arrow conversion.
+        let test_arr = StructArray::new(StructType::empty(), vec![], Bitmap::ones(0));
+        assert_eq!(
+            UdfArrowConvert::default()
+                .struct_to_arrow(
+                    &arrow_schema::DataType::Struct(arrow_schema::Fields::empty()),
+                    &test_arr
+                )
+                .unwrap()
+                .len(),
+            0
+        );
+
+        // Empty array - arrow to risingwave conversion.
+        let test_arr_2 = arrow_array::StructArray::from(vec![]);
+        assert_eq!(
+            UdfArrowConvert::default()
+                .from_struct_array(&test_arr_2)
+                .unwrap()
+                .len(),
+            0
+        );
+
+        // Struct array with primitive types. arrow to risingwave conversion.
+        let test_arrow_struct_array = arrow_array::StructArray::try_from(vec![
+            (
+                "a",
+                Arc::new(arrow_array::BooleanArray::from(vec![
+                    Some(false),
+                    Some(false),
+                    Some(true),
+                    None,
+                ])) as arrow_array::ArrayRef,
+            ),
+            (
+                "b",
+                Arc::new(arrow_array::Int32Array::from(vec![
+                    Some(42),
+                    Some(28),
+                    Some(19),
+                    None,
+                ])) as arrow_array::ArrayRef,
+            ),
+        ])
+        .unwrap();
+        let actual_risingwave_struct_array = UdfArrowConvert::default()
+            .from_struct_array(&test_arrow_struct_array)
+            .unwrap()
+            .into_struct();
+        let expected_risingwave_struct_array = StructArray::new(
+            StructType::new(vec![("a", DataType::Boolean), ("b", DataType::Int32)]),
+            vec![
+                BoolArray::from_iter([Some(false), Some(false), Some(true), None]).into_ref(),
+                I32Array::from_iter([Some(42), Some(28), Some(19), None]).into_ref(),
+            ],
+            [true, true, true, true].into_iter().collect(),
+        );
+        assert_eq!(
+            expected_risingwave_struct_array,
+            actual_risingwave_struct_array
+        );
+    }
+
+    #[test]
+    fn list() {
+        let array = ListArray::from_iter([None, Some(vec![0, -127, 127, 50]), Some(vec![0; 0])]);
+        let data_type = arrow_schema::DataType::new_list(arrow_schema::DataType::Int32, true);
+        let arrow = UdfArrowConvert::default()
+            .list_to_arrow(&data_type, &array)
+            .unwrap();
+        let rw_array = UdfArrowConvert::default()
+            .from_list_array(arrow.as_any().downcast_ref().unwrap())
+            .unwrap();
+        assert_eq!(rw_array.as_list(), &array);
+    }
+}
diff --git a/src/common/src/array/arrow/mod.rs b/src/common/src/array/arrow/mod.rs
index 4baea60f11b3e..67490b22315a1 100644
--- a/src/common/src/array/arrow/mod.rs
+++ b/src/common/src/array/arrow/mod.rs
@@ -12,12 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod arrow_default;
 mod arrow_deltalake;
 mod arrow_iceberg;
+mod arrow_udf;
 
-pub use arrow_default::{
-    to_record_batch_with_schema, ToArrowArrayWithTypeConvert, ToArrowTypeConvert,
-};
-pub use arrow_deltalake::to_deltalake_record_batch_with_schema;
-pub use arrow_iceberg::{iceberg_to_arrow_type, to_iceberg_record_batch_with_schema};
+pub use arrow_deltalake::DeltaLakeConvert;
+pub use arrow_iceberg::IcebergArrowConvert;
+pub use arrow_udf::{FromArrow, ToArrow, UdfArrowConvert};
diff --git a/src/common/src/array/bytes_array.rs b/src/common/src/array/bytes_array.rs
index 1257730b63c96..7160c77b0d1c9 100644
--- a/src/common/src/array/bytes_array.rs
+++ b/src/common/src/array/bytes_array.rs
@@ -108,12 +108,6 @@ impl Array for BytesArray {
     }
 }
 
-impl BytesArray {
-    pub(super) fn data(&self) -> &[u8] {
-        &self.data
-    }
-}
-
 impl<'a> FromIterator<Option<&'a [u8]>> for BytesArray {
     fn from_iter<I: IntoIterator<Item = Option<&'a [u8]>>>(iter: I) -> Self {
         let iter = iter.into_iter();
@@ -148,19 +142,24 @@ pub struct BytesArrayBuilder {
 impl ArrayBuilder for BytesArrayBuilder {
     type ArrayType = BytesArray;
 
-    fn new(capacity: usize) -> Self {
-        let mut offset = Vec::with_capacity(capacity + 1);
+    /// Creates a new `BytesArrayBuilder`.
+    ///
+    /// `item_capacity` is the number of items to pre-allocate. The size of the preallocated
+    /// buffer of offsets is the number of items plus one.
+    /// No additional memory is pre-allocated for the data buffer.
+    fn new(item_capacity: usize) -> Self {
+        let mut offset = Vec::with_capacity(item_capacity + 1);
         offset.push(0);
         Self {
             offset,
-            data: Vec::with_capacity(capacity),
-            bitmap: BitmapBuilder::with_capacity(capacity),
+            data: Vec::with_capacity(0),
+            bitmap: BitmapBuilder::with_capacity(item_capacity),
         }
     }
 
-    fn with_type(capacity: usize, ty: DataType) -> Self {
+    fn with_type(item_capacity: usize, ty: DataType) -> Self {
         assert_eq!(ty, DataType::Bytea);
-        Self::new(capacity)
+        Self::new(item_capacity)
     }
 
     fn append_n<'a>(&'a mut self, n: usize, value: Option<&'a [u8]>) {
diff --git a/src/common/src/array/data_chunk.rs b/src/common/src/array/data_chunk.rs
index 849979abc5155..4e08163817e23 100644
--- a/src/common/src/array/data_chunk.rs
+++ b/src/common/src/array/data_chunk.rs
@@ -13,10 +13,10 @@
 // limitations under the License.
 
 use std::borrow::Cow;
+use std::fmt;
 use std::fmt::Display;
 use std::hash::BuildHasher;
 use std::sync::Arc;
-use std::{fmt, usize};
 
 use bytes::Bytes;
 use either::Either;
@@ -140,11 +140,13 @@ impl DataChunk {
         self.columns.len()
     }
 
+    // TODO(rc): shall we rename this to `visible_size`? I sometimes find this confused with `capacity`.
     /// `cardinality` returns the number of visible tuples
     pub fn cardinality(&self) -> usize {
         self.visibility.count_ones()
     }
 
+    // TODO(rc): shall we rename this to `size`?
     /// `capacity` returns physical length of any chunk column
     pub fn capacity(&self) -> usize {
         self.visibility.len()
@@ -247,6 +249,7 @@ impl DataChunk {
         Self::new(columns, Bitmap::ones(cardinality))
     }
 
+    /// Scatter a compacted chunk to a new chunk with the given visibility.
     pub fn uncompact(self, vis: Bitmap) -> Self {
         let mut uncompact_builders: Vec<_> = self
             .columns
@@ -801,7 +804,7 @@ impl DataChunkTestExt for DataChunk {
                 let arr = col;
                 let mut builder = arr.create_builder(n * 2);
                 for v in arr.iter() {
-                    builder.append(&v.to_owned_datum());
+                    builder.append(v.to_owned_datum());
                     builder.append_null();
                 }
 
diff --git a/src/common/src/array/list_array.rs b/src/common/src/array/list_array.rs
index dae9a4a94bc93..748ab6777fa49 100644
--- a/src/common/src/array/list_array.rs
+++ b/src/common/src/array/list_array.rs
@@ -239,6 +239,11 @@ impl ListArray {
         }
     }
 
+    /// Return the inner array of the list array.
+    pub fn values(&self) -> &ArrayImpl {
+        &self.value
+    }
+
     pub fn from_protobuf(array: &PbArray) -> ArrayResult<ArrayImpl> {
         ensure!(
             array.values.is_empty(),
diff --git a/src/common/src/array/mod.rs b/src/common/src/array/mod.rs
index 93f6255038e9e..268c518b70c01 100644
--- a/src/common/src/array/mod.rs
+++ b/src/common/src/array/mod.rs
@@ -14,11 +14,7 @@
 
 //! `Array` defines all in-memory representations of vectorized execution framework.
 
-mod arrow;
-pub use arrow::{
-    iceberg_to_arrow_type, to_deltalake_record_batch_with_schema,
-    to_iceberg_record_batch_with_schema, to_record_batch_with_schema,
-};
+pub mod arrow;
 mod bool_array;
 pub mod bytes_array;
 mod chrono_array;
diff --git a/src/common/src/array/primitive_array.rs b/src/common/src/array/primitive_array.rs
index 9840ba68135c5..29c7bfb49289a 100644
--- a/src/common/src/array/primitive_array.rs
+++ b/src/common/src/array/primitive_array.rs
@@ -315,7 +315,6 @@ impl<T: PrimitiveArrayItemType> ArrayBuilder for PrimitiveArrayBuilder<T> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::types::{F32, F64};
 
     fn helper_test_builder<T: PrimitiveArrayItemType>(data: Vec<Option<T>>) -> PrimitiveArray<T> {
         let mut builder = PrimitiveArrayBuilder::<T>::new(data.len());
diff --git a/src/common/src/array/proto_reader.rs b/src/common/src/array/proto_reader.rs
index b0b133d1b2df0..073ad0b3de7ba 100644
--- a/src/common/src/array/proto_reader.rs
+++ b/src/common/src/array/proto_reader.rs
@@ -17,16 +17,10 @@ use std::io::{Cursor, Read};
 use anyhow::Context;
 use byteorder::{BigEndian, ReadBytesExt};
 use paste::paste;
-use risingwave_pb::data::{PbArray, PbArrayType};
+use risingwave_pb::data::PbArrayType;
 
 use super::*;
 use crate::array::value_reader::{PrimitiveValueReader, VarSizedValueReader};
-use crate::array::{
-    Array, ArrayBuilder, ArrayImpl, ArrayResult, BoolArray, DateArrayBuilder, IntervalArrayBuilder,
-    PrimitiveArrayBuilder, PrimitiveArrayItemType, TimeArrayBuilder, TimestampArrayBuilder,
-};
-use crate::buffer::Bitmap;
-use crate::types::{Date, Interval, Time, Timestamp};
 
 impl ArrayImpl {
     pub fn from_protobuf(array: &PbArray, cardinality: usize) -> ArrayResult<Self> {
@@ -252,12 +246,6 @@ fn read_string_array<B: ArrayBuilder, R: VarSizedValueReader<B>>(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::array::{
-        Array, ArrayBuilder, BoolArray, BoolArrayBuilder, DateArray, DateArrayBuilder,
-        DecimalArray, DecimalArrayBuilder, I32Array, I32ArrayBuilder, TimeArray, TimeArrayBuilder,
-        TimestampArray, TimestampArrayBuilder, Utf8Array, Utf8ArrayBuilder,
-    };
-    use crate::types::{Date, Decimal, Time, Timestamp};
 
     // Convert a column to protobuf, then convert it back to column, and ensures the two are
     // identical.
diff --git a/src/common/src/array/stream_chunk.rs b/src/common/src/array/stream_chunk.rs
index 157e98a429544..e3776de6d0fbc 100644
--- a/src/common/src/array/stream_chunk.rs
+++ b/src/common/src/array/stream_chunk.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
 use std::{fmt, mem};
 
 use either::Either;
+use enum_as_inner::EnumAsInner;
 use itertools::Itertools;
 use rand::prelude::SmallRng;
 use rand::{Rng, SeedableRng};
@@ -40,7 +41,7 @@ use crate::types::{DataType, DefaultOrdered, ToText};
 /// but always appear in pairs to represent an update operation.
 /// For example, table source, aggregation and outer join can generate updates by themselves,
 /// while most of the other operators only pass through updates with best effort.
-#[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)]
+#[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq, Hash, EnumAsInner)]
 pub enum Op {
     Insert,
     Delete,
@@ -79,6 +80,15 @@ impl Op {
             Op::UpdateInsert => Op::Insert,
         }
     }
+
+    pub fn to_i16(self) -> i16 {
+        match self {
+            Op::Insert => 1,
+            Op::Delete => 2,
+            Op::UpdateInsert => 3,
+            Op::UpdateDelete => 4,
+        }
+    }
 }
 
 pub type Ops<'a> = &'a [Op];
@@ -132,11 +142,7 @@ impl StreamChunk {
     /// Should prefer using [`StreamChunkBuilder`] instead to avoid unnecessary
     /// allocation of rows.
     pub fn from_rows(rows: &[(Op, impl Row)], data_types: &[DataType]) -> Self {
-        // `append_row` will cause the builder to finish immediately once capacity is met.
-        // Hence, we allocate an extra row here, to avoid the builder finishing prematurely.
-        // This just makes the code cleaner, since we can loop through all rows, and consume it finally.
-        // TODO: introduce `new_unlimited` to decouple memory reservation from builder capacity.
-        let mut builder = StreamChunkBuilder::new(rows.len() + 1, data_types.to_vec());
+        let mut builder = StreamChunkBuilder::unlimited(data_types.to_vec(), Some(rows.len()));
 
         for (op, row) in rows {
             let none = builder.append_row(*op, row);
@@ -658,11 +664,7 @@ impl StreamChunk {
         let data_types = chunks[0].data_types();
         let size = chunks.iter().map(|c| c.cardinality()).sum::<usize>();
 
-        // `append_row` will cause the builder to finish immediately once capacity is met.
-        // Hence, we allocate an extra row here, to avoid the builder finishing prematurely.
-        // This just makes the code cleaner, since we can loop through all rows, and consume it finally.
-        // TODO: introduce `new_unlimited` to decouple memory reservation from builder capacity.
-        let mut builder = StreamChunkBuilder::new(size + 1, data_types);
+        let mut builder = StreamChunkBuilder::unlimited(data_types, Some(size));
 
         for chunk in chunks {
             // TODO: directly append chunks.
diff --git a/src/common/src/array/stream_chunk_builder.rs b/src/common/src/array/stream_chunk_builder.rs
index 2d70c925d696f..a13cc3676792a 100644
--- a/src/common/src/array/stream_chunk_builder.rs
+++ b/src/common/src/array/stream_chunk_builder.rs
@@ -33,10 +33,15 @@ pub struct StreamChunkBuilder {
     /// Data types of columns
     data_types: Vec<DataType>,
 
-    /// Maximum capacity of column builder
-    capacity: usize,
+    /// Max number of rows in a chunk. When it's `Some(n)`, the chunk builder will, if necessary,
+    /// yield a chunk of which the size is strictly less than or equal to `n` when appending records.
+    /// When it's `None`, the chunk builder will yield chunks only when `take` is called.
+    max_chunk_size: Option<usize>,
 
-    /// Size of column builder
+    /// The initial capacity of `ops` and `ArrayBuilder`s.
+    initial_capacity: usize,
+
+    /// Number of currently pending rows.
     size: usize,
 }
 
@@ -52,40 +57,50 @@ impl Drop for StreamChunkBuilder {
     }
 }
 
+const MAX_INITIAL_CAPACITY: usize = 4096;
+const DEFAULT_INITIAL_CAPACITY: usize = 64;
+
 impl StreamChunkBuilder {
-    pub fn new(chunk_size: usize, data_types: Vec<DataType>) -> Self {
-        assert!(chunk_size > 0);
+    /// Create a new `StreamChunkBuilder` with a fixed max chunk size.
+    /// Note that in the case of ending with `Update`, the builder may yield a chunk with size
+    /// `max_chunk_size + 1`.
+    pub fn new(max_chunk_size: usize, data_types: Vec<DataType>) -> Self {
+        assert!(max_chunk_size > 0);
+
+        let initial_capacity = max_chunk_size.min(MAX_INITIAL_CAPACITY);
 
-        let ops = Vec::with_capacity(chunk_size);
+        let ops = Vec::with_capacity(initial_capacity);
         let column_builders = data_types
             .iter()
-            .map(|datatype| datatype.create_array_builder(chunk_size))
+            .map(|datatype| datatype.create_array_builder(initial_capacity))
             .collect();
+        let vis_builder = BitmapBuilder::with_capacity(initial_capacity);
         Self {
             ops,
             column_builders,
             data_types,
-            vis_builder: BitmapBuilder::default(),
-            capacity: chunk_size,
+            vis_builder,
+            max_chunk_size: Some(max_chunk_size),
+            initial_capacity,
             size: 0,
         }
     }
 
-    /// Increase chunk size
-    ///
-    /// A [`StreamChunk`] will be returned when `size == capacity`
-    #[must_use]
-    fn inc_size(&mut self) -> Option<StreamChunk> {
-        self.size += 1;
-
-        // Take a chunk when capacity is exceeded. Splitting `UpdateDelete` and `UpdateInsert`
-        // should be avoided, so when the last one is `UpdateDelete`, we delay the chunk until
-        // `UpdateInsert` comes. This means the output chunk size may exceed the given `chunk_size`,
-        // and theoretically at most `chunk_size + 1` if inputs are consistent.
-        if self.size >= self.capacity && self.ops[self.ops.len() - 1] != Op::UpdateDelete {
-            self.take()
-        } else {
-            None
+    /// Create a new `StreamChunkBuilder` with unlimited chunk size.
+    /// The builder will only yield chunks when `take` is called.
+    pub fn unlimited(data_types: Vec<DataType>, initial_capacity: Option<usize>) -> Self {
+        let initial_capacity = initial_capacity.unwrap_or(DEFAULT_INITIAL_CAPACITY);
+        Self {
+            ops: Vec::with_capacity(initial_capacity),
+            column_builders: data_types
+                .iter()
+                .map(|datatype| datatype.create_array_builder(initial_capacity))
+                .collect(),
+            data_types,
+            vis_builder: BitmapBuilder::default(),
+            max_chunk_size: None,
+            initial_capacity,
+            size: 0,
         }
     }
 
@@ -102,20 +117,6 @@ impl StreamChunkBuilder {
         self.append_iter_inner::<true>(op, iter)
     }
 
-    #[must_use]
-    fn append_iter_inner<'a, const VIS: bool>(
-        &mut self,
-        op: Op,
-        iter: impl IntoIterator<Item = (usize, DatumRef<'a>)>,
-    ) -> Option<StreamChunk> {
-        self.ops.push(op);
-        for (i, datum) in iter {
-            self.column_builders[i].append(datum);
-        }
-        self.vis_builder.append(VIS);
-        self.inc_size()
-    }
-
     /// Append a row to the builder, return a chunk if the builder is full.
     #[must_use]
     pub fn append_row(&mut self, op: Op, row: impl Row) -> Option<StreamChunk> {
@@ -132,39 +133,221 @@ impl StreamChunkBuilder {
     #[must_use]
     pub fn append_record(&mut self, record: Record<impl Row>) -> Option<StreamChunk> {
         match record {
-            Record::Insert { new_row } => self.append_row(Op::Insert, new_row),
-            Record::Delete { old_row } => self.append_row(Op::Delete, old_row),
+            Record::Insert { new_row } => {
+                self.append_iter_inner::<true>(Op::Insert, new_row.iter().enumerate())
+            }
+            Record::Delete { old_row } => {
+                self.append_iter_inner::<true>(Op::Delete, old_row.iter().enumerate())
+            }
             Record::Update { old_row, new_row } => {
-                let none = self.append_row(Op::UpdateDelete, old_row);
-                debug_assert!(none.is_none());
-                self.append_row(Op::UpdateInsert, new_row)
+                let none =
+                    self.append_iter_inner::<true>(Op::UpdateDelete, old_row.iter().enumerate());
+                assert!(none.is_none());
+                self.append_iter_inner::<true>(Op::UpdateInsert, new_row.iter().enumerate())
             }
         }
     }
 
+    /// Take all the pending data and return a chunk. If there is no pending data, return `None`.
+    /// Note that if this is an unlimited chunk builder, the only way to get a chunk is to call
+    /// `take`.
     #[must_use]
     pub fn take(&mut self) -> Option<StreamChunk> {
         if self.size == 0 {
             return None;
         }
-
         self.size = 0;
-        let new_columns = self
+
+        let ops = std::mem::replace(&mut self.ops, Vec::with_capacity(self.initial_capacity));
+        let columns = self
             .column_builders
             .iter_mut()
             .zip_eq_fast(&self.data_types)
             .map(|(builder, datatype)| {
-                std::mem::replace(builder, datatype.create_array_builder(self.capacity)).finish()
+                std::mem::replace(
+                    builder,
+                    datatype.create_array_builder(self.initial_capacity),
+                )
+                .finish()
             })
             .map(Into::into)
             .collect::<Vec<_>>();
-
         let vis = std::mem::take(&mut self.vis_builder).finish();
 
-        Some(StreamChunk::with_visibility(
-            std::mem::replace(&mut self.ops, Vec::with_capacity(self.capacity)),
-            new_columns,
-            vis,
-        ))
+        Some(StreamChunk::with_visibility(ops, columns, vis))
+    }
+
+    #[must_use]
+    fn append_iter_inner<'a, const VIS: bool>(
+        &mut self,
+        op: Op,
+        iter: impl IntoIterator<Item = (usize, DatumRef<'a>)>,
+    ) -> Option<StreamChunk> {
+        self.ops.push(op);
+        for (i, datum) in iter {
+            self.column_builders[i].append(datum);
+        }
+        self.vis_builder.append(VIS);
+        self.size += 1;
+
+        if let Some(max_chunk_size) = self.max_chunk_size {
+            if self.size == max_chunk_size && !op.is_update_delete() || self.size > max_chunk_size {
+                // Two situations here:
+                // 1. `self.size == max_chunk_size && op == Op::UpdateDelete`
+                //    We should wait for next `UpdateInsert` to join the chunk.
+                // 2. `self.size > max_chunk_size`
+                //    Here we assert that `self.size == max_chunk_size + 1`. It's possible that
+                //    the `Op` after `UpdateDelete` is not `UpdateInsert`, if something inconsistent
+                //    happens, we should still take the existing data.
+                self.take()
+            } else {
+                None
+            }
+        } else {
+            // unlimited
+            None
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::array::{Datum, StreamChunkTestExt};
+    use crate::row::OwnedRow;
+
+    #[test]
+    fn test_stream_chunk_builder() {
+        let row = OwnedRow::new(vec![Datum::None, Datum::None]);
+        let mut builder = StreamChunkBuilder::new(3, vec![DataType::Int32, DataType::Int32]);
+        let res = builder.append_row(Op::Delete, row.clone());
+        assert!(res.is_none());
+        let res = builder.append_row(Op::Insert, row.clone());
+        assert!(res.is_none());
+        let res = builder.append_row(Op::Insert, row.clone());
+        assert_eq!(
+            res,
+            Some(StreamChunk::from_pretty(
+                "  i i
+                 - . .
+                 + . .
+                 + . ."
+            ))
+        );
+        let res = builder.take();
+        assert!(res.is_none());
+
+        let res = builder.append_row_invisible(Op::Delete, row.clone());
+        assert!(res.is_none());
+        let res = builder.append_iter(Op::Delete, row.clone().iter().enumerate());
+        assert!(res.is_none());
+        let res = builder.append_record(Record::Insert {
+            new_row: row.clone(),
+        });
+        assert_eq!(
+            res,
+            Some(StreamChunk::from_pretty(
+                "  i i
+                 - . . D
+                 - . .
+                 + . ."
+            ))
+        );
+
+        let res = builder.append_row(Op::UpdateDelete, row.clone());
+        assert!(res.is_none());
+        let res = builder.append_row(Op::UpdateInsert, row.clone());
+        assert!(res.is_none());
+        let res = builder.append_record(Record::Update {
+            old_row: row.clone(),
+            new_row: row.clone(),
+        });
+        assert_eq!(
+            res,
+            Some(StreamChunk::from_pretty(
+                "  i i
+                U- . .
+                U+ . .
+                U- . .
+                U+ . ."
+            ))
+        );
+        let res = builder.take();
+        assert!(res.is_none());
+    }
+
+    #[test]
+    fn test_stream_chunk_builder_with_max_size_1() {
+        let row = OwnedRow::new(vec![Datum::None, Datum::None]);
+        let mut builder = StreamChunkBuilder::new(1, vec![DataType::Int32, DataType::Int32]);
+
+        let res = builder.append_row(Op::Delete, row.clone());
+        assert_eq!(
+            res,
+            Some(StreamChunk::from_pretty(
+                "  i i
+                 - . ."
+            ))
+        );
+        let res = builder.append_row(Op::Insert, row.clone());
+        assert_eq!(
+            res,
+            Some(StreamChunk::from_pretty(
+                "  i i
+                 + . ."
+            ))
+        );
+
+        let res = builder.append_record(Record::Update {
+            old_row: row.clone(),
+            new_row: row.clone(),
+        });
+        assert_eq!(
+            res,
+            Some(StreamChunk::from_pretty(
+                "  i i
+                U- . .
+                U+ . ."
+            ))
+        );
+
+        let res = builder.append_row(Op::UpdateDelete, row.clone());
+        assert!(res.is_none());
+        let res = builder.append_row(Op::UpdateDelete, row.clone()); // note this is an inconsistency
+        assert_eq!(
+            res,
+            Some(StreamChunk::from_pretty(
+                "  i i
+                U- . .
+                U- . ."
+            ))
+        );
+    }
+
+    #[test]
+    fn test_unlimited_stream_chunk_builder() {
+        let row = OwnedRow::new(vec![Datum::None, Datum::None]);
+        let mut builder =
+            StreamChunkBuilder::unlimited(vec![DataType::Int32, DataType::Int32], None);
+
+        let res = builder.append_row(Op::Delete, row.clone());
+        assert!(res.is_none());
+        let res = builder.append_row(Op::Insert, row.clone());
+        assert!(res.is_none());
+        let res = builder.append_row(Op::UpdateDelete, row.clone());
+        assert!(res.is_none());
+        let res = builder.append_row(Op::UpdateInsert, row.clone());
+        assert!(res.is_none());
+
+        for _ in 0..2048 {
+            let res = builder.append_record(Record::Update {
+                old_row: row.clone(),
+                new_row: row.clone(),
+            });
+            assert!(res.is_none());
+        }
+
+        let res = builder.take();
+        assert_eq!(res.unwrap().capacity(), 2048 * 2 + 4);
     }
 }
diff --git a/src/common/src/array/utf8_array.rs b/src/common/src/array/utf8_array.rs
index eddcc50bb8ec2..5580ab6cc5970 100644
--- a/src/common/src/array/utf8_array.rs
+++ b/src/common/src/array/utf8_array.rs
@@ -112,10 +112,6 @@ impl Utf8Array {
         }
         builder.finish()
     }
-
-    pub(super) fn data(&self) -> &[u8] {
-        self.bytes.data()
-    }
 }
 
 /// `Utf8ArrayBuilder` use `&str` to build an `Utf8Array`.
@@ -127,15 +123,20 @@ pub struct Utf8ArrayBuilder {
 impl ArrayBuilder for Utf8ArrayBuilder {
     type ArrayType = Utf8Array;
 
-    fn new(capacity: usize) -> Self {
+    /// Creates a new `Utf8ArrayBuilder`.
+    ///
+    /// `item_capacity` is the number of items to pre-allocate. The size of the preallocated
+    /// buffer of offsets is the number of items plus one.
+    /// No additional memory is pre-allocated for the data buffer.
+    fn new(item_capacity: usize) -> Self {
         Self {
-            bytes: BytesArrayBuilder::new(capacity),
+            bytes: BytesArrayBuilder::new(item_capacity),
         }
     }
 
-    fn with_type(capacity: usize, ty: DataType) -> Self {
+    fn with_type(item_capacity: usize, ty: DataType) -> Self {
         assert_eq!(ty, DataType::Varchar);
-        Self::new(capacity)
+        Self::new(item_capacity)
     }
 
     #[inline]
@@ -170,6 +171,17 @@ impl Utf8ArrayBuilder {
             bytes: self.bytes.writer(),
         }
     }
+
+    /// Append an element as the `Display` format to the array.
+    pub fn append_display(&mut self, value: Option<impl Display>) {
+        if let Some(s) = value {
+            let mut writer = self.writer().begin();
+            write!(writer, "{}", s).unwrap();
+            writer.finish();
+        } else {
+            self.append_null();
+        }
+    }
 }
 
 pub struct StringWriter<'a> {
@@ -297,12 +309,6 @@ mod tests {
 
         let array = Utf8Array::from_iter(&input);
         assert_eq!(array.len(), input.len());
-
-        assert_eq!(
-            array.bytes.data().len(),
-            input.iter().map(|s| s.unwrap_or("").len()).sum::<usize>()
-        );
-
         assert_eq!(input, array.iter().collect_vec());
     }
 
diff --git a/src/common/src/buffer/bitmap.rs b/src/common/src/buffer/bitmap.rs
index 272cae6404f1e..c6a11da093ed4 100644
--- a/src/common/src/buffer/bitmap.rs
+++ b/src/common/src/buffer/bitmap.rs
@@ -37,7 +37,7 @@
 #![allow(clippy::disallowed_methods)]
 
 use std::iter::{self, TrustedLen};
-use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, Not, RangeInclusive};
+use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, Not, Range, RangeInclusive};
 
 use risingwave_common_estimate_size::EstimateSize;
 use risingwave_pb::common::buffer::CompressionType;
@@ -318,6 +318,11 @@ impl Bitmap {
         self.count_ones
     }
 
+    /// Returns true if any bit is set to 1.
+    pub fn any(&self) -> bool {
+        self.count_ones != 0
+    }
+
     /// Returns the length of vector to store `num_bits` bits.
     fn vec_len(num_bits: usize) -> usize {
         (num_bits + BITS - 1) / BITS
@@ -415,6 +420,46 @@ impl Bitmap {
             self.count_ones
         )
     }
+
+    /// Creates a new bitmap with all bits in range set to 1.
+    ///
+    /// # Example
+    /// ```
+    /// use risingwave_common::buffer::Bitmap;
+    /// let bitmap = Bitmap::from_range(200, 100..180);
+    /// assert_eq!(bitmap.count_ones(), 80);
+    /// for i in 0..200 {
+    ///     assert_eq!(bitmap.is_set(i), i >= 100 && i < 180);
+    /// }
+    /// ```
+    pub fn from_range(num_bits: usize, range: Range<usize>) -> Self {
+        assert!(range.start <= range.end);
+        assert!(range.end <= num_bits);
+        if range.start == range.end {
+            return Self::zeros(num_bits);
+        } else if range == (0..num_bits) {
+            return Self::ones(num_bits);
+        }
+        let mut bits = vec![0; Self::vec_len(num_bits)];
+        let start = range.start / BITS;
+        let end = range.end / BITS;
+        let start_offset = range.start % BITS;
+        let end_offset = range.end % BITS;
+        if start == end {
+            bits[start] = ((1 << (end_offset - start_offset)) - 1) << start_offset;
+        } else {
+            bits[start] = !0 << start_offset;
+            bits[start + 1..end].fill(!0);
+            if end_offset != 0 {
+                bits[end] = (1 << end_offset) - 1;
+            }
+        }
+        Self {
+            bits: Some(bits.into()),
+            num_bits,
+            count_ones: range.len(),
+        }
+    }
 }
 
 impl From<usize> for Bitmap {
@@ -763,6 +808,16 @@ impl<'a> iter::Iterator for BitmapOnesIter<'a> {
             }
         }
     }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        match self {
+            BitmapOnesIter::Range { start, end } => {
+                let remaining = end - start;
+                (remaining, Some(remaining))
+            }
+            BitmapOnesIter::Buffer { bits, .. } => (0, Some(bits.len() * usize::BITS as usize)),
+        }
+    }
 }
 
 #[cfg(test)]
diff --git a/src/common/src/cache.rs b/src/common/src/cache.rs
index 99a373d6a94a8..e86ef432eea95 100644
--- a/src/common/src/cache.rs
+++ b/src/common/src/cache.rs
@@ -1019,14 +1019,12 @@ impl<K: LruKey, T: LruValue> Clone for CacheableEntry<K, T> {
 #[cfg(test)]
 mod tests {
     use std::collections::hash_map::DefaultHasher;
-    use std::hash::{Hash, Hasher};
+    use std::hash::Hasher;
     use std::pin::Pin;
     use std::sync::atomic::AtomicBool;
     use std::sync::atomic::Ordering::Relaxed;
-    use std::sync::Arc;
     use std::task::{Context, Poll};
 
-    use futures::FutureExt;
     use rand::rngs::SmallRng;
     use rand::{RngCore, SeedableRng};
     use tokio::sync::oneshot::error::TryRecvError;
@@ -1035,6 +1033,7 @@ mod tests {
 
     pub struct Block {
         pub offset: u64,
+        #[allow(dead_code)]
         pub sst: u64,
     }
 
diff --git a/src/common/src/catalog/external_table.rs b/src/common/src/catalog/external_table.rs
index 3de38a29a499c..f9c6448358066 100644
--- a/src/common/src/catalog/external_table.rs
+++ b/src/common/src/catalog/external_table.rs
@@ -40,8 +40,6 @@ pub struct CdcTableDesc {
     /// Column indices for primary keys.
     pub stream_key: Vec<usize>,
 
-    pub value_indices: Vec<usize>,
-
     /// properties will be passed into the `StreamScanNode`
     pub connect_properties: BTreeMap<String, String>,
 }
diff --git a/src/common/src/catalog/internal_table.rs b/src/common/src/catalog/internal_table.rs
index 72e377b04d417..ba6370cdfc634 100644
--- a/src/common/src/catalog/internal_table.rs
+++ b/src/common/src/catalog/internal_table.rs
@@ -23,14 +23,14 @@ use regex::Regex;
 pub const RW_INTERNAL_TABLE_FUNCTION_NAME: &str = "rw_table";
 
 pub fn generate_internal_table_name_with_type(
-    mview_name: &str,
+    job_name: &str,
     fragment_id: u32,
     table_id: u32,
     table_type: &str,
 ) -> String {
     format!(
         "__internal_{}_{}_{}_{}",
-        mview_name,
+        job_name,
         fragment_id,
         table_type.to_lowercase(),
         table_id
@@ -43,13 +43,6 @@ pub fn valid_table_name(table_name: &str) -> bool {
     !INTERNAL_TABLE_NAME.is_match(table_name)
 }
 
-pub fn is_subscription_internal_table(subscription_name: &str, table_name: &str) -> bool {
-    let regex =
-        Regex::new(format!(r"__internal_{}_(\d+)_subscription_(\d+)", subscription_name).as_str())
-            .unwrap();
-    regex.is_match(table_name)
-}
-
 pub fn get_dist_key_in_pk_indices<I: Eq + Copy + Debug, O: TryFrom<usize>>(
     dist_key_indices: &[I],
     pk_indices: &[I],
diff --git a/src/common/src/catalog/mod.rs b/src/common/src/catalog/mod.rs
index 1909ca1635f7c..86c6e8895c066 100644
--- a/src/common/src/catalog/mod.rs
+++ b/src/common/src/catalog/mod.rs
@@ -33,6 +33,7 @@ use risingwave_pb::catalog::{
 };
 use risingwave_pb::plan_common::ColumnDescVersion;
 pub use schema::{test_utils as schema_test_utils, Field, FieldDisplay, Schema};
+use serde::{Deserialize, Serialize};
 
 use crate::array::DataChunk;
 pub use crate::constants::hummock;
@@ -228,7 +229,20 @@ impl From<SchemaId> for u32 {
     }
 }
 
-#[derive(Clone, Copy, Debug, Display, Default, Hash, PartialOrd, PartialEq, Eq, Ord)]
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    Display,
+    Default,
+    Hash,
+    PartialOrd,
+    PartialEq,
+    Eq,
+    Ord,
+    Serialize,
+    Deserialize,
+)]
 #[display("{table_id}")]
 pub struct TableId {
     pub table_id: u32,
@@ -437,6 +451,41 @@ impl From<ConnectionId> for u32 {
     }
 }
 
+#[derive(Clone, Copy, Debug, Display, Default, Hash, PartialOrd, PartialEq, Eq, Ord)]
+pub struct SecretId(pub u32);
+
+impl SecretId {
+    pub const fn new(id: u32) -> Self {
+        SecretId(id)
+    }
+
+    pub const fn placeholder() -> Self {
+        SecretId(OBJECT_ID_PLACEHOLDER)
+    }
+
+    pub fn secret_id(&self) -> u32 {
+        self.0
+    }
+}
+
+impl From<u32> for SecretId {
+    fn from(id: u32) -> Self {
+        Self::new(id)
+    }
+}
+
+impl From<&u32> for SecretId {
+    fn from(id: &u32) -> Self {
+        Self::new(*id)
+    }
+}
+
+impl From<SecretId> for u32 {
+    fn from(id: SecretId) -> Self {
+        id.0
+    }
+}
+
 #[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum ConflictBehavior {
     #[default]
diff --git a/src/common/src/config.rs b/src/common/src/config.rs
index e1e8ad8cfcd87..d8bb65e90b062 100644
--- a/src/common/src/config.rs
+++ b/src/common/src/config.rs
@@ -24,7 +24,7 @@ use std::num::NonZeroUsize;
 use anyhow::Context;
 use clap::ValueEnum;
 use educe::Educe;
-use foyer::memory::{LfuConfig, LruConfig, S3FifoConfig};
+use foyer::{LfuConfig, LruConfig, S3FifoConfig};
 use risingwave_common_proc_macro::ConfigDoc;
 pub use risingwave_common_proc_macro::OverrideConfig;
 use risingwave_pb::meta::SystemParams;
@@ -309,9 +309,12 @@ pub struct MetaConfig {
     #[serde(default)]
     pub do_not_config_object_storage_lifecycle: bool,
 
+    /// Count of partition in split group. Meta will assign this value to every new group when it splits from default-group by automatically.
+    /// Each partition contains aligned data of `VirtualNode::COUNT / partition_vnode_count` consecutive virtual-nodes of one state table.
     #[serde(default = "default::meta::partition_vnode_count")]
     pub partition_vnode_count: u32,
 
+    /// The threshold of write throughput to trigger a group split. Increase this configuration value to avoid split too many groups with few data write.
     #[serde(default = "default::meta::table_write_throughput_threshold")]
     pub table_write_throughput_threshold: u64,
 
@@ -334,8 +337,23 @@ pub struct MetaConfig {
     #[config_doc(nested)]
     pub compaction_config: CompactionConfig,
 
-    #[serde(default = "default::meta::hybird_partition_vnode_count")]
-    pub hybird_partition_vnode_count: u32,
+    /// Count of partitions of tables in default group and materialized view group.
+    /// The meta node will decide according to some strategy whether to cut the boundaries of the file according to the vnode alignment.
+    /// Each partition contains aligned data of `VirtualNode::COUNT / hybrid_partition_vnode_count` consecutive virtual-nodes of one state table.
+    /// Set it zero to disable this feature.
+    #[serde(default = "default::meta::hybrid_partition_vnode_count")]
+    pub hybrid_partition_vnode_count: u32,
+
+    /// The threshold of table size in one compact task to decide whether to partition one table into `hybrid_partition_vnode_count` parts, which belongs to default group and materialized view group.
+    /// Set it max value of 64-bit number to disable this feature.
+    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_low")]
+    pub compact_task_table_size_partition_threshold_low: u64,
+
+    /// The threshold of table size in one compact task to decide whether to partition one table into `partition_vnode_count` parts, which belongs to default group and materialized view group.
+    /// Set it max value of 64-bit number to disable this feature.
+    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_high")]
+    pub compact_task_table_size_partition_threshold_high: u64,
+
     #[serde(default = "default::meta::event_log_enabled")]
     pub event_log_enabled: bool,
     /// Keeps the latest N events per channel.
@@ -348,6 +366,9 @@ pub struct MetaConfig {
     /// Whether compactor should rewrite row to remove dropped column.
     #[serde(default = "default::meta::enable_dropped_column_reclaim")]
     pub enable_dropped_column_reclaim: bool,
+
+    #[serde(default = "default::meta::secret_store_private_key")]
+    pub secret_store_private_key: Vec<u8>,
 }
 
 #[derive(Copy, Clone, Debug, Default)]
@@ -434,6 +455,11 @@ pub struct MetaDeveloperConfig {
     pub enable_trivial_move: bool,
     #[serde(default = "default::developer::enable_check_task_level_overlap")]
     pub enable_check_task_level_overlap: bool,
+    #[serde(default = "default::developer::max_trivial_move_task_count_per_loop")]
+    pub max_trivial_move_task_count_per_loop: usize,
+
+    #[serde(default = "default::developer::max_get_task_probe_times")]
+    pub max_get_task_probe_times: usize,
 }
 
 /// The section `[server]` in `risingwave.toml`.
@@ -504,6 +530,15 @@ pub struct BatchConfig {
     /// This is the secs used to mask a worker unavailable temporarily.
     #[serde(default = "default::batch::mask_worker_temporary_secs")]
     pub mask_worker_temporary_secs: usize,
+
+    /// Keywords on which SQL option redaction is based in the query log.
+    /// A SQL option with a name containing any of these keywords will be redacted.
+    #[serde(default = "default::batch::redact_sql_option_keywords")]
+    pub redact_sql_option_keywords: Vec<String>,
+
+    /// Enable the spill out to disk feature for batch queries.
+    #[serde(default = "default::batch::enable_spill")]
+    pub enable_spill: bool,
 }
 
 /// The section `[streaming]` in `risingwave.toml`.
@@ -578,6 +613,8 @@ pub enum CacheEvictionConfig {
     },
     S3Fifo {
         small_queue_capacity_ratio_in_percent: Option<usize>,
+        ghost_queue_capacity_ratio_in_percent: Option<usize>,
+        small_to_main_freq_threshold: Option<u8>,
     },
 }
 
@@ -611,8 +648,14 @@ pub struct StorageConfig {
     #[serde(default = "default::storage::shared_buffer_flush_ratio")]
     pub shared_buffer_flush_ratio: f32,
 
+    /// The minimum total flush size of shared buffer spill. When a shared buffer spilled is trigger,
+    /// the total flush size across multiple epochs should be at least higher than this size.
+    #[serde(default = "default::storage::shared_buffer_min_batch_flush_size_mb")]
+    pub shared_buffer_min_batch_flush_size_mb: usize,
+
     /// The threshold for the number of immutable memtables to merge to a new imm.
     #[serde(default = "default::storage::imm_merge_threshold")]
+    #[deprecated]
     pub imm_merge_threshold: usize,
 
     /// Whether to enable write conflict detection
@@ -667,6 +710,9 @@ pub struct StorageConfig {
     #[serde(default = "default::storage::sstable_id_remote_fetch_number")]
     pub sstable_id_remote_fetch_number: u32,
 
+    #[serde(default = "default::storage::min_sstable_size_mb")]
+    pub min_sstable_size_mb: u32,
+
     #[serde(default)]
     pub data_file_cache: FileCacheConfig,
 
@@ -680,10 +726,6 @@ pub struct StorageConfig {
     #[serde(default = "default::storage::min_sst_size_for_streaming_upload")]
     pub min_sst_size_for_streaming_upload: u64,
 
-    /// Max sub compaction task numbers
-    #[serde(default = "default::storage::max_sub_compaction")]
-    pub max_sub_compaction: u32,
-
     #[serde(default = "default::storage::max_concurrent_compaction_task_number")]
     pub max_concurrent_compaction_task_number: u64,
 
@@ -721,6 +763,14 @@ pub struct StorageConfig {
     #[serde(default = "default::storage::mem_table_spill_threshold")]
     pub mem_table_spill_threshold: usize,
 
+    /// The concurrent uploading number of `SSTables` of buidler
+    #[serde(default = "default::storage::compactor_concurrent_uploading_sst_count")]
+    pub compactor_concurrent_uploading_sst_count: Option<usize>,
+
+    /// Object storage configuration
+    /// 1. General configuration
+    /// 2. Some special configuration of Backend
+    /// 3. Retry and timeout configuration
     #[serde(default)]
     pub object_store: ObjectStoreConfig,
 }
@@ -776,12 +826,6 @@ pub struct FileCacheConfig {
     #[serde(default = "default::file_cache::file_capacity_mb")]
     pub file_capacity_mb: usize,
 
-    #[serde(default = "default::file_cache::device_align")]
-    pub device_align: usize,
-
-    #[serde(default = "default::file_cache::device_io_size")]
-    pub device_io_size: usize,
-
     #[serde(default = "default::file_cache::flushers")]
     pub flushers: usize,
 
@@ -791,17 +835,11 @@ pub struct FileCacheConfig {
     #[serde(default = "default::file_cache::recover_concurrency")]
     pub recover_concurrency: usize,
 
-    #[serde(default = "default::file_cache::lfu_window_to_cache_size_ratio")]
-    pub lfu_window_to_cache_size_ratio: usize,
-
-    #[serde(default = "default::file_cache::lfu_tiny_lru_capacity_ratio")]
-    pub lfu_tiny_lru_capacity_ratio: f64,
-
     #[serde(default = "default::file_cache::insert_rate_limit_mb")]
     pub insert_rate_limit_mb: usize,
 
-    #[serde(default = "default::file_cache::catalog_bits")]
-    pub catalog_bits: usize,
+    #[serde(default = "default::file_cache::indexer_shards")]
+    pub indexer_shards: usize,
 
     #[serde(default = "default::file_cache::compression")]
     pub compression: String,
@@ -922,11 +960,31 @@ pub struct StreamingDeveloperConfig {
     #[serde(default = "default::developer::memory_controller_threshold_stable")]
     pub memory_controller_threshold_stable: f64,
 
+    #[serde(default = "default::developer::memory_controller_eviction_factor_aggressive")]
+    pub memory_controller_eviction_factor_aggressive: f64,
+
+    #[serde(default = "default::developer::memory_controller_eviction_factor_graceful")]
+    pub memory_controller_eviction_factor_graceful: f64,
+
+    #[serde(default = "default::developer::memory_controller_eviction_factor_stable")]
+    pub memory_controller_eviction_factor_stable: f64,
+
+    #[serde(default = "default::developer::memory_controller_sequence_tls_step")]
+    pub memory_controller_sequence_tls_step: u64,
+
+    #[serde(default = "default::developer::memory_controller_sequence_tls_lag")]
+    pub memory_controller_sequence_tls_lag: u64,
+
     #[serde(default = "default::developer::stream_enable_arrangement_backfill")]
     /// Enable arrangement backfill
     /// If true, the arrangement backfill will be disabled,
     /// even if session variable set.
     pub enable_arrangement_backfill: bool,
+
+    #[serde(default = "default::developer::stream_high_join_amplification_threshold")]
+    /// If number of hash join matches exceeds this threshold number,
+    /// it will be logged.
+    pub high_join_amplification_threshold: usize,
 }
 
 /// The subsections `[batch.developer]`.
@@ -971,17 +1029,16 @@ for_all_params!(define_system_config);
 /// The subsections `[storage.object_store]`.
 #[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
 pub struct ObjectStoreConfig {
-    #[serde(default = "default::object_store_config::object_store_streaming_read_timeout_ms")]
-    pub object_store_streaming_read_timeout_ms: u64,
-    #[serde(default = "default::object_store_config::object_store_streaming_upload_timeout_ms")]
-    pub object_store_streaming_upload_timeout_ms: u64,
-    #[serde(default = "default::object_store_config::object_store_upload_timeout_ms")]
-    pub object_store_upload_timeout_ms: u64,
-    #[serde(default = "default::object_store_config::object_store_read_timeout_ms")]
-    pub object_store_read_timeout_ms: u64,
     #[serde(default = "default::object_store_config::object_store_set_atomic_write_dir")]
     pub object_store_set_atomic_write_dir: bool,
 
+    /// Retry and timeout configuration
+    /// Description retry strategy driven by exponential back-off
+    /// Exposes the timeout and retries of each Object store interface. Therefore, the total timeout for each interface is determined based on the interface's timeout/retry configuration and the exponential back-off policy.
+    #[serde(default)]
+    pub retry: ObjectStoreRetryConfig,
+
+    /// Some special configuration of S3 Backend
     #[serde(default)]
     pub s3: S3ObjectStoreConfig,
 }
@@ -1003,12 +1060,6 @@ pub struct S3ObjectStoreConfig {
     pub object_store_send_buffer_size: Option<usize>,
     #[serde(default = "default::object_store_config::s3::object_store_nodelay")]
     pub object_store_nodelay: Option<bool>,
-    #[serde(default = "default::object_store_config::s3::object_store_req_retry_interval_ms")]
-    pub object_store_req_retry_interval_ms: u64,
-    #[serde(default = "default::object_store_config::s3::object_store_req_retry_max_delay_ms")]
-    pub object_store_req_retry_max_delay_ms: u64,
-    #[serde(default = "default::object_store_config::s3::object_store_req_retry_max_attempts")]
-    pub object_store_req_retry_max_attempts: usize,
     /// For backwards compatibility, users should use `S3ObjectStoreDeveloperConfig` instead.
     #[serde(
         default = "default::object_store_config::s3::developer::object_store_retry_unknown_service_error"
@@ -1039,6 +1090,95 @@ pub struct S3ObjectStoreDeveloperConfig {
     pub use_opendal: bool,
 }
 
+#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)]
+pub struct ObjectStoreRetryConfig {
+    // A retry strategy driven by exponential back-off.
+    // The retry strategy is used for all object store operations.
+    /// Given a base duration for retry strategy in milliseconds.
+    #[serde(default = "default::object_store_config::object_store_req_backoff_interval_ms")]
+    pub req_backoff_interval_ms: u64,
+
+    /// The max delay interval for the retry strategy. No retry delay will be longer than this `Duration`.
+    #[serde(default = "default::object_store_config::object_store_req_backoff_max_delay_ms")]
+    pub req_backoff_max_delay_ms: u64,
+
+    /// A multiplicative factor that will be applied to the exponential back-off retry delay.
+    #[serde(default = "default::object_store_config::object_store_req_backoff_factor")]
+    pub req_backoff_factor: u64,
+
+    /// Maximum timeout for `upload` operation
+    #[serde(default = "default::object_store_config::object_store_upload_attempt_timeout_ms")]
+    pub upload_attempt_timeout_ms: u64,
+
+    /// Total counts of `upload` operation retries
+    #[serde(default = "default::object_store_config::object_store_upload_retry_attempts")]
+    pub upload_retry_attempts: usize,
+
+    /// Maximum timeout for `streaming_upload_init` and `streaming_upload`
+    #[serde(
+        default = "default::object_store_config::object_store_streaming_upload_attempt_timeout_ms"
+    )]
+    pub streaming_upload_attempt_timeout_ms: u64,
+
+    /// Total counts of `streaming_upload` operation retries
+    #[serde(
+        default = "default::object_store_config::object_store_streaming_upload_retry_attempts"
+    )]
+    pub streaming_upload_retry_attempts: usize,
+
+    /// Maximum timeout for `read` operation
+    #[serde(default = "default::object_store_config::object_store_read_attempt_timeout_ms")]
+    pub read_attempt_timeout_ms: u64,
+
+    /// Total counts of `read` operation retries
+    #[serde(default = "default::object_store_config::object_store_read_retry_attempts")]
+    pub read_retry_attempts: usize,
+
+    /// Maximum timeout for `streaming_read_init` and `streaming_read` operation
+    #[serde(
+        default = "default::object_store_config::object_store_streaming_read_attempt_timeout_ms"
+    )]
+    pub streaming_read_attempt_timeout_ms: u64,
+
+    /// Total counts of `streaming_read operation` retries
+    #[serde(default = "default::object_store_config::object_store_streaming_read_retry_attempts")]
+    pub streaming_read_retry_attempts: usize,
+
+    /// Maximum timeout for `metadata` operation
+    #[serde(default = "default::object_store_config::object_store_metadata_attempt_timeout_ms")]
+    pub metadata_attempt_timeout_ms: u64,
+
+    /// Total counts of `metadata` operation retries
+    #[serde(default = "default::object_store_config::object_store_metadata_retry_attempts")]
+    pub metadata_retry_attempts: usize,
+
+    /// Maximum timeout for `delete` operation
+    #[serde(default = "default::object_store_config::object_store_delete_attempt_timeout_ms")]
+    pub delete_attempt_timeout_ms: u64,
+
+    /// Total counts of `delete` operation retries
+    #[serde(default = "default::object_store_config::object_store_delete_retry_attempts")]
+    pub delete_retry_attempts: usize,
+
+    /// Maximum timeout for `delete_object` operation
+    #[serde(
+        default = "default::object_store_config::object_store_delete_objects_attempt_timeout_ms"
+    )]
+    pub delete_objects_attempt_timeout_ms: u64,
+
+    /// Total counts of `delete_object` operation retries
+    #[serde(default = "default::object_store_config::object_store_delete_objects_retry_attempts")]
+    pub delete_objects_retry_attempts: usize,
+
+    /// Maximum timeout for `list` operation
+    #[serde(default = "default::object_store_config::object_store_list_attempt_timeout_ms")]
+    pub list_attempt_timeout_ms: u64,
+
+    /// Total counts of `list` operation retries
+    #[serde(default = "default::object_store_config::object_store_list_retry_attempts")]
+    pub list_retry_attempts: usize,
+}
+
 impl SystemConfig {
     #![allow(deprecated)]
     pub fn into_init_system_params(self) -> SystemParams {
@@ -1182,10 +1322,18 @@ pub mod default {
             1024 * 1024 * 1024 // 1GB
         }
 
-        pub fn hybird_partition_vnode_count() -> u32 {
+        pub fn hybrid_partition_vnode_count() -> u32 {
             4
         }
 
+        pub fn compact_task_table_size_partition_threshold_low() -> u64 {
+            128 * 1024 * 1024 // 128MB
+        }
+
+        pub fn compact_task_table_size_partition_threshold_high() -> u64 {
+            512 * 1024 * 1024 // 512MB
+        }
+
         pub fn event_log_enabled() -> bool {
             true
         }
@@ -1205,9 +1353,14 @@ pub mod default {
         pub fn parallelism_control_trigger_first_delay_sec() -> u64 {
             30
         }
+
         pub fn enable_dropped_column_reclaim() -> bool {
             false
         }
+
+        pub fn secret_store_private_key() -> Vec<u8> {
+            "demo-secret-private-key".as_bytes().to_vec()
+        }
     }
 
     pub mod server {
@@ -1251,6 +1404,10 @@ pub mod default {
             0.8
         }
 
+        pub fn shared_buffer_min_batch_flush_size_mb() -> usize {
+            800
+        }
+
         pub fn imm_merge_threshold() -> usize {
             0 // disable
         }
@@ -1287,6 +1444,14 @@ pub mod default {
             10
         }
 
+        pub fn ghost_queue_capacity_ratio_in_percent() -> usize {
+            1000
+        }
+
+        pub fn small_to_main_freq_threshold() -> u8 {
+            1
+        }
+
         pub fn meta_cache_capacity_mb() -> usize {
             128
         }
@@ -1304,7 +1469,7 @@ pub mod default {
         }
 
         pub fn compactor_max_task_multiplier() -> f32 {
-            2.5000
+            3.0000
         }
 
         pub fn compactor_memory_available_proportion() -> f64 {
@@ -1315,15 +1480,15 @@ pub mod default {
             10
         }
 
+        pub fn min_sstable_size_mb() -> u32 {
+            32
+        }
+
         pub fn min_sst_size_for_streaming_upload() -> u64 {
             // 32MB
             32 * 1024 * 1024
         }
 
-        pub fn max_sub_compaction() -> u32 {
-            4
-        }
-
         pub fn max_concurrent_compaction_task_number() -> u64 {
             16
         }
@@ -1353,7 +1518,7 @@ pub mod default {
         }
 
         pub fn enable_fast_compaction() -> bool {
-            false
+            true
         }
 
         pub fn check_compaction_result() -> bool {
@@ -1363,6 +1528,7 @@ pub mod default {
         pub fn max_preload_io_retry_times() -> usize {
             3
         }
+
         pub fn mem_table_spill_threshold() -> usize {
             4 << 20
         }
@@ -1378,6 +1544,10 @@ pub mod default {
         pub fn max_prefetch_block_number() -> usize {
             16
         }
+
+        pub fn compactor_concurrent_uploading_sst_count() -> Option<usize> {
+            None
+        }
     }
 
     pub mod streaming {
@@ -1403,7 +1573,6 @@ pub mod default {
     }
 
     pub mod file_cache {
-
         pub fn dir() -> String {
             "".to_string()
         }
@@ -1416,14 +1585,6 @@ pub mod default {
             64
         }
 
-        pub fn device_align() -> usize {
-            4096
-        }
-
-        pub fn device_io_size() -> usize {
-            16 * 1024
-        }
-
         pub fn flushers() -> usize {
             4
         }
@@ -1436,20 +1597,12 @@ pub mod default {
             8
         }
 
-        pub fn lfu_window_to_cache_size_ratio() -> usize {
-            1
-        }
-
-        pub fn lfu_tiny_lru_capacity_ratio() -> f64 {
-            0.01
-        }
-
         pub fn insert_rate_limit_mb() -> usize {
             0
         }
 
-        pub fn catalog_bits() -> usize {
-            6
+        pub fn indexer_shards() -> usize {
+            64
         }
 
         pub fn compression() -> String {
@@ -1566,18 +1719,53 @@ pub mod default {
             false
         }
 
+        pub fn max_trivial_move_task_count_per_loop() -> usize {
+            256
+        }
+
+        pub fn max_get_task_probe_times() -> usize {
+            5
+        }
+
         pub fn memory_controller_threshold_aggressive() -> f64 {
             0.9
         }
+
         pub fn memory_controller_threshold_graceful() -> f64 {
             0.8
         }
+
         pub fn memory_controller_threshold_stable() -> f64 {
             0.7
         }
+
+        pub fn memory_controller_eviction_factor_aggressive() -> f64 {
+            2.0
+        }
+
+        pub fn memory_controller_eviction_factor_graceful() -> f64 {
+            1.5
+        }
+
+        pub fn memory_controller_eviction_factor_stable() -> f64 {
+            1.0
+        }
+
+        pub fn memory_controller_sequence_tls_step() -> u64 {
+            128
+        }
+
+        pub fn memory_controller_sequence_tls_lag() -> u64 {
+            32
+        }
+
         pub fn stream_enable_arrangement_backfill() -> bool {
             true
         }
+
+        pub fn stream_high_join_amplification_threshold() -> usize {
+            2048
+        }
     }
 
     pub use crate::system_param::default as system;
@@ -1587,6 +1775,10 @@ pub mod default {
             false
         }
 
+        pub fn enable_spill() -> bool {
+            true
+        }
+
         pub fn statement_timeout_in_sec() -> u32 {
             // 1 hour
             60 * 60
@@ -1599,6 +1791,20 @@ pub mod default {
         pub fn mask_worker_temporary_secs() -> usize {
             30
         }
+
+        pub fn redact_sql_option_keywords() -> Vec<String> {
+            [
+                "credential",
+                "key",
+                "password",
+                "private",
+                "secret",
+                "token",
+            ]
+            .into_iter()
+            .map(str::to_string)
+            .collect()
+        }
     }
 
     pub mod compaction_config {
@@ -1608,7 +1814,8 @@ pub mod default {
 
         // decrease this configure when the generation of checkpoint barrier is not frequent.
         const DEFAULT_TIER_COMPACT_TRIGGER_NUMBER: u64 = 12;
-        const DEFAULT_TARGET_FILE_SIZE_BASE: u64 = 32 * 1024 * 1024; // 32MB
+        const DEFAULT_TARGET_FILE_SIZE_BASE: u64 = 32 * 1024 * 1024;
+        // 32MB
         const DEFAULT_MAX_SUB_COMPACTION: u32 = 4;
         const DEFAULT_LEVEL_MULTIPLIER: u64 = 5;
         const DEFAULT_MAX_SPACE_RECLAIM_BYTES: u64 = 512 * 1024 * 1024; // 512MB;
@@ -1618,48 +1825,63 @@ pub mod default {
         const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 12;
         const DEFAULT_TOMBSTONE_RATIO_PERCENT: u32 = 40;
         const DEFAULT_EMERGENCY_PICKER: bool = true;
+        const DEFAULT_MAX_LEVEL: u32 = 6;
+        const DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT: u32 = 42;
 
         use crate::catalog::hummock::CompactionFilterFlag;
 
         pub fn max_bytes_for_level_base() -> u64 {
             DEFAULT_MAX_BYTES_FOR_LEVEL_BASE
         }
+
         pub fn max_bytes_for_level_multiplier() -> u64 {
             DEFAULT_LEVEL_MULTIPLIER
         }
+
         pub fn max_compaction_bytes() -> u64 {
             DEFAULT_MAX_COMPACTION_BYTES
         }
+
         pub fn sub_level_max_compaction_bytes() -> u64 {
             DEFAULT_MIN_COMPACTION_BYTES
         }
+
         pub fn level0_tier_compact_file_number() -> u64 {
             DEFAULT_TIER_COMPACT_TRIGGER_NUMBER
         }
+
         pub fn target_file_size_base() -> u64 {
             DEFAULT_TARGET_FILE_SIZE_BASE
         }
+
         pub fn compaction_filter_mask() -> u32 {
             (CompactionFilterFlag::STATE_CLEAN | CompactionFilterFlag::TTL).into()
         }
+
         pub fn max_sub_compaction() -> u32 {
             DEFAULT_MAX_SUB_COMPACTION
         }
+
         pub fn level0_stop_write_threshold_sub_level_number() -> u64 {
             DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER
         }
+
         pub fn level0_sub_level_compact_level_count() -> u32 {
             DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT
         }
+
         pub fn level0_overlapping_sub_level_compact_level_count() -> u32 {
             DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT
         }
+
         pub fn max_space_reclaim_bytes() -> u64 {
             DEFAULT_MAX_SPACE_RECLAIM_BYTES
         }
+
         pub fn level0_max_compact_file_number() -> u64 {
             DEFAULT_MAX_COMPACTION_FILE_COUNT
         }
+
         pub fn tombstone_reclaim_ratio() -> u32 {
             DEFAULT_TOMBSTONE_RATIO_PERCENT
         }
@@ -1667,34 +1889,105 @@ pub mod default {
         pub fn enable_emergency_picker() -> bool {
             DEFAULT_EMERGENCY_PICKER
         }
+
+        pub fn max_level() -> u32 {
+            DEFAULT_MAX_LEVEL
+        }
+
+        pub fn max_l0_compact_level_count() -> u32 {
+            DEFAULT_MAX_L0_COMPACT_LEVEL_COUNT
+        }
     }
 
     pub mod object_store_config {
-        pub fn object_store_streaming_read_timeout_ms() -> u64 {
-            8 * 60 * 1000
+        const DEFAULT_REQ_BACKOFF_INTERVAL_MS: u64 = 1000; // 1s
+        const DEFAULT_REQ_BACKOFF_MAX_DELAY_MS: u64 = 10 * 1000; // 10s
+        const DEFAULT_REQ_MAX_RETRY_ATTEMPTS: usize = 3;
+
+        pub fn object_store_set_atomic_write_dir() -> bool {
+            false
         }
 
-        pub fn object_store_streaming_upload_timeout_ms() -> u64 {
-            8 * 60 * 1000
+        pub fn object_store_req_backoff_interval_ms() -> u64 {
+            DEFAULT_REQ_BACKOFF_INTERVAL_MS
         }
 
-        pub fn object_store_upload_timeout_ms() -> u64 {
-            8 * 60 * 1000
+        pub fn object_store_req_backoff_max_delay_ms() -> u64 {
+            DEFAULT_REQ_BACKOFF_MAX_DELAY_MS // 10s
         }
 
-        pub fn object_store_read_timeout_ms() -> u64 {
-            8 * 60 * 1000
+        pub fn object_store_req_backoff_factor() -> u64 {
+            2
         }
 
-        pub fn object_store_set_atomic_write_dir() -> bool {
-            false
+        pub fn object_store_upload_attempt_timeout_ms() -> u64 {
+            8 * 1000 // 8s
+        }
+
+        pub fn object_store_upload_retry_attempts() -> usize {
+            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
+        }
+
+        // init + upload_part + finish
+        pub fn object_store_streaming_upload_attempt_timeout_ms() -> u64 {
+            5 * 1000 // 5s
+        }
+
+        pub fn object_store_streaming_upload_retry_attempts() -> usize {
+            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
+        }
+
+        // tips: depend on block_size
+        pub fn object_store_read_attempt_timeout_ms() -> u64 {
+            8 * 1000 // 8s
+        }
+
+        pub fn object_store_read_retry_attempts() -> usize {
+            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
+        }
+
+        pub fn object_store_streaming_read_attempt_timeout_ms() -> u64 {
+            3 * 1000 // 3s
+        }
+
+        pub fn object_store_streaming_read_retry_attempts() -> usize {
+            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
+        }
+
+        pub fn object_store_metadata_attempt_timeout_ms() -> u64 {
+            60 * 1000 // 1min
+        }
+
+        pub fn object_store_metadata_retry_attempts() -> usize {
+            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
+        }
+
+        pub fn object_store_delete_attempt_timeout_ms() -> u64 {
+            5 * 1000
+        }
+
+        pub fn object_store_delete_retry_attempts() -> usize {
+            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
+        }
+
+        // tips: depend on batch size
+        pub fn object_store_delete_objects_attempt_timeout_ms() -> u64 {
+            5 * 1000
+        }
+
+        pub fn object_store_delete_objects_retry_attempts() -> usize {
+            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
+        }
+
+        pub fn object_store_list_attempt_timeout_ms() -> u64 {
+            10 * 60 * 1000
+        }
+
+        pub fn object_store_list_retry_attempts() -> usize {
+            DEFAULT_REQ_MAX_RETRY_ATTEMPTS
         }
 
         pub mod s3 {
-            /// Retry config for compute node http timeout error.
-            const DEFAULT_RETRY_INTERVAL_MS: u64 = 20;
-            const DEFAULT_RETRY_MAX_DELAY_MS: u64 = 10 * 1000;
-            const DEFAULT_RETRY_MAX_ATTEMPTS: usize = 8;
             const DEFAULT_IDENTITY_RESOLUTION_TIMEOUT_S: u64 = 5;
 
             const DEFAULT_KEEPALIVE_MS: u64 = 600 * 1000; // 10min
@@ -1715,24 +2008,13 @@ pub mod default {
                 Some(true)
             }
 
-            pub fn object_store_req_retry_interval_ms() -> u64 {
-                DEFAULT_RETRY_INTERVAL_MS
-            }
-
-            pub fn object_store_req_retry_max_delay_ms() -> u64 {
-                DEFAULT_RETRY_MAX_DELAY_MS // 10s
-            }
-
-            pub fn object_store_req_retry_max_attempts() -> usize {
-                DEFAULT_RETRY_MAX_ATTEMPTS
-            }
-
             pub fn identity_resolution_timeout_s() -> u64 {
                 DEFAULT_IDENTITY_RESOLUTION_TIMEOUT_S
             }
 
             pub mod developer {
                 use crate::util::env_var::env_var_is_true_or;
+
                 const RW_USE_OPENDAL_FOR_S3: &str = "RW_USE_OPENDAL_FOR_S3";
 
                 pub fn object_store_retry_unknown_service_error() -> bool {
@@ -1770,6 +2052,16 @@ impl EvictionConfig {
     }
 }
 
+impl From<EvictionConfig> for foyer::EvictionConfig {
+    fn from(value: EvictionConfig) -> Self {
+        match value {
+            EvictionConfig::Lru(lru) => foyer::EvictionConfig::Lru(lru),
+            EvictionConfig::Lfu(lfu) => foyer::EvictionConfig::Lfu(lfu),
+            EvictionConfig::S3Fifo(s3fifo) => foyer::EvictionConfig::S3Fifo(s3fifo),
+        }
+    }
+}
+
 pub struct StorageMemoryConfig {
     pub block_cache_capacity_mb: usize,
     pub block_cache_shard_num: usize,
@@ -1856,11 +2148,19 @@ pub fn extract_storage_memory_config(s: &RwConfig) -> StorageMemoryConfig {
             }),
             CacheEvictionConfig::S3Fifo {
                 small_queue_capacity_ratio_in_percent,
+                ghost_queue_capacity_ratio_in_percent,
+                small_to_main_freq_threshold,
             } => EvictionConfig::S3Fifo(S3FifoConfig {
                 small_queue_capacity_ratio: small_queue_capacity_ratio_in_percent
                     .unwrap_or(default::storage::small_queue_capacity_ratio_in_percent())
                     as f64
                     / 100.0,
+                ghost_queue_capacity_ratio: ghost_queue_capacity_ratio_in_percent
+                    .unwrap_or(default::storage::ghost_queue_capacity_ratio_in_percent())
+                    as f64
+                    / 100.0,
+                small_to_main_freq_threshold: small_to_main_freq_threshold
+                    .unwrap_or(default::storage::small_to_main_freq_threshold()),
             }),
         }
     };
@@ -1930,12 +2230,12 @@ pub struct CompactionConfig {
     pub tombstone_reclaim_ratio: u32,
     #[serde(default = "default::compaction_config::enable_emergency_picker")]
     pub enable_emergency_picker: bool,
+    #[serde(default = "default::compaction_config::max_level")]
+    pub max_level: u32,
 }
 
 #[cfg(test)]
 mod tests {
-    use std::collections::BTreeMap;
-
     use super::*;
 
     /// This test ensures that `config/example.toml` is up-to-date with the default values specified
diff --git a/src/common/src/field_generator/varchar.rs b/src/common/src/field_generator/varchar.rs
index c4af1499620ca..9f68bcb287c46 100644
--- a/src/common/src/field_generator/varchar.rs
+++ b/src/common/src/field_generator/varchar.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::string::ToString;
-
 use rand::distributions::Alphanumeric;
 use rand::rngs::StdRng;
 use rand::{Rng, SeedableRng};
diff --git a/src/common/src/hash/consistent_hash/mapping.rs b/src/common/src/hash/consistent_hash/mapping.rs
index c542ab2050cf1..c057cf847c7db 100644
--- a/src/common/src/hash/consistent_hash/mapping.rs
+++ b/src/common/src/hash/consistent_hash/mapping.rs
@@ -12,14 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
-use std::fmt::Debug;
+use std::collections::{BTreeSet, HashMap};
+use std::fmt::{Debug, Display, Formatter};
 use std::hash::Hash;
 use std::ops::Index;
 
 use educe::Educe;
 use itertools::Itertools;
-use risingwave_pb::common::{ParallelUnit, ParallelUnitMapping as ParallelUnitMappingProto};
+use risingwave_pb::common::{
+    ParallelUnit, ParallelUnitMapping as ParallelUnitMappingProto, PbWorkerSlotMapping,
+};
 use risingwave_pb::stream_plan::ActorMapping as ActorMappingProto;
 
 use super::bitmap::VnodeBitmapExt;
@@ -31,6 +33,41 @@ use crate::util::iter_util::ZipEqDebug;
 // TODO: find a better place for this.
 pub type ActorId = u32;
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)]
+pub struct WorkerSlotId(u64);
+
+impl WorkerSlotId {
+    pub fn worker_id(&self) -> u32 {
+        (self.0 >> 32) as u32
+    }
+
+    pub fn slot_idx(&self) -> u32 {
+        self.0 as u32
+    }
+
+    pub fn new(worker_id: u32, slot_idx: usize) -> Self {
+        Self((worker_id as u64) << 32 | slot_idx as u64)
+    }
+}
+
+impl From<WorkerSlotId> for u64 {
+    fn from(id: WorkerSlotId) -> Self {
+        id.0
+    }
+}
+
+impl From<u64> for WorkerSlotId {
+    fn from(id: u64) -> Self {
+        Self(id)
+    }
+}
+
+impl Display for WorkerSlotId {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_fmt(format_args!("[{}:{}]", self.worker_id(), self.slot_idx()))
+    }
+}
+
 /// Trait for items that can be used as keys in [`VnodeMapping`].
 pub trait VnodeMappingItem {
     /// The type of the item.
@@ -254,6 +291,12 @@ pub mod marker {
     impl VnodeMappingItem for ParallelUnit {
         type Item = ParallelUnitId;
     }
+
+    /// A marker type for items of [`WorkerSlotId`].
+    pub struct WorkerSlot;
+    impl VnodeMappingItem for WorkerSlot {
+        type Item = WorkerSlotId;
+    }
 }
 
 /// A mapping from [`VirtualNode`] to [`ActorId`].
@@ -266,6 +309,11 @@ pub type ParallelUnitMapping = VnodeMapping<marker::ParallelUnit>;
 /// An expanded mapping from [`VirtualNode`] to [`ParallelUnitId`].
 pub type ExpandedParallelUnitMapping = ExpandedMapping<marker::ParallelUnit>;
 
+/// A mapping from [`VirtualNode`] to [`WorkerSlotId`].
+pub type WorkerSlotMapping = VnodeMapping<marker::WorkerSlot>;
+/// An expanded mapping from [`VirtualNode`] to [`WorkerSlotId`].
+pub type ExpandedWorkerSlotMapping = ExpandedMapping<marker::WorkerSlot>;
+
 impl ActorMapping {
     /// Transform this actor mapping to a parallel unit mapping, essentially `transform`.
     pub fn to_parallel_unit<M>(&self, to_map: &M) -> ParallelUnitMapping
@@ -293,6 +341,30 @@ impl ActorMapping {
     }
 }
 
+impl WorkerSlotMapping {
+    /// Create a uniform worker mapping from the given worker ids
+    pub fn build_from_ids(worker_slot_ids: &[WorkerSlotId]) -> Self {
+        Self::new_uniform(worker_slot_ids.iter().cloned())
+    }
+
+    /// Create a worker mapping from the protobuf representation.
+    pub fn from_protobuf(proto: &PbWorkerSlotMapping) -> Self {
+        assert_eq!(proto.original_indices.len(), proto.data.len());
+        Self {
+            original_indices: proto.original_indices.clone(),
+            data: proto.data.iter().map(|&id| WorkerSlotId(id)).collect(),
+        }
+    }
+
+    /// Convert this worker mapping to the protobuf representation.
+    pub fn to_protobuf(&self) -> PbWorkerSlotMapping {
+        PbWorkerSlotMapping {
+            original_indices: self.original_indices.clone(),
+            data: self.data.iter().map(|id| id.0).collect(),
+        }
+    }
+}
+
 impl ParallelUnitMapping {
     /// Create a uniform parallel unit mapping from the given parallel units, essentially
     /// `new_uniform`.
@@ -310,6 +382,28 @@ impl ParallelUnitMapping {
         self.transform(to_map)
     }
 
+    /// Transform this parallel unit mapping to an worker mapping, essentially `transform`.
+    pub fn to_worker_slot(&self, to_map: &HashMap<ParallelUnitId, u32>) -> WorkerSlotMapping {
+        let mut worker_to_parallel_units = HashMap::<_, BTreeSet<_>>::new();
+        for (parallel_unit_id, worker_id) in to_map {
+            worker_to_parallel_units
+                .entry(*worker_id)
+                .or_default()
+                .insert(*parallel_unit_id);
+        }
+
+        let mut parallel_unit_to_worker_slot = HashMap::with_capacity(to_map.len());
+
+        for (worker_id, parallel_unit_ids) in worker_to_parallel_units {
+            for (index, &parallel_unit_id) in parallel_unit_ids.iter().enumerate() {
+                parallel_unit_to_worker_slot
+                    .insert(parallel_unit_id, WorkerSlotId::new(worker_id, index));
+            }
+        }
+
+        self.transform(&parallel_unit_to_worker_slot)
+    }
+
     /// Create a parallel unit mapping from the protobuf representation.
     pub fn from_protobuf(proto: &ParallelUnitMappingProto) -> Self {
         assert_eq!(proto.original_indices.len(), proto.data.len());
@@ -335,7 +429,6 @@ mod tests {
     use rand::Rng;
 
     use super::*;
-    use crate::util::iter_util::ZipEqDebug;
 
     struct Test;
     impl VnodeMappingItem for Test {
diff --git a/src/common/src/hash/consistent_hash/vnode.rs b/src/common/src/hash/consistent_hash/vnode.rs
index 9bc49a9372ac0..fc6dac0978adf 100644
--- a/src/common/src/hash/consistent_hash/vnode.rs
+++ b/src/common/src/hash/consistent_hash/vnode.rs
@@ -169,7 +169,6 @@ mod tests {
     use super::*;
     use crate::array::DataChunkTestExt;
     use crate::row::OwnedRow;
-    use crate::types::ScalarImpl;
     use crate::util::row_id::RowIdGenerator;
 
     #[test]
diff --git a/src/common/src/hash/key.rs b/src/common/src/hash/key.rs
index c136710edb23f..c7e57173a3e74 100644
--- a/src/common/src/hash/key.rs
+++ b/src/common/src/hash/key.rs
@@ -21,7 +21,6 @@
 //! are encoded from both `t.b` and `t.c`. If `t.b="abc"` and `t.c=1`, the hashkey may be
 //! encoded in certain format of `("abc", 1)`.
 
-use std::convert::TryInto;
 use std::default::Default;
 use std::fmt::Debug;
 use std::hash::{BuildHasher, Hasher};
@@ -237,7 +236,7 @@ impl<T: BuildHasher> From<u64> for HashCode<T> {
 }
 
 impl<T: BuildHasher> HashCode<T> {
-    pub fn value(self) -> u64 {
+    pub fn value(&self) -> u64 {
         self.value
     }
 }
@@ -643,11 +642,9 @@ mod tests {
         DateArray, DecimalArray, F32Array, F64Array, I16Array, I32Array, I32ArrayBuilder, I64Array,
         TimeArray, TimestampArray, Utf8Array,
     };
-    use crate::hash::{
-        HashKey, Key128, Key16, Key256, Key32, Key64, KeySerialized, PrecomputedBuildHasher,
-    };
+    use crate::hash::{HashKey, Key128, Key16, Key256, Key32, Key64, KeySerialized};
     use crate::test_utils::rand_array::seed_rand_array_ref;
-    use crate::types::{DataType, Datum};
+    use crate::types::Datum;
 
     #[derive(Hash, PartialEq, Eq)]
     struct Row(Vec<Datum>);
diff --git a/src/common/src/hash/table_distribution.rs b/src/common/src/hash/table_distribution.rs
index c5bb2fd3d8dd2..c69c9e9a43aec 100644
--- a/src/common/src/hash/table_distribution.rs
+++ b/src/common/src/hash/table_distribution.rs
@@ -37,7 +37,7 @@ enum ComputeVnode {
         dist_key_in_pk_indices: Vec<usize>,
     },
     VnodeColumnIndex {
-        /// Indices of vnode columns.
+        /// Index of vnode column.
         vnode_col_idx_in_pk: usize,
     },
 }
@@ -87,7 +87,7 @@ impl TableDistribution {
 
         let vnodes = vnodes.unwrap_or_else(Self::singleton_vnode_bitmap);
         if let ComputeVnode::Singleton = &compute_vnode {
-            if &vnodes != Self::singleton_vnode_bitmap_ref() {
+            if &vnodes != Self::singleton_vnode_bitmap_ref() && &vnodes != Self::all_vnodes_ref() {
                 warn!(
                     ?vnodes,
                     "singleton distribution get non-singleton vnode bitmap"
diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs
index ea3cd4a4f7102..1cbb2d837aa78 100644
--- a/src/common/src/lib.rs
+++ b/src/common/src/lib.rs
@@ -34,16 +34,14 @@
 #![feature(inline_const_pat)]
 #![allow(incomplete_features)]
 #![feature(iterator_try_collect)]
-#![feature(round_ties_even)]
 #![feature(iter_order_by)]
 #![feature(exclusive_range_pattern)]
 #![feature(binary_heap_into_iter_sorted)]
 #![feature(impl_trait_in_assoc_type)]
 #![feature(map_entry_replace)]
 #![feature(negative_impls)]
-#![feature(bound_map)]
-#![feature(array_methods)]
 #![feature(register_tool)]
+#![feature(btree_cursors)]
 #![register_tool(rw)]
 
 #[cfg_attr(not(test), allow(unused_extern_crates))]
@@ -83,9 +81,11 @@ pub use risingwave_common_metrics::{
     register_guarded_histogram_vec_with_registry, register_guarded_int_counter_vec_with_registry,
     register_guarded_int_gauge_vec_with_registry,
 };
+pub mod lru;
 pub mod opts;
 pub mod range;
 pub mod row;
+pub mod sequence;
 pub mod session_config;
 pub mod system_param;
 pub mod telemetry;
diff --git a/src/common/src/lru.rs b/src/common/src/lru.rs
new file mode 100644
index 0000000000000..191c0cdf5579e
--- /dev/null
+++ b/src/common/src/lru.rs
@@ -0,0 +1,369 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::alloc::{Allocator, Global};
+use std::borrow::Borrow;
+use std::cell::RefCell;
+use std::hash::{BuildHasher, Hash};
+use std::mem::MaybeUninit;
+use std::ptr::NonNull;
+use std::sync::atomic::Ordering;
+
+pub use ahash::RandomState;
+use hashbrown::hash_table::Entry;
+use hashbrown::HashTable;
+
+use crate::sequence::{AtomicSequence, Sequence, Sequencer};
+
+thread_local! {
+    pub static SEQUENCER: RefCell<Sequencer> = const { RefCell::new(Sequencer::new(Sequencer::DEFAULT_STEP, Sequencer::DEFAULT_LAG)) };
+}
+
+static SEQUENCER_DEFAULT_STEP: AtomicSequence = AtomicSequence::new(Sequencer::DEFAULT_STEP);
+static SEQUENCER_DEFAULT_LAG: AtomicSequence = AtomicSequence::new(Sequencer::DEFAULT_LAG);
+
+pub fn init_global_sequencer_args(step: Sequence, lag: Sequence) {
+    SEQUENCER_DEFAULT_STEP.store(step, Ordering::Relaxed);
+    SEQUENCER_DEFAULT_LAG.store(lag, Ordering::Relaxed);
+}
+
+struct LruEntry<K, V>
+where
+    K: Hash + Eq,
+{
+    prev: Option<NonNull<LruEntry<K, V>>>,
+    next: Option<NonNull<LruEntry<K, V>>>,
+    key: MaybeUninit<K>,
+    value: MaybeUninit<V>,
+    hash: u64,
+    sequence: Sequence,
+}
+
+impl<K, V> LruEntry<K, V>
+where
+    K: Hash + Eq,
+{
+    fn key(&self) -> &K {
+        unsafe { self.key.assume_init_ref() }
+    }
+
+    fn value(&self) -> &V {
+        unsafe { self.value.assume_init_ref() }
+    }
+
+    fn value_mut(&mut self) -> &mut V {
+        unsafe { self.value.assume_init_mut() }
+    }
+}
+
+unsafe impl<K, V> Send for LruEntry<K, V> where K: Hash + Eq {}
+unsafe impl<K, V> Sync for LruEntry<K, V> where K: Hash + Eq {}
+
+pub struct LruCache<K, V, S = RandomState, A = Global>
+where
+    K: Hash + Eq,
+    S: BuildHasher + Send + Sync + 'static,
+    A: Clone + Allocator,
+{
+    map: HashTable<NonNull<LruEntry<K, V>>, A>,
+    /// dummy node of the lru linked list
+    dummy: Box<LruEntry<K, V>, A>,
+
+    alloc: A,
+    hash_builder: S,
+}
+
+unsafe impl<K, V, S, A> Send for LruCache<K, V, S, A>
+where
+    K: Hash + Eq,
+    S: BuildHasher + Send + Sync + 'static,
+    A: Clone + Allocator,
+{
+}
+unsafe impl<K, V, S, A> Sync for LruCache<K, V, S, A>
+where
+    K: Hash + Eq,
+    S: BuildHasher + Send + Sync + 'static,
+    A: Clone + Allocator,
+{
+}
+
+impl<K, V> LruCache<K, V>
+where
+    K: Hash + Eq,
+{
+    pub fn unbounded() -> Self {
+        Self::unbounded_with_hasher_in(RandomState::default(), Global)
+    }
+}
+
+impl<K, V, S, A> LruCache<K, V, S, A>
+where
+    K: Hash + Eq,
+    S: BuildHasher + Send + Sync + 'static,
+    A: Clone + Allocator,
+{
+    pub fn unbounded_with_hasher_in(hash_builder: S, alloc: A) -> Self {
+        let map = HashTable::new_in(alloc.clone());
+        let mut dummy = Box::new_in(
+            LruEntry {
+                prev: None,
+                next: None,
+                key: MaybeUninit::uninit(),
+                value: MaybeUninit::uninit(),
+                hash: 0,
+                sequence: Sequence::default(),
+            },
+            alloc.clone(),
+        );
+        let ptr = unsafe { NonNull::new_unchecked(dummy.as_mut() as *mut _) };
+        dummy.next = Some(ptr);
+        dummy.prev = Some(ptr);
+        Self {
+            map,
+            dummy,
+            alloc,
+            hash_builder,
+        }
+    }
+
+    pub fn put(&mut self, key: K, mut value: V) -> Option<V> {
+        unsafe {
+            let hash = self.hash_builder.hash_one(&key);
+
+            match self
+                .map
+                .entry(hash, |p| p.as_ref().key() == &key, |p| p.as_ref().hash)
+            {
+                Entry::Occupied(o) => {
+                    let mut ptr = *o.get();
+                    let entry = ptr.as_mut();
+                    std::mem::swap(&mut value, entry.value_mut());
+                    self.detach(ptr);
+                    self.attach(ptr);
+                    Some(value)
+                }
+                Entry::Vacant(v) => {
+                    let entry = Box::new_in(
+                        LruEntry {
+                            prev: None,
+                            next: None,
+                            key: MaybeUninit::new(key),
+                            value: MaybeUninit::new(value),
+                            hash,
+                            // sequence will be updated by `attach`
+                            sequence: 0,
+                        },
+                        self.alloc.clone(),
+                    );
+                    let ptr = NonNull::new_unchecked(Box::into_raw(entry));
+                    v.insert(ptr);
+                    self.attach(ptr);
+                    None
+                }
+            }
+        }
+    }
+
+    pub fn get<'a, Q>(&'a mut self, key: &Q) -> Option<&'a V>
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
+        unsafe {
+            let key = key.borrow();
+            let hash = self.hash_builder.hash_one(key);
+            if let Some(ptr) = self.map.find(hash, |p| p.as_ref().key().borrow() == key) {
+                let ptr = *ptr;
+                self.detach(ptr);
+                self.attach(ptr);
+                Some(ptr.as_ref().value())
+            } else {
+                None
+            }
+        }
+    }
+
+    pub fn get_mut<'a, Q>(&'a mut self, key: &Q) -> Option<&'a mut V>
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
+        unsafe {
+            let key = key.borrow();
+            let hash = self.hash_builder.hash_one(key);
+            if let Some(ptr) = self
+                .map
+                .find_mut(hash, |p| p.as_ref().key().borrow() == key)
+            {
+                let mut ptr = *ptr;
+                self.detach(ptr);
+                self.attach(ptr);
+                Some(ptr.as_mut().value_mut())
+            } else {
+                None
+            }
+        }
+    }
+
+    pub fn peek<'a, Q>(&'a self, key: &Q) -> Option<&'a V>
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
+        unsafe {
+            let key = key.borrow();
+            let hash = self.hash_builder.hash_one(key);
+            self.map
+                .find(hash, |p| p.as_ref().key().borrow() == key)
+                .map(|ptr| ptr.as_ref().value())
+        }
+    }
+
+    pub fn peek_mut<'a, Q>(&'a mut self, key: &Q) -> Option<&'a mut V>
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
+        unsafe {
+            let key = key.borrow();
+            let hash = self.hash_builder.hash_one(key);
+            self.map
+                .find(hash, |p| p.as_ref().key().borrow() == key)
+                .map(|ptr| ptr.clone().as_mut().value_mut())
+        }
+    }
+
+    pub fn contains<Q>(&self, key: &Q) -> bool
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq + ?Sized,
+    {
+        unsafe {
+            let key = key.borrow();
+            let hash = self.hash_builder.hash_one(key);
+            self.map
+                .find(hash, |p| p.as_ref().key().borrow() == key)
+                .is_some()
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.map.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Pop first entry if its sequence is less than the given sequence.
+    pub fn pop_with_sequence(&mut self, sequence: Sequence) -> Option<(K, V, Sequence)> {
+        unsafe {
+            if self.is_empty() {
+                return None;
+            }
+
+            let ptr = self.dummy.next.unwrap_unchecked();
+            if ptr.as_ref().sequence >= sequence {
+                return None;
+            }
+
+            self.detach(ptr);
+
+            let entry = Box::from_raw_in(ptr.as_ptr(), self.alloc.clone());
+
+            let key = entry.key.assume_init();
+            let value = entry.value.assume_init();
+            let sequence = entry.sequence;
+
+            let hash = self.hash_builder.hash_one(&key);
+
+            match self
+                .map
+                .entry(hash, |p| p.as_ref().key() == &key, |p| p.as_ref().hash)
+            {
+                Entry::Occupied(o) => {
+                    o.remove();
+                }
+                Entry::Vacant(_) => {}
+            }
+
+            Some((key, value, sequence))
+        }
+    }
+
+    pub fn clear(&mut self) {
+        unsafe {
+            let mut map = HashTable::new_in(self.alloc.clone());
+            std::mem::swap(&mut map, &mut self.map);
+
+            for ptr in map.drain() {
+                self.detach(ptr);
+                let mut entry = Box::from_raw_in(ptr.as_ptr(), self.alloc.clone());
+                entry.key.assume_init_drop();
+                entry.value.assume_init_drop();
+            }
+
+            debug_assert!(self.is_empty());
+            debug_assert_eq!(
+                self.dummy.as_mut() as *mut _,
+                self.dummy.next.unwrap_unchecked().as_ptr()
+            )
+        }
+    }
+
+    fn detach(&mut self, mut ptr: NonNull<LruEntry<K, V>>) {
+        unsafe {
+            let entry = ptr.as_mut();
+
+            debug_assert!(entry.prev.is_some() && entry.next.is_some());
+
+            entry.prev.unwrap_unchecked().as_mut().next = entry.next;
+            entry.next.unwrap_unchecked().as_mut().prev = entry.prev;
+
+            entry.next = None;
+            entry.prev = None;
+        }
+    }
+
+    fn attach(&mut self, mut ptr: NonNull<LruEntry<K, V>>) {
+        unsafe {
+            let entry = ptr.as_mut();
+
+            debug_assert!(entry.prev.is_none() && entry.next.is_none());
+
+            entry.next = Some(NonNull::new_unchecked(self.dummy.as_mut() as *mut _));
+            entry.prev = self.dummy.prev;
+
+            self.dummy.prev.unwrap_unchecked().as_mut().next = Some(ptr);
+            self.dummy.prev = Some(ptr);
+
+            entry.sequence = SEQUENCER.with(|s| s.borrow_mut().alloc());
+        }
+    }
+}
+
+impl<K, V, S, A> Drop for LruCache<K, V, S, A>
+where
+    K: Hash + Eq,
+    S: BuildHasher + Send + Sync + 'static,
+    A: Clone + Allocator,
+{
+    fn drop(&mut self) {
+        self.clear()
+    }
+}
+
+#[cfg(test)]
+mod tests {}
diff --git a/src/common/src/memory/alloc.rs b/src/common/src/memory/alloc.rs
index 81413a623239c..acff8f02ce631 100644
--- a/src/common/src/memory/alloc.rs
+++ b/src/common/src/memory/alloc.rs
@@ -43,13 +43,14 @@ impl MonitoredGlobalAlloc {
 unsafe impl<A: Allocator> Allocator for MonitoredAlloc<A> {
     fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
         let ret = self.alloc.allocate(layout)?;
+        // We don't throw an AllocError if the memory context is out of memory, otherwise the whole process will crash.
         self.ctx.add(layout.size() as i64);
         Ok(ret)
     }
 
     unsafe fn deallocate(&self, ptr: NonNull<u8>, layout: Layout) {
         self.alloc.deallocate(ptr, layout);
-        self.ctx.add(-(layout.size() as i64))
+        self.ctx.add(-(layout.size() as i64));
     }
 }
 
diff --git a/src/common/src/memory/mem_context.rs b/src/common/src/memory/mem_context.rs
index 5c538d9038fb4..a004b3e88b4b9 100644
--- a/src/common/src/memory/mem_context.rs
+++ b/src/common/src/memory/mem_context.rs
@@ -59,6 +59,7 @@ impl<const N: usize> MemCounter for LabelGuardedIntGauge<N> {
 struct MemoryContextInner {
     counter: Box<dyn MemCounter>,
     parent: Option<MemoryContext>,
+    mem_limit: u64,
 }
 
 #[derive(Clone)]
@@ -70,9 +71,22 @@ pub struct MemoryContext {
 
 impl MemoryContext {
     pub fn new(parent: Option<MemoryContext>, counter: impl MemCounter) -> Self {
+        let mem_limit = parent.as_ref().map_or_else(|| u64::MAX, |p| p.mem_limit());
+        Self::new_with_mem_limit(parent, counter, mem_limit)
+    }
+
+    pub fn new_with_mem_limit(
+        parent: Option<MemoryContext>,
+        counter: impl MemCounter,
+        mem_limit: u64,
+    ) -> Self {
         let c = Box::new(counter);
         Self {
-            inner: Some(Arc::new(MemoryContextInner { counter: c, parent })),
+            inner: Some(Arc::new(MemoryContextInner {
+                counter: c,
+                parent,
+                mem_limit,
+            })),
         }
     }
 
@@ -81,19 +95,28 @@ impl MemoryContext {
         Self { inner: None }
     }
 
-    pub fn root(counter: impl MemCounter) -> Self {
-        Self::new(None, counter)
+    pub fn root(counter: impl MemCounter, mem_limit: u64) -> Self {
+        Self::new_with_mem_limit(None, counter, mem_limit)
     }
 
     /// Add `bytes` memory usage. Pass negative value to decrease memory usage.
-    pub fn add(&self, bytes: i64) {
+    /// Returns `false` if the memory usage exceeds the limit.
+    pub fn add(&self, bytes: i64) -> bool {
         if let Some(inner) = &self.inner {
-            inner.counter.add(bytes);
-
+            if (inner.counter.get_bytes_used() + bytes) as u64 > inner.mem_limit {
+                return false;
+            }
             if let Some(parent) = &inner.parent {
-                parent.add(bytes);
+                if parent.add(bytes) {
+                    inner.counter.add(bytes);
+                } else {
+                    return false;
+                }
+            } else {
+                inner.counter.add(bytes);
             }
         }
+        true
     }
 
     pub fn get_bytes_used(&self) -> i64 {
@@ -104,6 +127,29 @@ impl MemoryContext {
         }
     }
 
+    pub fn mem_limit(&self) -> u64 {
+        if let Some(inner) = &self.inner {
+            inner.mem_limit
+        } else {
+            u64::MAX
+        }
+    }
+
+    /// Check if the memory usage exceeds the limit.
+    /// Returns `false` if the memory usage exceeds the limit.
+    pub fn check_memory_usage(&self) -> bool {
+        if let Some(inner) = &self.inner {
+            if inner.counter.get_bytes_used() as u64 > inner.mem_limit {
+                return false;
+            }
+            if let Some(parent) = &inner.parent {
+                return parent.check_memory_usage();
+            }
+        }
+
+        true
+    }
+
     /// Creates a new global allocator that reports memory usage to this context.
     pub fn global_allocator(&self) -> MonitoredGlobalAlloc {
         MonitoredGlobalAlloc::with_memory_context(self.clone())
@@ -113,7 +159,7 @@ impl MemoryContext {
 impl Drop for MemoryContextInner {
     fn drop(&mut self) {
         if let Some(p) = &self.parent {
-            p.add(-self.counter.get_bytes_used())
+            p.add(-self.counter.get_bytes_used());
         }
     }
 }
diff --git a/src/common/src/memory/monitored_heap.rs b/src/common/src/memory/monitored_heap.rs
index 58df24817a69b..5873194914edc 100644
--- a/src/common/src/memory/monitored_heap.rs
+++ b/src/common/src/memory/monitored_heap.rs
@@ -81,6 +81,10 @@ impl<T: Ord + EstimateSize> MemMonitoredHeap<T> {
         self.mem_ctx.add(-((old_cap * size_of::<T>()) as i64));
         ret
     }
+
+    pub fn mem_context(&self) -> &MemoryContext {
+        &self.mem_ctx
+    }
 }
 
 impl<T> Extend<T> for MemMonitoredHeap<T>
@@ -113,7 +117,7 @@ mod tests {
     #[test]
     fn test_heap() {
         let gauge = LabelGuardedIntGauge::<4>::test_int_gauge();
-        let mem_ctx = MemoryContext::root(gauge.clone());
+        let mem_ctx = MemoryContext::root(gauge.clone(), u64::MAX);
 
         let mut heap = MemMonitoredHeap::<u8>::new_with(mem_ctx);
         assert_eq!(0, gauge.get());
@@ -131,7 +135,7 @@ mod tests {
     #[test]
     fn test_heap_drop() {
         let gauge = LabelGuardedIntGauge::<4>::test_int_gauge();
-        let mem_ctx = MemoryContext::root(gauge.clone());
+        let mem_ctx = MemoryContext::root(gauge.clone(), u64::MAX);
 
         let vec = {
             let mut heap = MemMonitoredHeap::<u8>::new_with(mem_ctx);
diff --git a/src/common/src/row/owned_row.rs b/src/common/src/row/owned_row.rs
index 8ea3803538d84..15a01d192a64c 100644
--- a/src/common/src/row/owned_row.rs
+++ b/src/common/src/row/owned_row.rs
@@ -182,7 +182,7 @@ mod tests {
 
     use super::*;
     use crate::row::RowExt;
-    use crate::types::{DataType as Ty, Interval, ScalarImpl};
+    use crate::types::DataType as Ty;
     use crate::util::hash_util::Crc32FastBuilder;
 
     #[test]
diff --git a/src/common/src/sequence.rs b/src/common/src/sequence.rs
new file mode 100644
index 0000000000000..91f469cae9c8f
--- /dev/null
+++ b/src/common/src/sequence.rs
@@ -0,0 +1,86 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::atomic::{AtomicU64, Ordering};
+
+pub type Sequence = u64;
+pub type AtomicSequence = AtomicU64;
+
+pub static SEQUENCE_GLOBAL: AtomicSequence = AtomicSequence::new(0);
+
+/// An globally unique and approximate ascending sequence generator with local optimization.
+///
+/// [`Sequencer`] can be used to generate globally unique sequence (id) in an approximate order. [`Sequencer`] allow
+/// the generated sequence to be disordered in a certain window. The larger the allowed disordered sequence window is,
+/// the better multithreading performance of the generator will be.
+///
+/// The window is controlled with two arguments, `step` and `lag`. `step` controls the size of the batch of the
+/// sequences to allocate by the local sequence generator. `lag` controls the maximum lag between the local generator
+/// and the global generator to avoid skew.
+pub struct Sequencer {
+    local: Sequence,
+    target: Sequence,
+
+    step: Sequence,
+    lag: Sequence,
+}
+
+impl Sequencer {
+    pub const DEFAULT_LAG: Sequence = Self::DEFAULT_STEP * 32;
+    pub const DEFAULT_STEP: Sequence = 64;
+
+    /// Create a new local sequence generator.
+    pub const fn new(step: Sequence, lag: Sequence) -> Self {
+        Self {
+            local: 0,
+            target: 0,
+            step,
+            lag,
+        }
+    }
+
+    /// Get the global sequence to allocate.
+    pub fn global(&self) -> Sequence {
+        SEQUENCE_GLOBAL.load(Ordering::Relaxed)
+    }
+
+    /// Get the local sequence to allocate.
+    pub fn local(&self) -> Sequence {
+        self.local
+    }
+
+    /// Allocate a new sequence. The allocated sequences from the same [`Sequencer`] are strictly ascending, the
+    /// allocated sequences from different [`Sequencer`]s are approximate ascending.
+    pub fn alloc(&mut self) -> Sequence {
+        self.try_alloc();
+        let res = self.local;
+        self.local += 1;
+        res
+    }
+
+    #[inline(always)]
+    fn try_alloc(&mut self) {
+        if self.local == self.target
+            || self.local + self.lag < SEQUENCE_GLOBAL.load(Ordering::Relaxed)
+        {
+            self.alloc_inner()
+        }
+    }
+
+    #[inline(always)]
+    fn alloc_inner(&mut self) {
+        self.local = SEQUENCE_GLOBAL.fetch_add(self.step, Ordering::Relaxed);
+        self.target = self.local + self.step;
+    }
+}
diff --git a/src/common/src/session_config/mod.rs b/src/common/src/session_config/mod.rs
index 4ee7617ee751d..8fbfd2cd6e969 100644
--- a/src/common/src/session_config/mod.rs
+++ b/src/common/src/session_config/mod.rs
@@ -152,7 +152,11 @@ pub struct SessionConfig {
     #[parameter(default = true, rename = "rw_streaming_enable_bushy_join")]
     streaming_enable_bushy_join: bool,
 
-    /// Enable arrangement backfill for streaming queries. Defaults to false.
+    /// Enable arrangement backfill for streaming queries. Defaults to true.
+    /// When set to true, the parallelism of the upstream fragment will be
+    /// decoupled from the parallelism of the downstream scan fragment.
+    /// Or more generally, the parallelism of the upstream table / index / mv
+    /// will be decoupled from the parallelism of the downstream table / index / mv / sink.
     #[parameter(default = true)]
     streaming_use_arrangement_backfill: bool,
 
@@ -236,6 +240,10 @@ pub struct SessionConfig {
     #[parameter(default = 0)]
     lock_timeout: i32,
 
+    /// For limiting the startup time of a shareable CDC streaming source when the source is being created. Unit: seconds.
+    #[parameter(default = 30)]
+    cdc_source_wait_streaming_start_timeout: i32,
+
     /// see <https://www.postgresql.org/docs/current/runtime-config-client.html#GUC-ROW-SECURITY>.
     /// Unused in RisingWave, support for compatibility.
     #[parameter(default = true)]
diff --git a/src/common/src/session_config/search_path.rs b/src/common/src/session_config/search_path.rs
index 4573294ab24c4..7a99f07833b96 100644
--- a/src/common/src/session_config/search_path.rs
+++ b/src/common/src/session_config/search_path.rs
@@ -23,12 +23,12 @@ pub const USER_NAME_WILD_CARD: &str = "\"$user\"";
 /// see <https://www.postgresql.org/docs/14/runtime-config-client.html#GUC-SEARCH-PATH>
 ///
 /// 1. when we `select` or `drop` object and don't give a specified schema, it will search the
-/// object from the valid items in schema `rw_catalog`, `pg_catalog` and `search_path`. If schema
-/// `rw_catalog` and `pg_catalog` are not in `search_path`, we will search them firstly. If they're
-/// in `search_path`, we will follow the order in `search_path`.
+///    object from the valid items in schema `rw_catalog`, `pg_catalog` and `search_path`. If schema
+///    `rw_catalog` and `pg_catalog` are not in `search_path`, we will search them firstly. If they're
+///    in `search_path`, we will follow the order in `search_path`.
 ///
 /// 2. when we `create` a `source` or `mv` and don't give a specified schema, it will use the first
-/// valid schema in `search_path`.
+///    valid schema in `search_path`.
 ///
 /// 3. when we `create` a `index` or `sink`, it will use the schema of the associated table.
 #[derive(Clone, Debug, PartialEq)]
diff --git a/src/common/src/system_param/mod.rs b/src/common/src/system_param/mod.rs
index 06d4cce2e4e6b..f71324cb4e55d 100644
--- a/src/common/src/system_param/mod.rs
+++ b/src/common/src/system_param/mod.rs
@@ -77,10 +77,10 @@ macro_rules! for_all_params {
             { barrier_interval_ms,                      u32,    Some(1000_u32),                             true,   "The interval of periodic barrier.", },
             { checkpoint_frequency,                     u64,    Some(1_u64),                                true,   "There will be a checkpoint for every n barriers.", },
             { sstable_size_mb,                          u32,    Some(256_u32),                              false,  "Target size of the Sstable.", },
-            { parallel_compact_size_mb,                 u32,    Some(512_u32),                              false,  "", },
+            { parallel_compact_size_mb,                 u32,    Some(512_u32),                              false,  "The size of parallel task for one compact/flush job.", },
             { block_size_kb,                            u32,    Some(64_u32),                               false,  "Size of each block in bytes in SST.", },
             { bloom_false_positive,                     f64,    Some(0.001_f64),                            false,  "False positive probability of bloom filter.", },
-            { state_store,                              String, None,                                       false,  "", },
+            { state_store,                              String, None,                                       false,  "URL for the state store", },
             { data_directory,                           String, None,                                       false,  "Remote directory for storing data and metadata objects.", },
             { backup_storage_url,                       String, None,                                       true,   "Remote storage url for storing snapshots.", },
             { backup_storage_directory,                 String, None,                                       true,   "Remote directory for storing snapshots.", },
diff --git a/src/common/src/telemetry/mod.rs b/src/common/src/telemetry/mod.rs
index 54a88789a171f..0fbd526692da0 100644
--- a/src/common/src/telemetry/mod.rs
+++ b/src/common/src/telemetry/mod.rs
@@ -16,6 +16,7 @@ pub mod manager;
 pub mod pb_compatible;
 pub mod report;
 
+use std::env;
 use std::time::SystemTime;
 
 use serde::{Deserialize, Serialize};
@@ -25,6 +26,11 @@ use thiserror_ext::AsReport;
 use crate::util::env_var::env_var_is_true_or;
 use crate::util::resource_util::cpu::total_cpu_available;
 use crate::util::resource_util::memory::{system_memory_available_bytes, total_memory_used_bytes};
+use crate::RW_VERSION;
+
+pub const TELEMETRY_CLUSTER_TYPE: &str = "RW_TELEMETRY_TYPE";
+const TELEMETRY_CLUSTER_TYPE_HOSTED: &str = "hosted"; // hosted on RisingWave Cloud
+const TELEMETRY_CLUSTER_TYPE_TEST: &str = "test"; // test environment, eg. CI & Risedev
 
 /// Url of telemetry backend
 pub const TELEMETRY_REPORT_URL: &str = "https://telemetry.risingwave.dev/api/v2/report";
@@ -159,6 +165,34 @@ pub fn current_timestamp() -> u64 {
         .as_secs()
 }
 
+pub fn report_scarf_enabled() -> bool {
+    env::var(TELEMETRY_CLUSTER_TYPE)
+        .map(|deploy_type| {
+            !(deploy_type.eq_ignore_ascii_case(TELEMETRY_CLUSTER_TYPE_HOSTED)
+                || deploy_type.eq_ignore_ascii_case(TELEMETRY_CLUSTER_TYPE_TEST))
+        })
+        .unwrap_or(true)
+}
+
+// impl logic to report to Scarf service, containing RW version and deployment platform
+pub async fn report_to_scarf() {
+    let request_url = format!(
+        "https://risingwave.gateway.scarf.sh/telemetry/{}/{}",
+        RW_VERSION,
+        System::name().unwrap_or_default()
+    );
+    // keep trying every 1h until success
+    loop {
+        let res = reqwest::get(&request_url).await;
+        if let Ok(res) = res {
+            if res.status().is_success() {
+                break;
+            }
+        }
+        tokio::time::sleep(tokio::time::Duration::from_secs(3600)).await;
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/src/common/src/types/cow.rs b/src/common/src/types/cow.rs
new file mode 100644
index 0000000000000..88eebfe1e4c2a
--- /dev/null
+++ b/src/common/src/types/cow.rs
@@ -0,0 +1,80 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{Datum, DatumRef, ToDatumRef, ToOwnedDatum};
+
+/// 🐮 A borrowed [`DatumRef`] or an owned [`Datum`].
+///
+/// We do not use [`std::borrow::Cow`] because it requires the borrowed variant
+/// to be a reference, whereas what we have is a [`DatumRef`] with a lifetime.
+///
+/// # Usage
+///
+/// Generally, you don't need to match on the variants of `DatumCow` to access
+/// the underlying datum. Instead, you can...
+///
+/// - call [`to_datum_ref`](ToDatumRef::to_datum_ref) to get a borrowed
+///   [`DatumRef`] without any allocation, which can be used to append to an
+///   array builder or to encode into the storage representation,
+///
+/// - call [`to_owned_datum`](ToOwnedDatum::to_owned_datum) to get an owned
+///   [`Datum`] with potentially an allocation, which can be used to store in a
+///   struct without lifetime constraints.
+#[derive(Debug, Clone)]
+pub enum DatumCow<'a> {
+    Borrowed(DatumRef<'a>),
+    Owned(Datum),
+}
+
+impl PartialEq for DatumCow<'_> {
+    fn eq(&self, other: &Self) -> bool {
+        self.to_datum_ref() == other.to_datum_ref()
+    }
+}
+impl Eq for DatumCow<'_> {}
+
+impl From<Datum> for DatumCow<'_> {
+    fn from(datum: Datum) -> Self {
+        DatumCow::Owned(datum)
+    }
+}
+
+impl<'a> From<DatumRef<'a>> for DatumCow<'a> {
+    fn from(datum: DatumRef<'a>) -> Self {
+        DatumCow::Borrowed(datum)
+    }
+}
+
+impl ToDatumRef for DatumCow<'_> {
+    fn to_datum_ref(&self) -> DatumRef<'_> {
+        match self {
+            DatumCow::Borrowed(datum) => *datum,
+            DatumCow::Owned(datum) => datum.to_datum_ref(),
+        }
+    }
+}
+
+impl ToOwnedDatum for DatumCow<'_> {
+    fn to_owned_datum(self) -> Datum {
+        match self {
+            DatumCow::Borrowed(datum) => datum.to_owned_datum(),
+            DatumCow::Owned(datum) => datum,
+        }
+    }
+}
+
+impl DatumCow<'_> {
+    /// Equivalent to `DatumCow::Owned(Datum::None)`.
+    pub const NULL: DatumCow<'static> = DatumCow::Owned(None);
+}
diff --git a/src/common/src/types/datetime.rs b/src/common/src/types/datetime.rs
index 9199dd41b6e5b..7058d36ec6fd5 100644
--- a/src/common/src/types/datetime.rs
+++ b/src/common/src/types/datetime.rs
@@ -22,7 +22,7 @@ use std::str::FromStr;
 
 use bytes::{Bytes, BytesMut};
 use chrono::{Datelike, Days, Duration, NaiveDate, NaiveDateTime, NaiveTime, Timelike, Weekday};
-use postgres_types::{accepts, to_sql_checked, IsNull, ToSql, Type};
+use postgres_types::{accepts, to_sql_checked, FromSql, IsNull, ToSql, Type};
 use risingwave_common_estimate_size::ZeroHeapSize;
 use thiserror::Error;
 
@@ -88,6 +88,20 @@ impl ToSql for Date {
     }
 }
 
+impl<'a> FromSql<'a> for Date {
+    fn from_sql(
+        ty: &Type,
+        raw: &'a [u8],
+    ) -> std::result::Result<Self, Box<dyn std::error::Error + Sync + Send>> {
+        let instant = NaiveDate::from_sql(ty, raw)?;
+        Ok(Self::from(instant))
+    }
+
+    fn accepts(ty: &Type) -> bool {
+        matches!(*ty, Type::DATE)
+    }
+}
+
 impl ToSql for Time {
     accepts!(TIME);
 
@@ -105,6 +119,20 @@ impl ToSql for Time {
     }
 }
 
+impl<'a> FromSql<'a> for Time {
+    fn from_sql(
+        ty: &Type,
+        raw: &'a [u8],
+    ) -> std::result::Result<Self, Box<dyn std::error::Error + Sync + Send>> {
+        let instant = NaiveTime::from_sql(ty, raw)?;
+        Ok(Self::from(instant))
+    }
+
+    fn accepts(ty: &Type) -> bool {
+        matches!(*ty, Type::TIME)
+    }
+}
+
 impl ToSql for Timestamp {
     accepts!(TIMESTAMP);
 
@@ -122,6 +150,20 @@ impl ToSql for Timestamp {
     }
 }
 
+impl<'a> FromSql<'a> for Timestamp {
+    fn from_sql(
+        ty: &Type,
+        raw: &'a [u8],
+    ) -> std::result::Result<Self, Box<dyn std::error::Error + Sync + Send>> {
+        let instant = NaiveDateTime::from_sql(ty, raw)?;
+        Ok(Self::from(instant))
+    }
+
+    fn accepts(ty: &Type) -> bool {
+        matches!(*ty, Type::TIMESTAMP)
+    }
+}
+
 /// Parse a date from varchar.
 ///
 /// # Example
diff --git a/src/common/src/types/decimal.rs b/src/common/src/types/decimal.rs
index b6031a1665540..b38dbb4822e68 100644
--- a/src/common/src/types/decimal.rs
+++ b/src/common/src/types/decimal.rs
@@ -13,14 +13,15 @@
 // limitations under the License.
 
 use std::fmt::Debug;
-use std::io::{Read, Write};
+use std::io::{Cursor, Read, Write};
 use std::ops::{Add, Div, Mul, Neg, Rem, Sub};
 
+use byteorder::{BigEndian, ReadBytesExt};
 use bytes::{BufMut, Bytes, BytesMut};
 use num_traits::{
     CheckedAdd, CheckedDiv, CheckedMul, CheckedNeg, CheckedRem, CheckedSub, Num, One, Zero,
 };
-use postgres_types::{accepts, to_sql_checked, IsNull, ToSql, Type};
+use postgres_types::{accepts, to_sql_checked, FromSql, IsNull, ToSql, Type};
 use risingwave_common_estimate_size::ZeroHeapSize;
 use rust_decimal::prelude::FromStr;
 use rust_decimal::{Decimal as RustDecimal, Error, MathematicalOps as _, RoundingStrategy};
@@ -145,6 +146,28 @@ impl ToSql for Decimal {
     }
 }
 
+impl<'a> FromSql<'a> for Decimal {
+    fn from_sql(
+        ty: &Type,
+        raw: &'a [u8],
+    ) -> Result<Self, Box<dyn std::error::Error + 'static + Sync + Send>> {
+        let mut rdr = Cursor::new(raw);
+        let _n_digits = rdr.read_u16::<BigEndian>()?;
+        let _weight = rdr.read_i16::<BigEndian>()?;
+        let sign = rdr.read_u16::<BigEndian>()?;
+        match sign {
+            0xC000 => Ok(Self::NaN),
+            0xD000 => Ok(Self::PositiveInf),
+            0xF000 => Ok(Self::NegativeInf),
+            _ => RustDecimal::from_sql(ty, raw).map(Self::Normalized),
+        }
+    }
+
+    fn accepts(ty: &Type) -> bool {
+        matches!(*ty, Type::NUMERIC)
+    }
+}
+
 macro_rules! impl_convert_int {
     ($T:ty) => {
         impl core::convert::From<$T> for Decimal {
diff --git a/src/common/src/types/from_sql.rs b/src/common/src/types/from_sql.rs
new file mode 100644
index 0000000000000..ba1d49892c602
--- /dev/null
+++ b/src/common/src/types/from_sql.rs
@@ -0,0 +1,66 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use postgres_types::{FromSql, Type};
+use risingwave_common::types::{
+    Date, Interval, JsonbVal, ScalarImpl, Time, Timestamp, Timestamptz,
+};
+
+impl<'a> FromSql<'a> for ScalarImpl {
+    fn from_sql(
+        ty: &Type,
+        raw: &'a [u8],
+    ) -> Result<Self, Box<dyn std::error::Error + Sync + Send>> {
+        Ok(match *ty {
+            Type::BOOL => ScalarImpl::from(bool::from_sql(ty, raw)?),
+            Type::INT2 => ScalarImpl::from(i16::from_sql(ty, raw)?),
+            Type::INT4 => ScalarImpl::from(i32::from_sql(ty, raw)?),
+            Type::INT8 => ScalarImpl::from(i64::from_sql(ty, raw)?),
+            Type::FLOAT4 => ScalarImpl::from(f32::from_sql(ty, raw)?),
+            Type::FLOAT8 => ScalarImpl::from(f64::from_sql(ty, raw)?),
+            Type::DATE => ScalarImpl::from(Date::from_sql(ty, raw)?),
+            Type::TIME => ScalarImpl::from(Time::from_sql(ty, raw)?),
+            Type::TIMESTAMP => ScalarImpl::from(Timestamp::from_sql(ty, raw)?),
+            Type::TIMESTAMPTZ => ScalarImpl::from(Timestamptz::from_sql(ty, raw)?),
+            Type::JSONB => ScalarImpl::from(JsonbVal::from_sql(ty, raw)?),
+            Type::INTERVAL => ScalarImpl::from(Interval::from_sql(ty, raw)?),
+            Type::BYTEA => ScalarImpl::from(Vec::<u8>::from_sql(ty, raw)?.into_boxed_slice()),
+            Type::VARCHAR | Type::TEXT => ScalarImpl::from(String::from_sql(ty, raw)?),
+            // Serial, Int256, Struct, List and Decimal are not supported here
+            // Note: The Decimal type is specially handled in the `ScalarAdapter`.
+            _ => bail_not_implemented!("the postgres decoding for {ty} is unsupported"),
+        })
+    }
+
+    fn accepts(ty: &Type) -> bool {
+        matches!(
+            *ty,
+            Type::BOOL
+                | Type::INT2
+                | Type::INT4
+                | Type::INT8
+                | Type::FLOAT4
+                | Type::FLOAT8
+                | Type::DATE
+                | Type::TIME
+                | Type::TIMESTAMP
+                | Type::TIMESTAMPTZ
+                | Type::JSONB
+                | Type::INTERVAL
+                | Type::BYTEA
+                | Type::VARCHAR
+                | Type::TEXT
+        )
+    }
+}
diff --git a/src/common/src/types/interval.rs b/src/common/src/types/interval.rs
index 4436181059d78..495561561e937 100644
--- a/src/common/src/types/interval.rs
+++ b/src/common/src/types/interval.rs
@@ -22,15 +22,12 @@ use std::sync::LazyLock;
 
 use byteorder::{BigEndian, NetworkEndian, ReadBytesExt, WriteBytesExt};
 use bytes::BytesMut;
-use chrono::Timelike;
 use num_traits::{CheckedAdd, CheckedNeg, CheckedSub, Zero};
-use postgres_types::{to_sql_checked, FromSql};
+use postgres_types::to_sql_checked;
 use regex::Regex;
-use risingwave_common_estimate_size::ZeroHeapSize;
 use risingwave_pb::data::PbInterval;
 use rust_decimal::prelude::Decimal;
 
-use super::to_binary::ToBinary;
 use super::*;
 
 /// Every interval can be represented by a `Interval`.
diff --git a/src/common/src/types/jsonb.rs b/src/common/src/types/jsonb.rs
index 522ec788d8646..642b363a8c67e 100644
--- a/src/common/src/types/jsonb.rs
+++ b/src/common/src/types/jsonb.rs
@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::fmt;
+use std::fmt::{self, Write};
 use std::hash::Hash;
 
-use bytes::Buf;
+use bytes::{Buf, BufMut, BytesMut};
 use jsonbb::{Value, ValueRef};
+use postgres_types::{accepts, to_sql_checked, FromSql, IsNull, ToSql, Type};
 use risingwave_common_estimate_size::EstimateSize;
 
 use super::{Datum, IntoOrdered, ListValue, ScalarImpl, StructRef, ToOwnedDatum, F64};
@@ -238,10 +239,15 @@ impl<'a> JsonbRef<'a> {
     }
 
     /// Returns a jsonb `null` value.
-    pub fn null() -> Self {
+    pub const fn null() -> Self {
         Self(ValueRef::Null)
     }
 
+    /// Returns a value for empty string.
+    pub const fn empty_string() -> Self {
+        Self(ValueRef::String(""))
+    }
+
     /// Returns true if this is a jsonb `null`.
     pub fn is_jsonb_null(&self) -> bool {
         self.0.is_null()
@@ -539,3 +545,38 @@ impl<F: std::fmt::Write> std::io::Write for FmtToIoUnchecked<F> {
         Ok(())
     }
 }
+
+impl ToSql for JsonbVal {
+    accepts!(JSONB);
+
+    to_sql_checked!();
+
+    fn to_sql(
+        &self,
+        _ty: &Type,
+        out: &mut BytesMut,
+    ) -> Result<IsNull, Box<dyn std::error::Error + Sync + Send>>
+    where
+        Self: Sized,
+    {
+        out.put_u8(1);
+        write!(out, "{}", self.0).unwrap();
+        Ok(IsNull::No)
+    }
+}
+
+impl<'a> FromSql<'a> for JsonbVal {
+    fn from_sql(
+        _ty: &Type,
+        mut raw: &'a [u8],
+    ) -> Result<Self, Box<dyn std::error::Error + Sync + Send>> {
+        if raw.is_empty() || raw.get_u8() != 1 {
+            return Err("invalid jsonb encoding".into());
+        }
+        Ok(JsonbVal::from(Value::from_text(raw)?))
+    }
+
+    fn accepts(ty: &Type) -> bool {
+        matches!(*ty, Type::JSONB)
+    }
+}
diff --git a/src/common/src/types/mod.rs b/src/common/src/types/mod.rs
index 9364f438cfacf..d793561f5957c 100644
--- a/src/common/src/types/mod.rs
+++ b/src/common/src/types/mod.rs
@@ -17,7 +17,6 @@
 // NOTE: When adding or modifying data types, remember to update the type matrix in
 // src/expr/macro/src/types.rs
 
-use std::convert::TryFrom;
 use std::fmt::Debug;
 use std::hash::Hash;
 use std::str::FromStr;
@@ -46,9 +45,11 @@ use crate::{
     for_all_scalar_variants, for_all_type_pairs,
 };
 
+mod cow;
 mod datetime;
 mod decimal;
 mod fields;
+mod from_sql;
 mod interval;
 mod jsonb;
 mod macros;
@@ -72,6 +73,7 @@ mod with_data_type;
 pub use fields::Fields;
 pub use risingwave_fields_derive::Fields;
 
+pub use self::cow::DatumCow;
 pub use self::datetime::{Date, Time, Timestamp};
 pub use self::decimal::{Decimal, PowError as DecimalPowError};
 pub use self::interval::{test_utils, DateTimeField, Interval, IntervalDisplay};
diff --git a/src/common/src/types/num256.rs b/src/common/src/types/num256.rs
index 43c97fd8b155a..2acbcf636a037 100644
--- a/src/common/src/types/num256.rs
+++ b/src/common/src/types/num256.rs
@@ -335,7 +335,6 @@ impl EstimateSize for Int256 {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::types::F64;
 
     macro_rules! check_op {
         ($t:ty, $lhs:expr, $rhs:expr, [$($op:tt),+]) => {
diff --git a/src/common/src/types/ordered.rs b/src/common/src/types/ordered.rs
index 77a0588e06672..bb2b0c9f316c1 100644
--- a/src/common/src/types/ordered.rs
+++ b/src/common/src/types/ordered.rs
@@ -14,7 +14,7 @@
 
 //! `ScalarImpl` and `Datum` wrappers that implement `PartialOrd` and `Ord` with default order type.
 
-use std::cmp::{Ord, Ordering};
+use std::cmp::Ordering;
 use std::ops::Deref;
 
 use risingwave_common_estimate_size::EstimateSize;
diff --git a/src/common/src/types/scalar_impl.rs b/src/common/src/types/scalar_impl.rs
index db2732ca67c48..d222473fa6bbb 100644
--- a/src/common/src/types/scalar_impl.rs
+++ b/src/common/src/types/scalar_impl.rs
@@ -15,8 +15,6 @@
 use std::hash::Hasher;
 
 use super::*;
-use crate::array::list_array::{ListRef, ListValue};
-use crate::array::struct_array::{StructRef, StructValue};
 use crate::{dispatch_scalar_ref_variants, dispatch_scalar_variants, for_all_native_types};
 
 /// `ScalarPartialOrd` allows comparison between `Scalar` and `ScalarRef`.
diff --git a/src/common/src/types/timestamptz.rs b/src/common/src/types/timestamptz.rs
index 29369d36e9757..feafee6e212bf 100644
--- a/src/common/src/types/timestamptz.rs
+++ b/src/common/src/types/timestamptz.rs
@@ -19,7 +19,7 @@ use std::str::FromStr;
 use bytes::{Bytes, BytesMut};
 use chrono::{DateTime, Datelike, TimeZone, Utc};
 use chrono_tz::Tz;
-use postgres_types::{accepts, to_sql_checked, IsNull, ToSql, Type};
+use postgres_types::{accepts, to_sql_checked, FromSql, IsNull, ToSql, Type};
 use risingwave_common_estimate_size::ZeroHeapSize;
 use serde::{Deserialize, Serialize};
 
@@ -51,6 +51,20 @@ impl ToSql for Timestamptz {
     }
 }
 
+impl<'a> FromSql<'a> for Timestamptz {
+    fn from_sql(
+        ty: &Type,
+        raw: &'a [u8],
+    ) -> Result<Self, Box<dyn std::error::Error + Sync + Send>> {
+        let instant = DateTime::<Utc>::from_sql(ty, raw)?;
+        Ok(Self::from(instant))
+    }
+
+    fn accepts(ty: &Type) -> bool {
+        matches!(*ty, Type::TIMESTAMPTZ)
+    }
+}
+
 impl ToBinary for Timestamptz {
     fn to_binary_with_type(&self, _ty: &DataType) -> super::to_binary::Result<Option<Bytes>> {
         let instant = self.to_datetime_utc();
@@ -130,6 +144,7 @@ impl Timestamptz {
 
     pub fn lookup_time_zone(time_zone: &str) -> std::result::Result<Tz, String> {
         Tz::from_str_insensitive(time_zone)
+            .map_err(|_| format!("'{time_zone}' is not a valid timezone"))
     }
 
     pub fn from_protobuf(timestamp_micros: i64) -> ArrayResult<Self> {
diff --git a/src/common/src/types/to_sql.rs b/src/common/src/types/to_sql.rs
index 71957b3bf35c8..3ece8a574c450 100644
--- a/src/common/src/types/to_sql.rs
+++ b/src/common/src/types/to_sql.rs
@@ -15,11 +15,11 @@
 use std::error::Error;
 
 use bytes::BytesMut;
-use postgres_types::{accepts, to_sql_checked, IsNull, ToSql, Type};
+use postgres_types::{to_sql_checked, IsNull, ToSql, Type};
 
-use crate::types::{JsonbRef, ScalarRefImpl};
+use crate::types::ScalarImpl;
 
-impl ToSql for ScalarRefImpl<'_> {
+impl ToSql for ScalarImpl {
     to_sql_checked!();
 
     fn to_sql(&self, ty: &Type, out: &mut BytesMut) -> Result<IsNull, Box<dyn Error + Sync + Send>>
@@ -27,25 +27,23 @@ impl ToSql for ScalarRefImpl<'_> {
         Self: Sized,
     {
         match self {
-            ScalarRefImpl::Int16(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Int32(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Int64(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Serial(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Float32(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Float64(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Utf8(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Bool(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Decimal(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Interval(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Date(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Timestamp(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Timestamptz(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Time(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Bytea(v) => v.to_sql(ty, out),
-            ScalarRefImpl::Jsonb(_) // jsonbb::Value doesn't implement ToSql yet
-            | ScalarRefImpl::Int256(_)
-            | ScalarRefImpl::Struct(_)
-            | ScalarRefImpl::List(_) => {
+            ScalarImpl::Int16(v) => v.to_sql(ty, out),
+            ScalarImpl::Int32(v) => v.to_sql(ty, out),
+            ScalarImpl::Int64(v) => v.to_sql(ty, out),
+            ScalarImpl::Serial(v) => v.to_sql(ty, out),
+            ScalarImpl::Float32(v) => v.to_sql(ty, out),
+            ScalarImpl::Float64(v) => v.to_sql(ty, out),
+            ScalarImpl::Utf8(v) => v.to_sql(ty, out),
+            ScalarImpl::Bool(v) => v.to_sql(ty, out),
+            ScalarImpl::Decimal(v) => v.to_sql(ty, out),
+            ScalarImpl::Interval(v) => v.to_sql(ty, out),
+            ScalarImpl::Date(v) => v.to_sql(ty, out),
+            ScalarImpl::Timestamp(v) => v.to_sql(ty, out),
+            ScalarImpl::Timestamptz(v) => v.to_sql(ty, out),
+            ScalarImpl::Time(v) => v.to_sql(ty, out),
+            ScalarImpl::Bytea(v) => (&**v).to_sql(ty, out),
+            ScalarImpl::Jsonb(v) => v.to_sql(ty, out),
+            ScalarImpl::Int256(_) | ScalarImpl::Struct(_) | ScalarImpl::List(_) => {
                 bail_not_implemented!("the postgres encoding for {ty} is unsupported")
             }
         }
@@ -59,18 +57,3 @@ impl ToSql for ScalarRefImpl<'_> {
         true
     }
 }
-
-impl ToSql for JsonbRef<'_> {
-    accepts!(JSONB);
-
-    to_sql_checked!();
-
-    fn to_sql(&self, _: &Type, out: &mut BytesMut) -> Result<IsNull, Box<dyn Error + Sync + Send>>
-    where
-        Self: Sized,
-    {
-        let buf = self.value_serialize();
-        out.extend(buf);
-        Ok(IsNull::No)
-    }
-}
diff --git a/src/common/src/types/to_text.rs b/src/common/src/types/to_text.rs
index d7722aade977b..ca140b93c37b7 100644
--- a/src/common/src/types/to_text.rs
+++ b/src/common/src/types/to_text.rs
@@ -47,15 +47,18 @@ pub trait ToText {
     /// - `ScalarRefImpl::Float32` -> `DataType::Float32`
     /// - `ScalarRefImpl::Float64` -> `DataType::Float64`
     /// - `ScalarRefImpl::Decimal` -> `DataType::Decimal`
-    /// - `ScalarRefImpl::Boolean` -> `DataType::Boolean`
+    /// - `ScalarRefImpl::Bool` -> `DataType::Boolean`
     /// - `ScalarRefImpl::Utf8` -> `DataType::Varchar`
     /// - `ScalarRefImpl::Bytea` -> `DataType::Bytea`
     /// - `ScalarRefImpl::Date` -> `DataType::Date`
     /// - `ScalarRefImpl::Time` -> `DataType::Time`
     /// - `ScalarRefImpl::Timestamp` -> `DataType::Timestamp`
+    /// - `ScalarRefImpl::Timestamptz` -> `DataType::Timestamptz`
     /// - `ScalarRefImpl::Interval` -> `DataType::Interval`
+    /// - `ScalarRefImpl::Jsonb` -> `DataType::Jsonb`
     /// - `ScalarRefImpl::List` -> `DataType::List`
     /// - `ScalarRefImpl::Struct` -> `DataType::Struct`
+    /// - `ScalarRefImpl::Serial` -> `DataType::Serial`
     fn to_text(&self) -> String {
         let mut s = String::new();
         self.write(&mut s).unwrap();
diff --git a/src/common/src/util/chunk_coalesce.rs b/src/common/src/util/chunk_coalesce.rs
index 5cdeb8026a0f1..94ca4cd783449 100644
--- a/src/common/src/util/chunk_coalesce.rs
+++ b/src/common/src/util/chunk_coalesce.rs
@@ -459,13 +459,13 @@ mod tests {
 
         let mut left_array_builder = DataType::Int32.create_array_builder(5);
         for v in [1, 2, 3, 4, 5] {
-            left_array_builder.append(&Some(ScalarImpl::Int32(v)));
+            left_array_builder.append(Some(ScalarImpl::Int32(v)));
         }
         let left_arrays = [left_array_builder.finish()];
 
         let mut right_array_builder = DataType::Int64.create_array_builder(5);
         for v in [5, 4, 3, 2, 1] {
-            right_array_builder.append(&Some(ScalarImpl::Int64(v)));
+            right_array_builder.append(Some(ScalarImpl::Int64(v)));
         }
         let right_arrays = [right_array_builder.finish()];
 
diff --git a/src/common/src/util/memcmp_encoding.rs b/src/common/src/util/memcmp_encoding.rs
index 5709d42d226f8..5a5ad598093af 100644
--- a/src/common/src/util/memcmp_encoding.rs
+++ b/src/common/src/util/memcmp_encoding.rs
@@ -333,15 +333,12 @@ pub fn decode_row(
 mod tests {
     use std::ops::Neg;
 
-    use itertools::Itertools;
     use rand::thread_rng;
 
     use super::*;
-    use crate::array::{DataChunk, ListValue, StructValue};
-    use crate::row::{OwnedRow, RowExt};
-    use crate::types::{DataType, FloatExt, ScalarImpl, F32};
-    use crate::util::iter_util::ZipEqFast;
-    use crate::util::sort_util::{ColumnOrder, OrderType};
+    use crate::array::{ListValue, StructValue};
+    use crate::row::RowExt;
+    use crate::types::FloatExt;
 
     #[test]
     fn test_memcomparable() {
@@ -546,7 +543,7 @@ mod tests {
         use rand::seq::SliceRandom;
 
         fn serialize(f: F32) -> MemcmpEncoded {
-            encode_value(&Some(ScalarImpl::from(f)), OrderType::default()).unwrap()
+            encode_value(Some(ScalarImpl::from(f)), OrderType::default()).unwrap()
         }
 
         fn deserialize(data: MemcmpEncoded) -> F32 {
diff --git a/src/common/src/util/panic.rs b/src/common/src/util/panic.rs
index 43db76df4f768..517a040e03b09 100644
--- a/src/common/src/util/panic.rs
+++ b/src/common/src/util/panic.rs
@@ -71,8 +71,8 @@ pub fn is_catching_unwind() -> bool {
 }
 
 #[cfg(all(test, not(madsim)))]
+#[expect(clippy::disallowed_methods)]
 mod tests {
-    #![allow(clippy::disallowed_methods)]
 
     use rusty_fork::rusty_fork_test;
 
diff --git a/src/common/src/util/prost.rs b/src/common/src/util/prost.rs
index 4a4f846f543a5..8145a37a8a202 100644
--- a/src/common/src/util/prost.rs
+++ b/src/common/src/util/prost.rs
@@ -12,7 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::btree_map::Entry;
+use std::fmt::{Display, Formatter};
+use std::ops::Deref;
+
 use risingwave_pb::batch_plan;
+use risingwave_pb::monitor_service::StackTraceResponse;
+use tracing::warn;
 
 pub trait TypeUrl {
     fn type_url() -> &'static str;
@@ -23,3 +29,104 @@ impl TypeUrl for batch_plan::ExchangeNode {
         "type.googleapis.com/plan.ExchangeNode"
     }
 }
+
+pub struct StackTraceResponseOutput<'a>(&'a StackTraceResponse);
+
+impl<'a> Deref for StackTraceResponseOutput<'a> {
+    type Target = StackTraceResponse;
+
+    fn deref(&self) -> &Self::Target {
+        self.0
+    }
+}
+
+impl<'a> Display for StackTraceResponseOutput<'a> {
+    fn fmt(&self, s: &mut Formatter<'_>) -> std::fmt::Result {
+        if !self.actor_traces.is_empty() {
+            writeln!(s, "--- Actor Traces ---")?;
+            for (actor_id, trace) in &self.actor_traces {
+                writeln!(s, ">> Actor {}", *actor_id)?;
+                writeln!(s, "{trace}")?;
+            }
+        }
+        if !self.rpc_traces.is_empty() {
+            let _ = writeln!(s, "--- RPC Traces ---");
+            for (name, trace) in &self.rpc_traces {
+                writeln!(s, ">> RPC {name}")?;
+                writeln!(s, "{trace}")?;
+            }
+        }
+        if !self.compaction_task_traces.is_empty() {
+            writeln!(s, "--- Compactor Traces ---")?;
+            for (name, trace) in &self.compaction_task_traces {
+                writeln!(s, ">> Compaction Task {name}")?;
+                writeln!(s, "{trace}")?;
+            }
+        }
+
+        if !self.inflight_barrier_traces.is_empty() {
+            writeln!(s, "--- Inflight Barrier Traces ---")?;
+            for (name, trace) in &self.inflight_barrier_traces {
+                writeln!(s, ">> Barrier {name}")?;
+                writeln!(s, "{trace}")?;
+            }
+        }
+
+        writeln!(s, "\n\n--- Barrier Worker States ---")?;
+        for (worker_id, state) in &self.barrier_worker_state {
+            writeln!(s, ">> Worker {worker_id}")?;
+            writeln!(s, "{state}\n")?;
+        }
+
+        if !self.jvm_stack_traces.is_empty() {
+            writeln!(s, "\n\n--- JVM Stack Traces ---")?;
+            for (worker_id, state) in &self.jvm_stack_traces {
+                writeln!(s, ">> Worker {worker_id}")?;
+                writeln!(s, "{state}\n")?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
+#[easy_ext::ext(StackTraceResponseExt)]
+impl StackTraceResponse {
+    pub fn merge_other(&mut self, b: StackTraceResponse) {
+        self.actor_traces.extend(b.actor_traces);
+        self.rpc_traces.extend(b.rpc_traces);
+        self.compaction_task_traces.extend(b.compaction_task_traces);
+        self.inflight_barrier_traces
+            .extend(b.inflight_barrier_traces);
+        for (worker_id, worker_state) in b.barrier_worker_state {
+            match self.barrier_worker_state.entry(worker_id) {
+                Entry::Occupied(_entry) => {
+                    warn!(
+                        worker_id,
+                        worker_state, "duplicate barrier worker state. skipped"
+                    );
+                }
+                Entry::Vacant(entry) => {
+                    entry.insert(worker_state);
+                }
+            }
+        }
+        for (worker_id, worker_state) in b.jvm_stack_traces {
+            match self.jvm_stack_traces.entry(worker_id) {
+                Entry::Occupied(_entry) => {
+                    warn!(
+                        worker_id,
+                        worker_state, "duplicate jvm stack trace. skipped"
+                    );
+                }
+                Entry::Vacant(entry) => {
+                    entry.insert(worker_state);
+                }
+            }
+        }
+    }
+
+    pub fn output(&self) -> StackTraceResponseOutput<'_> {
+        StackTraceResponseOutput(self)
+    }
+}
diff --git a/src/common/src/util/sort_util.rs b/src/common/src/util/sort_util.rs
index 015222136d382..ff4bde652496d 100644
--- a/src/common/src/util/sort_util.rs
+++ b/src/common/src/util/sort_util.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::cmp::{Ord, Ordering};
+use std::cmp::Ordering;
 use std::fmt;
 use std::sync::Arc;
 
@@ -574,13 +574,11 @@ pub fn cmp_rows(lhs: impl Row, rhs: impl Row, order_types: &[OrderType]) -> Orde
 
 #[cfg(test)]
 mod tests {
-    use std::cmp::Ordering;
-
     use itertools::Itertools;
 
     use super::*;
-    use crate::array::{DataChunk, ListValue, StructValue};
-    use crate::row::{OwnedRow, Row};
+    use crate::array::{ListValue, StructValue};
+    use crate::row::OwnedRow;
     use crate::types::{DataType, Datum, ScalarImpl};
 
     #[test]
diff --git a/src/common/src/util/stream_graph_visitor.rs b/src/common/src/util/stream_graph_visitor.rs
index 81f1189693c0d..553ede90bcb77 100644
--- a/src/common/src/util/stream_graph_visitor.rs
+++ b/src/common/src/util/stream_graph_visitor.rs
@@ -36,8 +36,8 @@ where
     visit_inner(stream_node, &mut f)
 }
 
-/// A utility for to accessing the [`StreamNode`]. The returned bool is used to determine whether the access needs to continue.
-pub fn visit_stream_node_cont<F>(stream_node: &mut StreamNode, mut f: F)
+/// A utility for to accessing the [`StreamNode`] mutably. The returned bool is used to determine whether the access needs to continue.
+pub fn visit_stream_node_cont_mut<F>(stream_node: &mut StreamNode, mut f: F)
 where
     F: FnMut(&mut StreamNode) -> bool,
 {
@@ -56,6 +56,26 @@ where
     visit_inner(stream_node, &mut f)
 }
 
+/// A utility for to accessing the [`StreamNode`] immutably. The returned bool is used to determine whether the access needs to continue.
+pub fn visit_stream_node_cont<F>(stream_node: &StreamNode, mut f: F)
+where
+    F: FnMut(&StreamNode) -> bool,
+{
+    fn visit_inner<F>(stream_node: &StreamNode, f: &mut F)
+    where
+        F: FnMut(&StreamNode) -> bool,
+    {
+        if !f(stream_node) {
+            return;
+        }
+        for input in &stream_node.input {
+            visit_inner(input, f);
+        }
+    }
+
+    visit_inner(stream_node, &mut f)
+}
+
 /// A utility for visiting and mutating the [`NodeBody`] of the [`StreamNode`]s in a
 /// [`StreamFragment`] recursively.
 pub fn visit_fragment<F>(fragment: &mut StreamFragment, f: F)
@@ -200,12 +220,6 @@ pub fn visit_stream_node_tables_inner<F>(
                 optional!(node.table, "Sink")
             }
 
-            // Subscription
-            NodeBody::Subscription(node) => {
-                // A Subscription should have a log store
-                optional!(node.log_store_table, "Subscription")
-            }
-
             // Now
             NodeBody::Now(node) => {
                 always!(node.state_table, "Now");
diff --git a/src/common/src/util/value_encoding/column_aware_row_encoding.rs b/src/common/src/util/value_encoding/column_aware_row_encoding.rs
index 28fcb65c4d8db..1bfbc0c926410 100644
--- a/src/common/src/util/value_encoding/column_aware_row_encoding.rs
+++ b/src/common/src/util/value_encoding/column_aware_row_encoding.rs
@@ -26,7 +26,6 @@ use bitflags::bitflags;
 
 use super::*;
 use crate::catalog::ColumnId;
-use crate::row::Row;
 
 // deprecated design of have a Width to represent number of datum
 // may be considered should `ColumnId` representation be optimized
diff --git a/src/common/src/util/value_encoding/mod.rs b/src/common/src/util/value_encoding/mod.rs
index 57e21c3782aa0..aa80c15617048 100644
--- a/src/common/src/util/value_encoding/mod.rs
+++ b/src/common/src/util/value_encoding/mod.rs
@@ -20,7 +20,7 @@ use either::{for_both, Either};
 use enum_as_inner::EnumAsInner;
 use risingwave_pb::data::PbDatum;
 
-use crate::array::{ArrayImpl, ListRef, ListValue, StructRef, StructValue};
+use crate::array::ArrayImpl;
 use crate::row::{Row, RowDeserializer as BasicDeserializer};
 use crate::types::*;
 
diff --git a/src/common/src/util/worker_util.rs b/src/common/src/util/worker_util.rs
index 893cd95ecbbbb..80ecd3b822536 100644
--- a/src/common/src/util/worker_util.rs
+++ b/src/common/src/util/worker_util.rs
@@ -12,23 +12,4 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
-
-use risingwave_pb::common::WorkerNode;
-
-use crate::hash::ParallelUnitId;
-
 pub type WorkerNodeId = u32;
-
-pub fn get_pu_to_worker_mapping(nodes: &[WorkerNode]) -> HashMap<ParallelUnitId, WorkerNode> {
-    let mut pu_to_worker = HashMap::new();
-
-    for node in nodes {
-        for pu in &node.parallel_units {
-            let res = pu_to_worker.insert(pu.id, node.clone());
-            assert!(res.is_none(), "duplicate parallel unit id");
-        }
-    }
-
-    pu_to_worker
-}
diff --git a/src/common/src/vnode_mapping/vnode_placement.rs b/src/common/src/vnode_mapping/vnode_placement.rs
index 49f45d66512eb..036cfebe792bb 100644
--- a/src/common/src/vnode_mapping/vnode_placement.rs
+++ b/src/common/src/vnode_mapping/vnode_placement.rs
@@ -13,46 +13,48 @@
 // limitations under the License.
 
 use std::collections::{HashMap, HashSet, LinkedList, VecDeque};
+use std::ops::BitOrAssign;
 
 use itertools::Itertools;
 use num_integer::Integer;
+use risingwave_common::hash::WorkerSlotId;
 use risingwave_pb::common::WorkerNode;
 
 use crate::buffer::{Bitmap, BitmapBuilder};
-use crate::hash::{ParallelUnitId, ParallelUnitMapping, VirtualNode};
+use crate::hash::{VirtualNode, WorkerSlotMapping};
 
 /// Calculate a new vnode mapping, keeping locality and balance on a best effort basis.
 /// The strategy is similar to `rebalance_actor_vnode` used in meta node, but is modified to
 /// consider `max_parallelism` too.
 pub fn place_vnode(
-    hint_pu_mapping: Option<&ParallelUnitMapping>,
-    new_workers: &[WorkerNode],
+    hint_worker_slot_mapping: Option<&WorkerSlotMapping>,
+    workers: &[WorkerNode],
     max_parallelism: Option<usize>,
-) -> Option<ParallelUnitMapping> {
-    // Get all serving parallel units from all available workers, grouped by worker id and ordered
-    // by parallel unit id in each group.
-    let mut new_pus: LinkedList<_> = new_workers
+) -> Option<WorkerSlotMapping> {
+    // Get all serving worker slots from all available workers, grouped by worker id and ordered
+    // by worker slot id in each group.
+    let mut worker_slots: LinkedList<_> = workers
         .iter()
         .filter(|w| w.property.as_ref().map_or(false, |p| p.is_serving))
         .sorted_by_key(|w| w.id)
-        .map(|w| w.parallel_units.clone().into_iter().sorted_by_key(|p| p.id))
+        .map(|w| (0..w.parallel_units.len()).map(|idx| WorkerSlotId::new(w.id, idx)))
         .collect();
 
-    // Set serving parallelism to the minimum of total number of parallel units, specified
+    // Set serving parallelism to the minimum of total number of worker slots, specified
     // `max_parallelism` and total number of virtual nodes.
     let serving_parallelism = std::cmp::min(
-        new_pus.iter().map(|pus| pus.len()).sum(),
+        worker_slots.iter().map(|slots| slots.len()).sum(),
         std::cmp::min(max_parallelism.unwrap_or(usize::MAX), VirtualNode::COUNT),
     );
 
-    // Select `serving_parallelism` parallel units in a round-robin fashion, to distribute workload
+    // Select `serving_parallelism` worker slots in a round-robin fashion, to distribute workload
     // evenly among workers.
-    let mut selected_pu_ids = Vec::new();
-    while !new_pus.is_empty() {
-        new_pus
-            .extract_if(|ps| {
-                if let Some(p) = ps.next() {
-                    selected_pu_ids.push(p.id);
+    let mut selected_slots = Vec::new();
+    while !worker_slots.is_empty() {
+        worker_slots
+            .extract_if(|slots| {
+                if let Some(slot) = slots.next() {
+                    selected_slots.push(slot);
                     false
                 } else {
                     true
@@ -60,57 +62,61 @@ pub fn place_vnode(
             })
             .for_each(drop);
     }
-    selected_pu_ids.drain(serving_parallelism..);
-    let selected_pu_id_set: HashSet<ParallelUnitId> = selected_pu_ids.iter().cloned().collect();
-    if selected_pu_id_set.is_empty() {
+    selected_slots.drain(serving_parallelism..);
+    let selected_slots_set: HashSet<WorkerSlotId> = selected_slots.iter().cloned().collect();
+    if selected_slots_set.is_empty() {
         return None;
     }
 
-    // Calculate balance for each selected parallel unit. Initially, each parallel unit is assigned
+    // Calculate balance for each selected worker slot. Initially, each worker slot is assigned
     // no vnodes. Thus its negative balance means that many vnodes should be assigned to it later.
-    // `is_temp` is a mark for a special temporary parallel unit, only to simplify implementation.
+    // `is_temp` is a mark for a special temporary worker slot, only to simplify implementation.
     #[derive(Debug)]
     struct Balance {
-        pu_id: ParallelUnitId,
+        slot: WorkerSlotId,
         balance: i32,
         builder: BitmapBuilder,
         is_temp: bool,
     }
-    let (expected, mut remain) = VirtualNode::COUNT.div_rem(&selected_pu_ids.len());
-    let mut balances: HashMap<ParallelUnitId, Balance> = HashMap::default();
-    for pu_id in &selected_pu_ids {
+
+    let (expected, mut remain) = VirtualNode::COUNT.div_rem(&selected_slots.len());
+    let mut balances: HashMap<WorkerSlotId, Balance> = HashMap::default();
+
+    for slot in &selected_slots {
         let mut balance = Balance {
-            pu_id: *pu_id,
+            slot: *slot,
             balance: -(expected as i32),
             builder: BitmapBuilder::zeroed(VirtualNode::COUNT),
             is_temp: false,
         };
+
         if remain > 0 {
             balance.balance -= 1;
             remain -= 1;
         }
-        balances.insert(*pu_id, balance);
+        balances.insert(*slot, balance);
     }
 
-    // Now to maintain affinity, if a hint has been provided via `hint_pu_mapping`, follow
+    // Now to maintain affinity, if a hint has been provided via `hint_worker_slot_mapping`, follow
     // that mapping to adjust balances.
-    let mut temp_pu = Balance {
-        pu_id: 0, // This id doesn't matter for `temp_pu`. It's distinguishable via `is_temp`.
+    let mut temp_slot = Balance {
+        slot: WorkerSlotId::new(0u32, usize::MAX), /* This id doesn't matter for `temp_slot`. It's distinguishable via `is_temp`. */
         balance: 0,
         builder: BitmapBuilder::zeroed(VirtualNode::COUNT),
         is_temp: true,
     };
-    match hint_pu_mapping {
-        Some(hint_pu_mapping) => {
-            for (vnode, pu_id) in hint_pu_mapping.iter_with_vnode() {
-                let b = if selected_pu_id_set.contains(&pu_id) {
-                    // Assign vnode to the same parallel unit as hint.
-                    balances.get_mut(&pu_id).unwrap()
+    match hint_worker_slot_mapping {
+        Some(hint_worker_slot_mapping) => {
+            for (vnode, worker_slot) in hint_worker_slot_mapping.iter_with_vnode() {
+                let b = if selected_slots_set.contains(&worker_slot) {
+                    // Assign vnode to the same worker slot as hint.
+                    balances.get_mut(&worker_slot).unwrap()
                 } else {
-                    // Assign vnode that doesn't belong to any parallel unit to `temp_pu`
+                    // Assign vnode that doesn't belong to any worker slot to `temp_slot`
                     // temporarily. They will be reassigned later.
-                    &mut temp_pu
+                    &mut temp_slot
                 };
+
                 b.balance += 1;
                 b.builder.set(vnode.to_index(), true);
             }
@@ -118,31 +124,33 @@ pub fn place_vnode(
         None => {
             // No hint is provided, assign all vnodes to `temp_pu`.
             for vnode in VirtualNode::all() {
-                temp_pu.balance += 1;
-                temp_pu.builder.set(vnode.to_index(), true);
+                temp_slot.balance += 1;
+                temp_slot.builder.set(vnode.to_index(), true);
             }
         }
     }
 
-    // The final step is to move vnodes from parallel units with positive balance to parallel units
-    // with negative balance, until all parallel units are of 0 balance.
-    // A double-ended queue with parallel units ordered by balance in descending order is consumed:
-    // 1. Peek 2 parallel units from front and back.
+    // The final step is to move vnodes from worker slots with positive balance to worker slots
+    // with negative balance, until all worker slots are of 0 balance.
+    // A double-ended queue with worker slots ordered by balance in descending order is consumed:
+    // 1. Peek 2 worker slots from front and back.
     // 2. It any of them is of 0 balance, pop it and go to step 1.
     // 3. Otherwise, move vnodes from front to back.
     let mut balances: VecDeque<_> = balances
         .into_values()
-        .chain(std::iter::once(temp_pu))
+        .chain(std::iter::once(temp_slot))
         .sorted_by_key(|b| b.balance)
         .rev()
         .collect();
-    let mut results: HashMap<ParallelUnitId, Bitmap> = HashMap::default();
+
+    let mut results: HashMap<WorkerSlotId, Bitmap> = HashMap::default();
+
     while !balances.is_empty() {
         if balances.len() == 1 {
             let single = balances.pop_front().unwrap();
             assert_eq!(single.balance, 0);
             if !single.is_temp {
-                results.insert(single.pu_id, single.builder.finish());
+                results.insert(single.slot, single.builder.finish());
             }
             break;
         }
@@ -166,32 +174,42 @@ pub fn place_vnode(
         if src.balance != 0 {
             balances.push_front(src);
         } else if !src.is_temp {
-            results.insert(src.pu_id, src.builder.finish());
+            results.insert(src.slot, src.builder.finish());
         }
 
         if dst.balance != 0 {
             balances.push_back(dst);
         } else if !dst.is_temp {
-            results.insert(dst.pu_id, dst.builder.finish());
+            results.insert(dst.slot, dst.builder.finish());
         }
     }
 
-    Some(ParallelUnitMapping::from_bitmaps(&results))
+    let mut worker_result = HashMap::new();
+
+    for (worker_slot, bitmap) in results {
+        worker_result
+            .entry(worker_slot)
+            .or_insert(BitmapBuilder::zeroed(VirtualNode::COUNT).finish())
+            .bitor_assign(&bitmap);
+    }
+
+    Some(WorkerSlotMapping::from_bitmaps(&worker_result))
 }
 
 #[cfg(test)]
 mod tests {
     use std::collections::HashMap;
 
+    use risingwave_common::hash::WorkerSlotMapping;
     use risingwave_pb::common::worker_node::Property;
     use risingwave_pb::common::{ParallelUnit, WorkerNode};
 
-    use crate::hash::{ParallelUnitId, ParallelUnitMapping, VirtualNode};
+    use crate::hash::{ParallelUnitId, VirtualNode};
     use crate::vnode_mapping::vnode_placement::place_vnode;
-
     #[test]
     fn test_place_vnode() {
         assert_eq!(VirtualNode::COUNT, 256);
+
         let mut pu_id_counter: ParallelUnitId = 0;
         let mut pu_to_worker: HashMap<ParallelUnitId, u32> = Default::default();
         let serving_property = Property {
@@ -216,13 +234,13 @@ mod tests {
                 results
             };
 
-        let count_same_vnode_mapping = |pm1: &ParallelUnitMapping, pm2: &ParallelUnitMapping| {
-            assert_eq!(pm1.len(), 256);
-            assert_eq!(pm2.len(), 256);
+        let count_same_vnode_mapping = |wm1: &WorkerSlotMapping, wm2: &WorkerSlotMapping| {
+            assert_eq!(wm1.len(), 256);
+            assert_eq!(wm2.len(), 256);
             let mut count: usize = 0;
             for idx in 0..VirtualNode::COUNT {
                 let vnode = VirtualNode::from_index(idx);
-                if pm1.get(vnode) == pm2.get(vnode) {
+                if wm1.get(vnode) == wm2.get(vnode) {
                     count += 1;
                 }
             }
@@ -235,29 +253,32 @@ mod tests {
             property: Some(serving_property.clone()),
             ..Default::default()
         };
+
         assert!(
             place_vnode(None, &[worker_1.clone()], Some(0)).is_none(),
             "max_parallelism should >= 0"
         );
 
-        let re_pu_mapping_2 = place_vnode(None, &[worker_1.clone()], None).unwrap();
-        assert_eq!(re_pu_mapping_2.iter_unique().count(), 1);
+        let re_worker_mapping_2 = place_vnode(None, &[worker_1.clone()], None).unwrap();
+        assert_eq!(re_worker_mapping_2.iter_unique().count(), 1);
+
         let worker_2 = WorkerNode {
             id: 2,
             parallel_units: gen_pus_for_worker(2, 50, &mut pu_to_worker),
             property: Some(serving_property.clone()),
             ..Default::default()
         };
-        let re_pu_mapping = place_vnode(
-            Some(&re_pu_mapping_2),
+
+        let re_worker_mapping = place_vnode(
+            Some(&re_worker_mapping_2),
             &[worker_1.clone(), worker_2.clone()],
             None,
         )
         .unwrap();
 
-        assert_eq!(re_pu_mapping.iter_unique().count(), 51);
+        assert_eq!(re_worker_mapping.iter_unique().count(), 51);
         // 1 * 256 + 0 -> 51 * 5 + 1
-        let score = count_same_vnode_mapping(&re_pu_mapping_2, &re_pu_mapping);
+        let score = count_same_vnode_mapping(&re_worker_mapping_2, &re_worker_mapping);
         assert!(score >= 5);
 
         let worker_3 = WorkerNode {
@@ -267,7 +288,7 @@ mod tests {
             ..Default::default()
         };
         let re_pu_mapping_2 = place_vnode(
-            Some(&re_pu_mapping),
+            Some(&re_worker_mapping),
             &[worker_1.clone(), worker_2.clone(), worker_3.clone()],
             None,
         )
@@ -276,7 +297,7 @@ mod tests {
         // limited by total pu number
         assert_eq!(re_pu_mapping_2.iter_unique().count(), 111);
         // 51 * 5 + 1 -> 111 * 2 + 34
-        let score = count_same_vnode_mapping(&re_pu_mapping_2, &re_pu_mapping);
+        let score = count_same_vnode_mapping(&re_pu_mapping_2, &re_worker_mapping);
         assert!(score >= (2 + 50 * 2));
         let re_pu_mapping = place_vnode(
             Some(&re_pu_mapping_2),
diff --git a/src/compute/src/lib.rs b/src/compute/src/lib.rs
index c3d82dddfde94..87f052470ea7d 100644
--- a/src/compute/src/lib.rs
+++ b/src/compute/src/lib.rs
@@ -76,10 +76,6 @@ pub struct ComputeNodeOpts {
     #[clap(long, env = "RW_META_ADDR", default_value = "http://127.0.0.1:5690")]
     pub meta_address: MetaAddressStrategy,
 
-    /// Payload format of connector sink rpc
-    #[clap(long, env = "RW_CONNECTOR_RPC_SINK_PAYLOAD_FORMAT")]
-    pub connector_rpc_sink_payload_format: Option<String>,
-
     /// The path of `risingwave.toml` configuration file.
     ///
     /// If empty, default configuration values will be used.
@@ -90,6 +86,13 @@ pub struct ComputeNodeOpts {
     #[clap(long, env = "RW_TOTAL_MEMORY_BYTES", default_value_t = default_total_memory_bytes())]
     pub total_memory_bytes: usize,
 
+    /// Reserved memory for the compute node in bytes.
+    /// If not set, a portion (default to 30%) for the `total_memory_bytes` will be used as the reserved memory.
+    ///
+    /// The total memory compute and storage can use is `total_memory_bytes` - `reserved_memory_bytes`.
+    #[clap(long, env = "RW_RESERVED_MEMORY_BYTES")]
+    pub reserved_memory_bytes: Option<usize>,
+
     /// The parallelism that the compute node will register to the scheduler of the meta service.
     #[clap(long, env = "RW_PARALLELISM", default_value_t = default_parallelism())]
     #[override_opts(if_absent, path = streaming.actor_runtime_worker_threads_num)]
diff --git a/src/compute/src/memory/config.rs b/src/compute/src/memory/config.rs
index ca04f2e411b64..2a6607c268a67 100644
--- a/src/compute/src/memory/config.rs
+++ b/src/compute/src/memory/config.rs
@@ -12,20 +12,24 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use foyer::memory::{LfuConfig, LruConfig, S3FifoConfig};
+use foyer::{LfuConfig, LruConfig, S3FifoConfig};
 use risingwave_common::config::{
     CacheEvictionConfig, EvictionConfig, StorageConfig, StorageMemoryConfig,
     MAX_BLOCK_CACHE_SHARD_BITS, MAX_META_CACHE_SHARD_BITS, MIN_BUFFER_SIZE_PER_SHARD,
 };
 use risingwave_common::util::pretty_bytes::convert;
 
+use crate::ComputeNodeOpts;
+
 /// The minimal memory requirement of computing tasks in megabytes.
 pub const MIN_COMPUTE_MEMORY_MB: usize = 512;
 /// The memory reserved for system usage (stack and code segment of processes, allocation
 /// overhead, network buffer, etc.) in megabytes.
 pub const MIN_SYSTEM_RESERVED_MEMORY_MB: usize = 512;
 
-const SYSTEM_RESERVED_MEMORY_PROPORTION: f64 = 0.3;
+const RESERVED_MEMORY_LEVELS: [usize; 2] = [16 << 30, usize::MAX];
+
+const RESERVED_MEMORY_PROPORTIONS: [f64; 2] = [0.3, 0.2];
 
 const STORAGE_MEMORY_PROPORTION: f64 = 0.3;
 
@@ -38,23 +42,55 @@ const STORAGE_SHARED_BUFFER_MAX_MEMORY_MB: usize = 4096;
 const STORAGE_META_CACHE_MEMORY_PROPORTION: f64 = 0.35;
 const STORAGE_SHARED_BUFFER_MEMORY_PROPORTION: f64 = 0.3;
 
+/// The proportion of compute memory used for batch processing.
+const COMPUTE_BATCH_MEMORY_PROPORTION: f64 = 0.3;
+
 /// Each compute node reserves some memory for stack and code segment of processes, allocation
-/// overhead, network buffer, etc. based on `SYSTEM_RESERVED_MEMORY_PROPORTION`. The reserve memory
+/// overhead, network buffer, etc. based on gradient reserve memory proportion. The reserve memory
 /// size must be larger than `MIN_SYSTEM_RESERVED_MEMORY_MB`
-pub fn reserve_memory_bytes(total_memory_bytes: usize) -> (usize, usize) {
-    if total_memory_bytes < MIN_COMPUTE_MEMORY_MB << 20 {
+pub fn reserve_memory_bytes(opts: &ComputeNodeOpts) -> (usize, usize) {
+    if opts.total_memory_bytes < MIN_COMPUTE_MEMORY_MB << 20 {
         panic!(
             "The total memory size ({}) is too small. It must be at least {} MB.",
-            convert(total_memory_bytes as _),
+            convert(opts.total_memory_bytes as _),
             MIN_COMPUTE_MEMORY_MB
         );
     }
 
-    let reserved = std::cmp::max(
-        (total_memory_bytes as f64 * SYSTEM_RESERVED_MEMORY_PROPORTION).ceil() as usize,
-        MIN_SYSTEM_RESERVED_MEMORY_MB << 20,
-    );
-    (reserved, total_memory_bytes - reserved)
+    // If `reserved_memory_bytes` is not set, calculate total_memory_bytes based on gradient reserve memory proportion.
+    let reserved = opts
+        .reserved_memory_bytes
+        .unwrap_or_else(|| gradient_reserve_memory_bytes(opts.total_memory_bytes));
+
+    // Should have at least `MIN_SYSTEM_RESERVED_MEMORY_MB` for reserved memory.
+    let reserved = std::cmp::max(reserved, MIN_SYSTEM_RESERVED_MEMORY_MB << 20);
+
+    (reserved, opts.total_memory_bytes - reserved)
+}
+
+/// Calculate the reserved memory based on the total memory size.
+/// The reserved memory size is calculated based on the following gradient:
+/// - 30% of the first 16GB
+/// - 20% of the rest
+fn gradient_reserve_memory_bytes(total_memory_bytes: usize) -> usize {
+    let mut total_memory_bytes = total_memory_bytes;
+    let mut reserved = 0;
+    for i in 0..RESERVED_MEMORY_LEVELS.len() {
+        let level_diff = if i == 0 {
+            RESERVED_MEMORY_LEVELS[0]
+        } else {
+            RESERVED_MEMORY_LEVELS[i] - RESERVED_MEMORY_LEVELS[i - 1]
+        };
+        if total_memory_bytes <= level_diff {
+            reserved += (total_memory_bytes as f64 * RESERVED_MEMORY_PROPORTIONS[i]) as usize;
+            break;
+        } else {
+            reserved += (level_diff as f64 * RESERVED_MEMORY_PROPORTIONS[i]) as usize;
+            total_memory_bytes -= level_diff;
+        }
+    }
+
+    reserved
 }
 
 /// Decide the memory limit for each storage cache. If not specified in `StorageConfig`, memory
@@ -63,6 +99,7 @@ pub fn storage_memory_config(
     non_reserved_memory_bytes: usize,
     embedded_compactor_enabled: bool,
     storage_config: &StorageConfig,
+    is_serving: bool,
 ) -> StorageMemoryConfig {
     let (storage_memory_proportion, compactor_memory_proportion) = if embedded_compactor_enabled {
         (STORAGE_MEMORY_PROPORTION, COMPACTOR_MEMORY_PROPORTION)
@@ -104,14 +141,18 @@ pub fn storage_memory_config(
         * STORAGE_SHARED_BUFFER_MEMORY_PROPORTION)
         .ceil() as usize)
         >> 20;
-    let shared_buffer_capacity_mb =
+    let mut shared_buffer_capacity_mb =
         storage_config
             .shared_buffer_capacity_mb
             .unwrap_or(std::cmp::min(
                 default_shared_buffer_capacity_mb,
                 STORAGE_SHARED_BUFFER_MAX_MEMORY_MB,
             ));
-    if shared_buffer_capacity_mb != default_shared_buffer_capacity_mb {
+    if is_serving {
+        default_block_cache_capacity_mb += default_shared_buffer_capacity_mb;
+        // set 1 to pass internal check
+        shared_buffer_capacity_mb = 1;
+    } else if shared_buffer_capacity_mb != default_shared_buffer_capacity_mb {
         default_block_cache_capacity_mb += default_shared_buffer_capacity_mb;
         default_block_cache_capacity_mb =
             default_block_cache_capacity_mb.saturating_sub(shared_buffer_capacity_mb);
@@ -202,12 +243,22 @@ pub fn storage_memory_config(
         }),
         CacheEvictionConfig::S3Fifo {
             small_queue_capacity_ratio_in_percent,
+            ghost_queue_capacity_ratio_in_percent,
+            small_to_main_freq_threshold,
         } => EvictionConfig::S3Fifo(S3FifoConfig {
             small_queue_capacity_ratio: small_queue_capacity_ratio_in_percent.unwrap_or(
                 risingwave_common::config::default::storage::small_queue_capacity_ratio_in_percent(
                 ),
             ) as f64
                 / 100.0,
+            ghost_queue_capacity_ratio: ghost_queue_capacity_ratio_in_percent.unwrap_or(
+                risingwave_common::config::default::storage::ghost_queue_capacity_ratio_in_percent(
+                ),
+            ) as f64
+                / 100.0,
+            small_to_main_freq_threshold: small_to_main_freq_threshold.unwrap_or(
+                risingwave_common::config::default::storage::small_to_main_freq_threshold(),
+            ),
         }),
     };
 
@@ -228,23 +279,47 @@ pub fn storage_memory_config(
     }
 }
 
+pub fn batch_mem_limit(compute_memory_bytes: usize) -> u64 {
+    (compute_memory_bytes as f64 * COMPUTE_BATCH_MEMORY_PROPORTION) as u64
+}
+
 #[cfg(test)]
 mod tests {
+    use clap::Parser;
     use risingwave_common::config::StorageConfig;
 
     use super::{reserve_memory_bytes, storage_memory_config};
+    use crate::ComputeNodeOpts;
 
     #[test]
     fn test_reserve_memory_bytes() {
         // at least 512 MB
-        let (reserved, non_reserved) = reserve_memory_bytes(1536 << 20);
-        assert_eq!(reserved, 512 << 20);
-        assert_eq!(non_reserved, 1024 << 20);
+        {
+            let mut opts = ComputeNodeOpts::parse_from(vec![] as Vec<String>);
+            opts.total_memory_bytes = 1536 << 20;
+            let (reserved, non_reserved) = reserve_memory_bytes(&opts);
+            assert_eq!(reserved, 512 << 20);
+            assert_eq!(non_reserved, 1024 << 20);
+        }
 
         // reserve based on proportion
-        let (reserved, non_reserved) = reserve_memory_bytes(10 << 30);
-        assert_eq!(reserved, 3 << 30);
-        assert_eq!(non_reserved, 7 << 30);
+        {
+            let mut opts = ComputeNodeOpts::parse_from(vec![] as Vec<String>);
+            opts.total_memory_bytes = 10 << 30;
+            let (reserved, non_reserved) = reserve_memory_bytes(&opts);
+            assert_eq!(reserved, 3 << 30);
+            assert_eq!(non_reserved, 7 << 30);
+        }
+
+        // reserve based on opts
+        {
+            let mut opts = ComputeNodeOpts::parse_from(vec![] as Vec<String>);
+            opts.total_memory_bytes = 10 << 30;
+            opts.reserved_memory_bytes = Some(2 << 30);
+            let (reserved, non_reserved) = reserve_memory_bytes(&opts);
+            assert_eq!(reserved, 2 << 30);
+            assert_eq!(non_reserved, 8 << 30);
+        }
     }
 
     #[test]
@@ -253,17 +328,36 @@ mod tests {
 
         let total_non_reserved_memory_bytes = 8 << 30;
 
-        let memory_config =
-            storage_memory_config(total_non_reserved_memory_bytes, true, &storage_config);
+        let memory_config = storage_memory_config(
+            total_non_reserved_memory_bytes,
+            true,
+            &storage_config,
+            false,
+        );
         assert_eq!(memory_config.block_cache_capacity_mb, 737);
         assert_eq!(memory_config.meta_cache_capacity_mb, 860);
         assert_eq!(memory_config.shared_buffer_capacity_mb, 737);
         assert_eq!(memory_config.compactor_memory_limit_mb, 819);
 
+        let memory_config = storage_memory_config(
+            total_non_reserved_memory_bytes,
+            false,
+            &storage_config,
+            true,
+        );
+        assert_eq!(memory_config.block_cache_capacity_mb, 1966);
+        assert_eq!(memory_config.meta_cache_capacity_mb, 1146);
+        assert_eq!(memory_config.shared_buffer_capacity_mb, 1);
+        assert_eq!(memory_config.compactor_memory_limit_mb, 0);
+
         storage_config.data_file_cache.dir = "data".to_string();
         storage_config.meta_file_cache.dir = "meta".to_string();
-        let memory_config =
-            storage_memory_config(total_non_reserved_memory_bytes, true, &storage_config);
+        let memory_config = storage_memory_config(
+            total_non_reserved_memory_bytes,
+            true,
+            &storage_config,
+            false,
+        );
         assert_eq!(memory_config.block_cache_capacity_mb, 737);
         assert_eq!(memory_config.meta_cache_capacity_mb, 860);
         assert_eq!(memory_config.shared_buffer_capacity_mb, 737);
@@ -273,10 +367,23 @@ mod tests {
         storage_config.cache.meta_cache_capacity_mb = Some(128);
         storage_config.shared_buffer_capacity_mb = Some(1024);
         storage_config.compactor_memory_limit_mb = Some(512);
-        let memory_config = storage_memory_config(0, true, &storage_config);
+        let memory_config = storage_memory_config(0, true, &storage_config, false);
         assert_eq!(memory_config.block_cache_capacity_mb, 512);
         assert_eq!(memory_config.meta_cache_capacity_mb, 128);
         assert_eq!(memory_config.shared_buffer_capacity_mb, 1024);
         assert_eq!(memory_config.compactor_memory_limit_mb, 512);
     }
+
+    #[test]
+    fn test_gradient_reserve_memory_bytes() {
+        assert_eq!(super::gradient_reserve_memory_bytes(4 << 30), 1288490188);
+        assert_eq!(super::gradient_reserve_memory_bytes(8 << 30), 2576980377);
+        assert_eq!(super::gradient_reserve_memory_bytes(16 << 30), 5153960755);
+        assert_eq!(super::gradient_reserve_memory_bytes(24 << 30), 6871947673);
+        assert_eq!(super::gradient_reserve_memory_bytes(32 << 30), 8589934591);
+        assert_eq!(super::gradient_reserve_memory_bytes(54 << 30), 13314398617);
+        assert_eq!(super::gradient_reserve_memory_bytes(64 << 30), 15461882265);
+        assert_eq!(super::gradient_reserve_memory_bytes(100 << 30), 23192823398);
+        assert_eq!(super::gradient_reserve_memory_bytes(128 << 30), 29205777612);
+    }
 }
diff --git a/src/compute/src/memory/controller.rs b/src/compute/src/memory/controller.rs
index 55d0547249117..2857b0e6b0d66 100644
--- a/src/compute/src/memory/controller.rs
+++ b/src/compute/src/memory/controller.rs
@@ -12,28 +12,29 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
-use risingwave_common::util::epoch::Epoch;
+use risingwave_common::sequence::{Sequence, SEQUENCE_GLOBAL};
 use risingwave_jni_core::jvm_runtime::load_jvm_memory_stats;
 use risingwave_stream::executor::monitor::StreamingMetrics;
 
 use super::manager::MemoryManagerConfig;
 
-/// Internal state of [`LruWatermarkController`] that saves the state in previous tick.
-struct State {
-    pub used_memory_bytes: usize,
-    pub lru_watermark_step: u64,
-    pub lru_watermark_time_ms: u64,
+pub enum LruEvictionPolicy {
+    None,
+    Stable,
+    Graceful,
+    Aggressive,
 }
 
-impl Default for State {
-    fn default() -> Self {
-        let physical_now = Epoch::physical_now();
-        Self {
-            used_memory_bytes: 0,
-            lru_watermark_step: 0,
-            lru_watermark_time_ms: physical_now,
+impl From<LruEvictionPolicy> for u8 {
+    fn from(value: LruEvictionPolicy) -> Self {
+        match value {
+            LruEvictionPolicy::None => 0,
+            LruEvictionPolicy::Stable => 1,
+            LruEvictionPolicy::Graceful => 2,
+            LruEvictionPolicy::Aggressive => 3,
         }
     }
 }
@@ -75,8 +76,11 @@ pub struct LruWatermarkController {
     threshold_graceful: usize,
     threshold_aggressive: usize,
 
-    /// The state from previous tick
-    state: State,
+    eviction_factor_stable: f64,
+    eviction_factor_graceful: f64,
+    eviction_factor_aggressive: f64,
+
+    watermark_sequence: Sequence,
 }
 
 impl LruWatermarkController {
@@ -91,7 +95,10 @@ impl LruWatermarkController {
             threshold_stable,
             threshold_graceful,
             threshold_aggressive,
-            state: State::default(),
+            eviction_factor_stable: config.eviction_factor_stable,
+            eviction_factor_graceful: config.eviction_factor_graceful,
+            eviction_factor_aggressive: config.eviction_factor_aggressive,
+            watermark_sequence: 0,
         }
     }
 }
@@ -131,7 +138,7 @@ fn jemalloc_memory_stats() -> MemoryStats {
 }
 
 impl LruWatermarkController {
-    pub fn tick(&mut self, interval_ms: u32) -> Epoch {
+    pub fn tick(&mut self) -> Sequence {
         // NOTE: Be careful! The meaning of `allocated` and `active` differ in JeMalloc and JVM
         let MemoryStats {
             allocated: jemalloc_allocated_bytes,
@@ -143,87 +150,69 @@ impl LruWatermarkController {
 
         let cur_used_memory_bytes = jemalloc_active_bytes + jvm_allocated_bytes;
 
-        let last_step = self.state.lru_watermark_step;
-        let last_used_memory_bytes = self.state.used_memory_bytes;
-
-        // The watermark calculation works in the following way:
-        //
-        // 1. When the streaming memory usage is below the graceful threshold, we do not evict
-        // any caches, and simply reset the step to 0.
+        // To calculate the total amount of memory that needs to be evicted, we sequentially calculate and accumulate
+        // the memory amounts exceeding the thresholds for aggressive, graceful, and stable, multiplying each by
+        // different weights.
         //
-        // 2. When the memory usage is between the graceful and aggressive threshold:
-        //   - If the last eviction memory usage decreases after last eviction, we set the eviction step
-        //     to 1.
-        //   - Otherwise, we set the step to last_step + 1.
+        //   (range)                 : (weight)
+        // * aggressive ~ inf        : evict factor aggressive
+        // *   graceful ~ aggressive : evict factor graceful
+        // *     stable ~ graceful   : evict factor stable
+        // *          0 ~ stable     : no eviction
         //
-        // 3. When the memory usage exceeds the aggressive threshold:
-        //   - If the memory usage decreases after the last eviction, we set the eviction step to
-        //     last_step.
-        //   - Otherwise, we set the step to last_step * 2.
-        let mut step = if cur_used_memory_bytes < self.threshold_stable {
-            // Do not evict if the memory usage is lower than `threshold_stable`
-            0
-        } else if cur_used_memory_bytes < self.threshold_graceful {
-            // Evict in equal speed of time before `threshold_graceful`
-            1
-        } else if cur_used_memory_bytes < self.threshold_aggressive {
-            // Gracefully evict
-            if last_used_memory_bytes > cur_used_memory_bytes {
-                1
-            } else {
-                last_step + 1
-            }
-        } else if last_used_memory_bytes < cur_used_memory_bytes {
-            // Aggressively evict
-            if last_step == 0 {
-                2
-            } else {
-                last_step * 2
-            }
+        // Why different weights instead of 1.0? It acts like a penalty factor, used to penalize the system for not
+        // keeping the memory usage down at the lower thresholds.
+        let to_evict_bytes = cur_used_memory_bytes.saturating_sub(self.threshold_aggressive) as f64
+            * self.eviction_factor_aggressive
+            + cur_used_memory_bytes
+                .saturating_sub(self.threshold_graceful)
+                .min(self.threshold_aggressive - self.threshold_graceful) as f64
+                * self.eviction_factor_graceful
+            + cur_used_memory_bytes
+                .saturating_sub(self.threshold_stable)
+                .min(self.threshold_graceful - self.threshold_stable) as f64
+                * self.eviction_factor_stable;
+        let ratio = to_evict_bytes / cur_used_memory_bytes as f64;
+        let latest_sequence = SEQUENCE_GLOBAL.load(Ordering::Relaxed);
+        let sequence_diff =
+            ((latest_sequence - self.watermark_sequence) as f64 * ratio) as Sequence;
+        self.watermark_sequence = latest_sequence.min(self.watermark_sequence + sequence_diff);
+
+        let policy = if cur_used_memory_bytes > self.threshold_aggressive {
+            LruEvictionPolicy::Aggressive
+        } else if cur_used_memory_bytes > self.threshold_graceful {
+            LruEvictionPolicy::Graceful
+        } else if cur_used_memory_bytes > self.threshold_stable {
+            LruEvictionPolicy::Stable
         } else {
-            last_step
-        };
-
-        let physical_now = Epoch::physical_now();
-        let watermark_time_ms =
-            if (physical_now - self.state.lru_watermark_time_ms) / (interval_ms as u64) < step {
-                // We do not increase the step and watermark here to prevent a too-advanced watermark. The
-                // original condition is `prev_watermark_time_ms + interval_ms * step > now`.
-                step = last_step;
-                physical_now
-            } else {
-                // Increase by (steps * interval)
-                self.state.lru_watermark_time_ms + (interval_ms as u64 * step)
-            };
-
-        self.state = State {
-            used_memory_bytes: cur_used_memory_bytes,
-            lru_watermark_step: step,
-            lru_watermark_time_ms: watermark_time_ms,
+            LruEvictionPolicy::None
         };
 
+        self.metrics.lru_latest_sequence.set(latest_sequence as _);
         self.metrics
-            .lru_current_watermark_time_ms
-            .set(watermark_time_ms as i64);
-        self.metrics.lru_physical_now_ms.set(physical_now as i64);
-        self.metrics.lru_watermark_step.set(step as i64);
+            .lru_watermark_sequence
+            .set(self.watermark_sequence as _);
+        self.metrics
+            .lru_eviction_policy
+            .set(Into::<u8>::into(policy) as _);
+
         self.metrics
             .jemalloc_allocated_bytes
-            .set(jemalloc_allocated_bytes as i64);
+            .set(jemalloc_allocated_bytes as _);
         self.metrics
             .jemalloc_active_bytes
-            .set(jemalloc_active_bytes as i64);
+            .set(jemalloc_active_bytes as _);
         self.metrics
             .jemalloc_resident_bytes
-            .set(jemalloc_resident_bytes as i64);
+            .set(jemalloc_resident_bytes as _);
         self.metrics
             .jemalloc_metadata_bytes
-            .set(jemalloc_metadata_bytes as i64);
+            .set(jemalloc_metadata_bytes as _);
         self.metrics
             .jvm_allocated_bytes
-            .set(jvm_allocated_bytes as i64);
-        self.metrics.jvm_active_bytes.set(jvm_active_bytes as i64);
+            .set(jvm_allocated_bytes as _);
+        self.metrics.jvm_active_bytes.set(jvm_active_bytes as _);
 
-        Epoch::from_physical_time(watermark_time_ms)
+        self.watermark_sequence
     }
 }
diff --git a/src/compute/src/memory/manager.rs b/src/compute/src/memory/manager.rs
index 5e8e1b5be54ff..b90624193c703 100644
--- a/src/compute/src/memory/manager.rs
+++ b/src/compute/src/memory/manager.rs
@@ -16,6 +16,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
+use risingwave_common::sequence::AtomicSequence;
 use risingwave_common::system_param::local_manager::SystemParamsReaderRef;
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_stream::executor::monitor::StreamingMetrics;
@@ -29,13 +30,17 @@ pub struct MemoryManagerConfig {
     pub threshold_graceful: f64,
     pub threshold_stable: f64,
 
+    pub eviction_factor_stable: f64,
+    pub eviction_factor_graceful: f64,
+    pub eviction_factor_aggressive: f64,
+
     pub metrics: Arc<StreamingMetrics>,
 }
 
 /// Compute node uses [`MemoryManager`] to limit the memory usage.
 pub struct MemoryManager {
     /// All cached data before the watermark should be evicted.
-    watermark_epoch: Arc<AtomicU64>,
+    watermark_sequence: Arc<AtomicSequence>,
 
     metrics: Arc<StreamingMetrics>,
 
@@ -52,14 +57,14 @@ impl MemoryManager {
         tracing::info!("LRU watermark controller: {:?}", &controller);
 
         Arc::new(Self {
-            watermark_epoch: Arc::new(0.into()),
+            watermark_sequence: Arc::new(0.into()),
             metrics: config.metrics,
             controller,
         })
     }
 
-    pub fn get_watermark_epoch(&self) -> Arc<AtomicU64> {
-        self.watermark_epoch.clone()
+    pub fn get_watermark_sequence(&self) -> Arc<AtomicU64> {
+        self.watermark_sequence.clone()
     }
 
     pub async fn run(
@@ -81,20 +86,21 @@ impl MemoryManager {
             // Wait for a while to check if need eviction.
             tokio::select! {
                 Ok(_) = system_params_change_rx.changed() => {
-                    let params = system_params_change_rx.borrow().load();
-                    let new_interval_ms = std::cmp::max(params.barrier_interval_ms(), Self::MIN_TICK_INTERVAL_MS);
-                    if new_interval_ms != interval_ms {
-                        interval_ms = new_interval_ms;
-                        tick_interval = tokio::time::interval(Duration::from_millis(interval_ms as u64));
-                        tracing::info!("updated MemoryManager interval to {}ms", interval_ms);
-                    }
+                     let params = system_params_change_rx.borrow().load();
+                     let new_interval_ms = std::cmp::max(params.barrier_interval_ms(), Self::MIN_TICK_INTERVAL_MS);
+                     if new_interval_ms != interval_ms {
+                         interval_ms = new_interval_ms;
+                         tick_interval = tokio::time::interval(Duration::from_millis(interval_ms as u64));
+                         tracing::info!("updated MemoryManager interval to {}ms", interval_ms);
+                     }
                 }
 
                 _ = tick_interval.tick() => {
-                    let new_watermark_epoch = self.controller.lock().unwrap().tick(interval_ms);
-                    self.watermark_epoch.store(new_watermark_epoch.0, Ordering::Relaxed);
+                     let new_watermark_sequence = self.controller.lock().unwrap().tick();
+
+                     self.watermark_sequence.store(new_watermark_sequence, Ordering::Relaxed);
 
-                    self.metrics.lru_runtime_loop_count.inc();
+                     self.metrics.lru_runtime_loop_count.inc();
                 }
             }
         }
diff --git a/src/compute/src/rpc/service/config_service.rs b/src/compute/src/rpc/service/config_service.rs
index 97793f71722c9..e827ef176413e 100644
--- a/src/compute/src/rpc/service/config_service.rs
+++ b/src/compute/src/rpc/service/config_service.rs
@@ -18,7 +18,6 @@ use risingwave_common::error::tonic::ToTonicStatus;
 use risingwave_pb::compute::config_service_server::ConfigService;
 use risingwave_pb::compute::{ShowConfigRequest, ShowConfigResponse};
 use risingwave_stream::task::LocalStreamManager;
-use serde_json;
 use tonic::{Code, Request, Response, Status};
 
 pub struct ConfigServiceImpl {
diff --git a/src/compute/src/rpc/service/monitor_service.rs b/src/compute/src/rpc/service/monitor_service.rs
index d2542ca9bd085..e1c6247296b00 100644
--- a/src/compute/src/rpc/service/monitor_service.rs
+++ b/src/compute/src/rpc/service/monitor_service.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::BTreeMap;
 use std::ffi::CString;
 use std::fs;
 use std::path::Path;
@@ -21,6 +22,7 @@ use itertools::Itertools;
 use prometheus::core::Collector;
 use risingwave_common::config::{MetricLevel, ServerConfig};
 use risingwave_common_heap_profiling::{AUTO_DUMP_SUFFIX, COLLAPSED_SUFFIX, MANUALLY_DUMP_SUFFIX};
+use risingwave_jni_core::jvm_runtime::dump_jvm_stack_traces;
 use risingwave_pb::monitor_service::monitor_service_server::MonitorService;
 use risingwave_pb::monitor_service::{
     AnalyzeHeapRequest, AnalyzeHeapResponse, BackPressureInfo, GetBackPressureRequest,
@@ -99,11 +101,29 @@ impl MonitorService for MonitorServiceImpl {
             Default::default()
         };
 
+        let barrier_worker_state = self.stream_mgr.inspect_barrier_state().await?;
+
+        let jvm_stack_traces = match dump_jvm_stack_traces() {
+            Ok(None) => None,
+            Err(err) => Some(err.as_report().to_string()),
+            Ok(Some(stack_traces)) => Some(stack_traces),
+        };
+
         Ok(Response::new(StackTraceResponse {
             actor_traces,
             rpc_traces,
             compaction_task_traces,
             inflight_barrier_traces: barrier_traces,
+            barrier_worker_state: BTreeMap::from_iter([(
+                self.stream_mgr.env.worker_id(),
+                barrier_worker_state,
+            )]),
+            jvm_stack_traces: match jvm_stack_traces {
+                Some(stack_traces) => {
+                    BTreeMap::from_iter([(self.stream_mgr.env.worker_id(), stack_traces)])
+                }
+                None => BTreeMap::new(),
+            },
         }))
     }
 
diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs
index bdf47cbd1309f..21ec9de730228 100644
--- a/src/compute/src/server.rs
+++ b/src/compute/src/server.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::net::SocketAddr;
-use std::sync::atomic::AtomicU32;
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -21,11 +20,13 @@ use risingwave_batch::monitor::{
     GLOBAL_BATCH_EXECUTOR_METRICS, GLOBAL_BATCH_MANAGER_METRICS, GLOBAL_BATCH_TASK_METRICS,
 };
 use risingwave_batch::rpc::service::task_service::BatchServiceImpl;
+use risingwave_batch::spill::spill_op::SpillOp;
 use risingwave_batch::task::{BatchEnvironment, BatchManager};
 use risingwave_common::config::{
     load_config, AsyncStackTraceOption, MetricLevel, StorageMemoryConfig,
     MAX_CONNECTION_WINDOW_SIZE, STREAM_WINDOW_SIZE,
 };
+use risingwave_common::lru::init_global_sequencer_args;
 use risingwave_common::monitor::connection::{RouterExt, TcpConfig};
 use risingwave_common::system_param::local_manager::LocalSystemParamsManager;
 use risingwave_common::system_param::reader::SystemParamsRead;
@@ -42,7 +43,6 @@ use risingwave_connector::source::monitor::GLOBAL_SOURCE_METRICS;
 use risingwave_dml::dml_manager::DmlManager;
 use risingwave_pb::common::WorkerType;
 use risingwave_pb::compute::config_service_server::ConfigServiceServer;
-use risingwave_pb::connector_service::SinkPayloadFormat;
 use risingwave_pb::health::health_server::HealthServer;
 use risingwave_pb::meta::add_worker_node_request::Property;
 use risingwave_pb::monitor_service::monitor_service_server::MonitorServiceServer;
@@ -68,7 +68,9 @@ use tokio::sync::oneshot::Sender;
 use tokio::task::JoinHandle;
 use tower::Layer;
 
-use crate::memory::config::{reserve_memory_bytes, storage_memory_config, MIN_COMPUTE_MEMORY_MB};
+use crate::memory::config::{
+    batch_mem_limit, reserve_memory_bytes, storage_memory_config, MIN_COMPUTE_MEMORY_MB,
+};
 use crate::memory::manager::{MemoryManager, MemoryManagerConfig};
 use crate::observer::observer_manager::ComputeObserverNode;
 use crate::rpc::service::config_service::ConfigServiceImpl;
@@ -101,9 +103,21 @@ pub async fn compute_node_serve(
     let stream_config = Arc::new(config.streaming.clone());
     let batch_config = Arc::new(config.batch.clone());
 
+    // Initialize operator lru cache global sequencer args.
+    init_global_sequencer_args(
+        config
+            .streaming
+            .developer
+            .memory_controller_sequence_tls_step,
+        config
+            .streaming
+            .developer
+            .memory_controller_sequence_tls_lag,
+    );
+
     // Register to the cluster. We're not ready to serve until activate is called.
     let (meta_client, system_params) = MetaClient::register_new(
-        opts.meta_address,
+        opts.meta_address.clone(),
         WorkerType::ComputeNode,
         &advertise_addr,
         Property {
@@ -122,12 +136,12 @@ pub async fn compute_node_serve(
     let embedded_compactor_enabled =
         embedded_compactor_enabled(state_store_url, config.storage.disable_remote_compactor);
 
-    let (reserved_memory_bytes, non_reserved_memory_bytes) =
-        reserve_memory_bytes(opts.total_memory_bytes);
+    let (reserved_memory_bytes, non_reserved_memory_bytes) = reserve_memory_bytes(&opts);
     let storage_memory_config = storage_memory_config(
         non_reserved_memory_bytes,
         embedded_compactor_enabled,
         &config.storage,
+        !opts.role.for_streaming(),
     );
 
     let storage_memory_bytes = total_storage_memory_limit_bytes(&storage_memory_config);
@@ -216,12 +230,6 @@ pub async fn compute_node_serve(
             ));
 
             let compaction_executor = Arc::new(CompactionExecutor::new(Some(1)));
-            let max_task_parallelism = Arc::new(AtomicU32::new(
-                (compaction_executor.worker_num() as f32
-                    * storage_opts.compactor_max_task_multiplier)
-                    .ceil() as u32,
-            ));
-
             let compactor_context = CompactorContext {
                 storage_opts,
                 sstable_store: storage.sstable_store(),
@@ -234,8 +242,6 @@ pub async fn compute_node_serve(
                 await_tree_reg: await_tree_config
                     .clone()
                     .map(new_compaction_await_tree_reg_ref),
-                running_task_parallelism: Arc::new(AtomicU32::new(0)),
-                max_task_parallelism,
             };
 
             let (handle, shutdown_sender) = start_compactor(
@@ -272,6 +278,7 @@ pub async fn compute_node_serve(
     let batch_mgr = Arc::new(BatchManager::new(
         config.batch.clone(),
         batch_manager_metrics,
+        batch_mem_limit(compute_memory_bytes),
     ));
 
     // NOTE: Due to some limits, we use `compute_memory_bytes + storage_memory_bytes` as
@@ -295,6 +302,18 @@ pub async fn compute_node_serve(
             .streaming
             .developer
             .memory_controller_threshold_stable,
+        eviction_factor_stable: config
+            .streaming
+            .developer
+            .memory_controller_eviction_factor_stable,
+        eviction_factor_graceful: config
+            .streaming
+            .developer
+            .memory_controller_eviction_factor_graceful,
+        eviction_factor_aggressive: config
+            .streaming
+            .developer
+            .memory_controller_eviction_factor_aggressive,
         metrics: streaming_metrics.clone(),
     });
 
@@ -332,28 +351,9 @@ pub async fn compute_node_serve(
         config.server.metrics_level,
     );
 
-    info!(
-        "connector param: payload_format={:?}",
-        opts.connector_rpc_sink_payload_format
-    );
-
-    let connector_params = risingwave_connector::ConnectorParams {
-        sink_payload_format: match opts.connector_rpc_sink_payload_format.as_deref() {
-            None | Some("stream_chunk") => SinkPayloadFormat::StreamChunk,
-            Some("json") => SinkPayloadFormat::Json,
-            _ => {
-                unreachable!(
-                    "invalid sink payload format: {:?}. Should be either json or stream_chunk",
-                    opts.connector_rpc_sink_payload_format
-                )
-            }
-        },
-    };
-
     // Initialize the streaming environment.
     let stream_env = StreamEnvironment::new(
         advertise_addr.clone(),
-        connector_params,
         stream_config,
         worker_id,
         state_store,
@@ -367,7 +367,7 @@ pub async fn compute_node_serve(
         stream_env.clone(),
         streaming_metrics.clone(),
         await_tree_config.clone(),
-        memory_mgr.get_watermark_epoch(),
+        memory_mgr.get_watermark_sequence(),
     );
 
     // Boot the runtime gRPC services.
@@ -392,6 +392,10 @@ pub async fn compute_node_serve(
         tracing::info!("Telemetry didn't start due to config");
     }
 
+    // Clean up the spill directory.
+    #[cfg(not(madsim))]
+    SpillOp::clean_spill_directory().await.unwrap();
+
     let (shutdown_send, mut shutdown_recv) = tokio::sync::oneshot::channel::<()>();
     let join_handle = tokio::spawn(async move {
         tonic::transport::Server::builder()
diff --git a/src/compute/src/telemetry.rs b/src/compute/src/telemetry.rs
index 143ec51c5bc39..c35a6063ae267 100644
--- a/src/compute/src/telemetry.rs
+++ b/src/compute/src/telemetry.rs
@@ -33,6 +33,7 @@ impl ComputeTelemetryCreator {
 
 #[async_trait::async_trait]
 impl TelemetryReportCreator for ComputeTelemetryCreator {
+    #[allow(refining_impl_trait)]
     async fn create_report(
         &self,
         tracking_id: String,
diff --git a/src/compute/tests/cdc_tests.rs b/src/compute/tests/cdc_tests.rs
index 760c76ca6deba..26f822bae558f 100644
--- a/src/compute/tests/cdc_tests.rs
+++ b/src/compute/tests/cdc_tests.rs
@@ -25,14 +25,17 @@ use futures::stream::StreamExt;
 use futures_async_stream::try_stream;
 use itertools::Itertools;
 use risingwave_batch::executor::{Executor as BatchExecutor, RowSeqScanExecutor, ScanRange};
-use risingwave_common::array::{Array, ArrayBuilder, DataChunk, Op, StreamChunk, Utf8ArrayBuilder};
+use risingwave_common::array::{
+    Array, ArrayBuilder, DataChunk, DataChunkTestExt, Op, StreamChunk, Utf8ArrayBuilder,
+};
 use risingwave_common::catalog::{ColumnDesc, ColumnId, ConflictBehavior, Field, Schema, TableId};
 use risingwave_common::types::{Datum, JsonbVal};
 use risingwave_common::util::epoch::{test_epoch, EpochExt};
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use risingwave_connector::source::cdc::external::mock_external_table::MockExternalTableReader;
+use risingwave_connector::source::cdc::external::mysql::MySqlOffset;
 use risingwave_connector::source::cdc::external::{
-    DebeziumOffset, DebeziumSourceOffset, ExternalTableReaderImpl, MySqlOffset, SchemaTableName,
+    DebeziumOffset, DebeziumSourceOffset, ExternalTableReaderImpl, SchemaTableName,
 };
 use risingwave_connector::source::cdc::DebeziumCdcSplit;
 use risingwave_connector::source::SplitImpl;
@@ -45,8 +48,8 @@ use risingwave_stream::executor::monitor::StreamingMetrics;
 use risingwave_stream::executor::test_utils::MockSource;
 use risingwave_stream::executor::{
     expect_first_barrier, ActorContext, AddMutation, Barrier, BoxedMessageStream,
-    CdcBackfillExecutor, Execute, Executor as StreamExecutor, ExecutorInfo, ExternalStorageTable,
-    MaterializeExecutor, Message, Mutation, StreamExecutorError,
+    CdcBackfillExecutor, CdcScanOptions, Execute, Executor as StreamExecutor, ExecutorInfo,
+    ExternalStorageTable, MaterializeExecutor, Message, Mutation, StreamExecutorError,
 };
 
 // mock upstream binlog offset starting from "1.binlog, pos=0"
@@ -167,7 +170,10 @@ async fn test_cdc_backfill() -> StreamResult<()> {
         MySqlOffset::new(binlog_file.clone(), 10),
     ];
 
-    let table_name = SchemaTableName::new("mock_table".to_string(), "public".to_string());
+    let table_name = SchemaTableName {
+        schema_name: "public".to_string(),
+        table_name: "mock_table".to_string(),
+    };
     let table_schema = Schema::new(vec![
         Field::with_name(DataType::Int64, "id"), // primary key
         Field::with_name(DataType::Float64, "price"),
@@ -177,15 +183,16 @@ async fn test_cdc_backfill() -> StreamResult<()> {
     let external_table = ExternalStorageTable::new(
         TableId::new(1234),
         table_name,
+        "mydb".to_string(),
         ExternalTableReaderImpl::Mock(MockExternalTableReader::new(binlog_watermarks)),
         table_schema.clone(),
         table_pk_order_types,
         table_pk_indices.clone(),
-        vec![0, 1],
     );
 
     let actor_id = 0x1a;
 
+    // create state table
     let state_schema = Schema::new(vec![
         Field::with_name(DataType::Varchar, "split_id"),
         Field::with_name(DataType::Int64, "id"), // pk
@@ -211,6 +218,11 @@ async fn test_cdc_backfill() -> StreamResult<()> {
     )
     .await;
 
+    let output_columns = vec![
+        ColumnDesc::named("id", ColumnId::new(1), DataType::Int64), // primary key
+        ColumnDesc::named("price", ColumnId::new(2), DataType::Float64),
+    ];
+
     let cdc_backfill = StreamExecutor::new(
         ExecutorInfo {
             schema: table_schema.clone(),
@@ -222,11 +234,16 @@ async fn test_cdc_backfill() -> StreamResult<()> {
             external_table,
             mock_offset_executor,
             vec![0, 1],
+            output_columns,
             None,
             Arc::new(StreamingMetrics::unused()),
             state_table,
-            4, // 4 rows in a snapshot chunk
-            false,
+            Some(4), // limit a snapshot chunk to have <= 4 rows by rate limit
+            CdcScanOptions {
+                disable_backfill: false,
+                snapshot_interval: 1,
+                snapshot_batch_size: 4,
+            },
         )
         .boxed(),
     );
@@ -246,9 +263,10 @@ async fn test_cdc_backfill() -> StreamResult<()> {
     .boxed()
     .execute();
 
+    // construct upstream chunks
     let chunk1_payload = vec![
         r#"{ "payload": { "before": null, "after": { "id": 1, "price": 10.01}, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002"}, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
-        r#"{ "payload": { "before": null, "after": { "id": 2, "price": 2.02}, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002"}, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
+        r#"{ "payload": { "before": null, "after": { "id": 2, "price": 22.22}, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002"}, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
         r#"{ "payload": { "before": null, "after": { "id": 3, "price": 3.03}, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002"}, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
         r#"{ "payload": { "before": null, "after": { "id": 4, "price": 4.04}, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002"}, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
         r#"{ "payload": { "before": null, "after": { "id": 5, "price": 5.05}, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002"}, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
@@ -256,6 +274,7 @@ async fn test_cdc_backfill() -> StreamResult<()> {
     ];
 
     let chunk2_payload = vec![
+        r#"{ "payload": { "before": null, "after": { "id": 1, "price": 11.11}, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002"}, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
         r#"{ "payload": { "before": null, "after": { "id": 6, "price": 10.08}, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002"}, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
         r#"{ "payload": { "before": null, "after": { "id": 199, "price": 40.5}, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002"}, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
         r#"{ "payload": { "before": null, "after": { "id": 978, "price": 72.6}, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002"}, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#,
@@ -311,11 +330,22 @@ async fn test_cdc_backfill() -> StreamResult<()> {
 
     // ingest data and barrier
     let interval = Duration::from_millis(10);
+
+    // send a dummy barrier to trigger the backfill, since
+    // cdc backfill will wait for a barrier before start
+    curr_epoch.inc_epoch();
+    tx.push_barrier(curr_epoch, false);
+
+    // first chunk
     tx.push_chunk(stream_chunk1);
+
     tokio::time::sleep(interval).await;
+
+    // barrier to trigger emit buffered events
     curr_epoch.inc_epoch();
     tx.push_barrier(curr_epoch, false);
 
+    // second chunk
     tx.push_chunk(stream_chunk2);
 
     tokio::time::sleep(interval).await;
@@ -352,9 +382,25 @@ async fn test_cdc_backfill() -> StreamResult<()> {
         None,
         None,
     ));
+
+    // check result
     let mut stream = scan.execute();
     while let Some(message) = stream.next().await {
-        println!("[scan] chunk: {:#?}", message.unwrap());
+        let chunk = message.expect("scan a chunk");
+        let expect = DataChunk::from_pretty(
+            "I F
+                1 11.11
+                2 22.22
+                3 3.03
+                4 4.04
+                5 5.05
+                6 10.08
+                8 1.0008
+                134 41.7
+                199 40.5
+                978 72.6",
+        );
+        assert_eq!(expect, chunk);
     }
 
     mview_handle.await.unwrap()?;
diff --git a/src/compute/tests/integration_tests.rs b/src/compute/tests/integration_tests.rs
index f34a2940e9ce5..14dbfed466158 100644
--- a/src/compute/tests/integration_tests.rs
+++ b/src/compute/tests/integration_tests.rs
@@ -21,7 +21,7 @@ use std::sync::Arc;
 use futures::stream::StreamExt;
 use futures_async_stream::try_stream;
 use itertools::Itertools;
-use maplit::{convert_args, hashmap};
+use maplit::{btreemap, convert_args};
 use risingwave_batch::error::BatchError;
 use risingwave_batch::executor::{
     BoxedDataChunkStream, BoxedExecutor, DeleteExecutor, Executor as BatchExecutor, InsertExecutor,
@@ -40,7 +40,6 @@ use risingwave_common::util::epoch::{test_epoch, EpochExt, EpochPair};
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use risingwave_connector::source::reader::desc::test_utils::create_source_desc_builder;
-use risingwave_connector::source::SourceCtrlOpts;
 use risingwave_dml::dml_manager::DmlManager;
 use risingwave_hummock_sdk::to_committed_batch_query_epoch;
 use risingwave_pb::catalog::StreamSourceInfo;
@@ -115,7 +114,7 @@ async fn test_table_materialize() -> StreamResult<()> {
         row_format: PbRowFormatType::Json as i32,
         ..Default::default()
     };
-    let properties = convert_args!(hashmap!(
+    let properties = convert_args!(btreemap!(
         "connector" => "datagen",
         "fields.v1.min" => "1",
         "fields.v1.max" => "1000",
@@ -173,10 +172,8 @@ async fn test_table_materialize() -> StreamResult<()> {
             Arc::new(StreamingMetrics::unused()),
             barrier_rx,
             system_params_manager.get_params(),
-            SourceCtrlOpts {
-                chunk_size: 1024,
-                rate_limit: None,
-            },
+            None,
+            false,
         )
         .boxed(),
     );
diff --git a/src/config/ci-recovery.toml b/src/config/ci-recovery.toml
index 95e5834621115..69de86279c1c5 100644
--- a/src/config/ci-recovery.toml
+++ b/src/config/ci-recovery.toml
@@ -10,5 +10,6 @@ imm_merge_threshold = 2
 
 [system]
 barrier_interval_ms = 250
+# If this is changed, remember to also change e2e_test/source_inline/pubsub/pubsub.slt
 checkpoint_frequency = 5
 max_concurrent_creating_streaming_jobs = 0
diff --git a/src/config/ci.toml b/src/config/ci.toml
index 02e1509546f27..43e9836bd304f 100644
--- a/src/config/ci.toml
+++ b/src/config/ci.toml
@@ -16,6 +16,12 @@ stream_exchange_concurrent_barriers = 10
 [storage]
 imm_merge_threshold = 2
 
+[storage.object_store.retry]
+streaming_upload_attempt_timeout_ms = 10000
+upload_retry_attempts = 5
+read_attempt_timeout_ms = 16000
+read_retry_attempts = 6
+
 [system]
 barrier_interval_ms = 250
 checkpoint_frequency = 5
diff --git a/src/config/docs.md b/src/config/docs.md
index eacd8039a5d75..ab33559260162 100644
--- a/src/config/docs.md
+++ b/src/config/docs.md
@@ -8,9 +8,11 @@ This page is automatically generated by `./risedev generate-example-config`
 |--------|-------------|---------|
 | distributed_query_limit | This is the max number of queries per sql session. |  |
 | enable_barrier_read |  | false |
+| enable_spill | Enable the spill out to disk feature for batch queries. | true |
 | frontend_compute_runtime_worker_threads | frontend compute runtime worker threads | 4 |
 | mask_worker_temporary_secs | This is the secs used to mask a worker unavailable temporarily. | 30 |
 | max_batch_queries_per_frontend_node | This is the max number of batch queries per frontend node. |  |
+| redact_sql_option_keywords | Keywords on which SQL option redaction is based in the query log. A SQL option with a name containing any of these keywords will be redacted. | ["credential", "key", "password", "private", "secret", "token"] |
 | statement_timeout_in_sec | Timeout for a batch query in seconds. | 3600 |
 | worker_threads_num | The thread number of the batch task runtime in the compute node. The default value is decided by `tokio`. |  |
 
@@ -20,6 +22,8 @@ This page is automatically generated by `./risedev generate-example-config`
 |--------|-------------|---------|
 | backend |  | "Mem" |
 | collect_gc_watermark_spin_interval_sec | The spin interval when collecting global GC watermark in hummock. | 5 |
+| compact_task_table_size_partition_threshold_high | The threshold of table size in one compact task to decide whether to partition one table into `partition_vnode_count` parts, which belongs to default group and materialized view group. Set it max value of 64-bit number to disable this feature. | 536870912 |
+| compact_task_table_size_partition_threshold_low | The threshold of table size in one compact task to decide whether to partition one table into `hybrid_partition_vnode_count` parts, which belongs to default group and materialized view group. Set it max value of 64-bit number to disable this feature. | 134217728 |
 | compaction_task_max_heartbeat_interval_secs |  | 30 |
 | compaction_task_max_progress_interval_secs |  | 600 |
 | cut_table_size_limit |  | 1073741824 |
@@ -36,7 +40,7 @@ This page is automatically generated by `./risedev generate-example-config`
 | event_log_enabled |  | true |
 | full_gc_interval_sec | Interval of automatic hummock full GC. | 86400 |
 | hummock_version_checkpoint_interval_sec | Interval of hummock version checkpoint. | 30 |
-| hybird_partition_vnode_count |  | 4 |
+| hybrid_partition_vnode_count | Count of partitions of tables in default group and materialized view group. The meta node will decide according to some strategy whether to cut the boundaries of the file according to the vnode alignment. Each partition contains aligned data of `VirtualNode::COUNT / hybrid_partition_vnode_count` consecutive virtual-nodes of one state table. Set it zero to disable this feature. | 4 |
 | max_heartbeat_interval_secs | Maximum allowed heartbeat interval in seconds. | 60 |
 | meta_leader_lease_secs |  | 30 |
 | min_delta_log_num_for_hummock_version_checkpoint | The minimum delta log number a new checkpoint should compact, otherwise the checkpoint attempt is rejected. | 10 |
@@ -47,14 +51,15 @@ This page is automatically generated by `./risedev generate-example-config`
 | parallelism_control_batch_size | The number of streaming jobs per scaling operation. | 10 |
 | parallelism_control_trigger_first_delay_sec | The first delay of parallelism control. | 30 |
 | parallelism_control_trigger_period_sec | The period of parallelism control trigger. | 10 |
-| partition_vnode_count |  | 16 |
+| partition_vnode_count | Count of partition in split group. Meta will assign this value to every new group when it splits from default-group by automatically. Each partition contains aligned data of `VirtualNode::COUNT / partition_vnode_count` consecutive virtual-nodes of one state table. | 16 |
 | periodic_compaction_interval_sec | Schedule compaction for all compaction groups with this interval. | 60 |
 | periodic_space_reclaim_compaction_interval_sec | Schedule `space_reclaim` compaction for all compaction groups with this interval. | 3600 |
 | periodic_split_compact_group_interval_sec |  | 10 |
 | periodic_tombstone_reclaim_compaction_interval_sec |  | 600 |
 | periodic_ttl_reclaim_compaction_interval_sec | Schedule `ttl_reclaim` compaction for all compaction groups with this interval. | 1800 |
+| secret_store_private_key |  | [100, 101, 109, 111, 45, 115, 101, 99, 114, 101, 116, 45, 112, 114, 105, 118, 97, 116, 101, 45, 107, 101, 121] |
 | split_group_size_limit |  | 68719476736 |
-| table_write_throughput_threshold |  | 16777216 |
+| table_write_throughput_threshold | The threshold of write throughput to trigger a group split. Increase this configuration value to avoid split too many groups with few data write. | 16777216 |
 | unrecognized |  |  |
 | vacuum_interval_sec | Interval of invoking a vacuum job, to remove stale metadata from meta store and objects from object store. | 30 |
 | vacuum_spin_interval_ms | The spin interval inside a vacuum job. It avoids the vacuum job monopolizing resources of meta node. | 10 |
@@ -73,6 +78,7 @@ This page is automatically generated by `./risedev generate-example-config`
 | max_bytes_for_level_base |  | 536870912 |
 | max_bytes_for_level_multiplier |  | 5 |
 | max_compaction_bytes |  | 2147483648 |
+| max_level |  | 6 |
 | max_space_reclaim_bytes |  | 536870912 |
 | max_sub_compaction |  | 4 |
 | sub_level_max_compaction_bytes |  | 134217728 |
@@ -99,36 +105,38 @@ This page is automatically generated by `./risedev generate-example-config`
 | cache_refill |  |  |
 | check_compaction_result |  | false |
 | compact_iter_recreate_timeout_ms |  | 600000 |
+| compactor_concurrent_uploading_sst_count | The concurrent uploading number of `SSTables` of buidler |  |
 | compactor_fast_max_compact_delete_ratio |  | 40 |
 | compactor_fast_max_compact_task_size |  | 2147483648 |
 | compactor_iter_max_io_retry_times |  | 8 |
 | compactor_max_sst_key_count |  | 2097152 |
 | compactor_max_sst_size |  | 536870912 |
-| compactor_max_task_multiplier | Compactor calculates the maximum number of tasks that can be executed on the node based on `worker_num` and `compactor_max_task_multiplier`. `max_pull_task_count` = `worker_num` * `compactor_max_task_multiplier` | 2.5 |
+| compactor_max_task_multiplier | Compactor calculates the maximum number of tasks that can be executed on the node based on `worker_num` and `compactor_max_task_multiplier`. `max_pull_task_count` = `worker_num` * `compactor_max_task_multiplier` | 3.0 |
 | compactor_memory_available_proportion | The percentage of memory available when compactor is deployed separately. `non_reserved_memory_bytes` = `system_memory_available_bytes` * `compactor_memory_available_proportion` | 0.8 |
 | compactor_memory_limit_mb |  |  |
 | data_file_cache |  |  |
 | disable_remote_compactor |  | false |
-| enable_fast_compaction |  | false |
+| enable_fast_compaction |  | true |
 | high_priority_ratio_in_percent | DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_eviction.high_priority_ratio_in_percent` with `storage.cache.block_cache_eviction.algorithm = "Lru"` instead. |  |
 | imm_merge_threshold | The threshold for the number of immutable memtables to merge to a new imm. | 0 |
 | max_concurrent_compaction_task_number |  | 16 |
 | max_prefetch_block_number | max prefetch block number | 16 |
 | max_preload_io_retry_times |  | 3 |
 | max_preload_wait_time_mill |  | 0 |
-| max_sub_compaction | Max sub compaction task numbers | 4 |
 | max_version_pinning_duration_sec |  | 10800 |
 | mem_table_spill_threshold | The spill threshold for mem table. | 4194304 |
 | meta_cache_capacity_mb | DEPRECATED: This config will be deprecated in the future version, use `storage.cache.meta_cache_capacity_mb` instead. |  |
 | meta_file_cache |  |  |
 | min_sst_size_for_streaming_upload | Whether to enable streaming upload for sstable. | 33554432 |
-| object_store |  |  |
+| min_sstable_size_mb |  | 32 |
+| object_store | Object storage configuration 1. General configuration 2. Some special configuration of Backend 3. Retry and timeout configuration |  |
 | prefetch_buffer_capacity_mb | max memory usage for large query |  |
 | share_buffer_compaction_worker_threads_number | Worker threads number of dedicated tokio runtime for share buffer compaction. 0 means use tokio's default value (number of CPU core). | 4 |
 | share_buffer_upload_concurrency | Number of tasks shared buffer can upload in parallel. | 8 |
 | share_buffers_sync_parallelism | parallelism while syncing share buffers into L0 SST. Should NOT be 0. | 1 |
 | shared_buffer_capacity_mb | Maximum shared buffer size, writes attempting to exceed the capacity will stall until there is enough space. |  |
 | shared_buffer_flush_ratio | The shared buffer will start flushing data to object when the ratio of memory usage to the shared buffer capacity exceed such ratio. | 0.800000011920929 |
+| shared_buffer_min_batch_flush_size_mb | The minimum total flush size of shared buffer spill. When a shared buffer spilled is trigger, the total flush size across multiple epochs should be at least higher than this size. | 800 |
 | sstable_id_remote_fetch_number | Number of SST ids fetched from meta per RPC | 10 |
 | write_conflict_detection_enabled | Whether to enable write conflict detection | true |
 
@@ -155,7 +163,7 @@ This page is automatically generated by `./risedev generate-example-config`
 | data_directory | Remote directory for storing data and metadata objects. |  |
 | enable_tracing | Whether to enable distributed tracing. | false |
 | max_concurrent_creating_streaming_jobs | Max number of concurrent creating streaming jobs. | 1 |
-| parallel_compact_size_mb |  | 512 |
+| parallel_compact_size_mb | The size of parallel task for one compact/flush job. | 512 |
 | pause_on_next_bootstrap | Whether to pause all data sources on next bootstrap. | false |
 | sstable_size_mb | Target size of the Sstable. | 256 |
-| state_store |  |  |
+| state_store | URL for the state store |  |
diff --git a/src/config/example.toml b/src/config/example.toml
index 397bf9a89f69b..b35590c85059b 100644
--- a/src/config/example.toml
+++ b/src/config/example.toml
@@ -48,10 +48,13 @@ table_write_throughput_threshold = 16777216
 min_table_split_write_throughput = 4194304
 compaction_task_max_heartbeat_interval_secs = 30
 compaction_task_max_progress_interval_secs = 600
-hybird_partition_vnode_count = 4
+hybrid_partition_vnode_count = 4
+compact_task_table_size_partition_threshold_low = 134217728
+compact_task_table_size_partition_threshold_high = 536870912
 event_log_enabled = true
 event_log_channel_max_size = 10
 enable_dropped_column_reclaim = false
+secret_store_private_key = [100, 101, 109, 111, 45, 115, 101, 99, 114, 101, 116, 45, 112, 114, 105, 118, 97, 116, 101, 45, 107, 101, 121]
 
 [meta.compaction_config]
 max_bytes_for_level_base = 536870912
@@ -69,18 +72,23 @@ max_space_reclaim_bytes = 536870912
 level0_max_compact_file_number = 100
 tombstone_reclaim_ratio = 40
 enable_emergency_picker = true
+max_level = 6
 
 [meta.developer]
 meta_cached_traces_num = 256
 meta_cached_traces_memory_limit_bytes = 134217728
 meta_enable_trivial_move = true
 meta_enable_check_task_level_overlap = false
+meta_max_trivial_move_task_count_per_loop = 256
+meta_max_get_task_probe_times = 5
 
 [batch]
 enable_barrier_read = false
 statement_timeout_in_sec = 3600
 frontend_compute_runtime_worker_threads = 4
 mask_worker_temporary_secs = 30
+redact_sql_option_keywords = ["credential", "key", "password", "private", "secret", "token"]
+enable_spill = true
 
 [batch.developer]
 batch_connector_message_buffer_size = 16
@@ -107,29 +115,36 @@ stream_hash_agg_max_dirty_groups_heap_size = 67108864
 stream_memory_controller_threshold_aggressive = 0.9
 stream_memory_controller_threshold_graceful = 0.8
 stream_memory_controller_threshold_stable = 0.7
+stream_memory_controller_eviction_factor_aggressive = 2.0
+stream_memory_controller_eviction_factor_graceful = 1.5
+stream_memory_controller_eviction_factor_stable = 1.0
+stream_memory_controller_sequence_tls_step = 128
+stream_memory_controller_sequence_tls_lag = 32
 stream_enable_arrangement_backfill = true
+stream_high_join_amplification_threshold = 2048
 
 [storage]
 share_buffers_sync_parallelism = 1
 share_buffer_compaction_worker_threads_number = 4
 shared_buffer_flush_ratio = 0.800000011920929
+shared_buffer_min_batch_flush_size_mb = 800
 imm_merge_threshold = 0
 write_conflict_detection_enabled = true
 max_prefetch_block_number = 16
 disable_remote_compactor = false
 share_buffer_upload_concurrency = 8
-compactor_max_task_multiplier = 2.5
+compactor_max_task_multiplier = 3.0
 compactor_memory_available_proportion = 0.8
 sstable_id_remote_fetch_number = 10
+min_sstable_size_mb = 32
 min_sst_size_for_streaming_upload = 33554432
-max_sub_compaction = 4
 max_concurrent_compaction_task_number = 16
 max_preload_wait_time_mill = 0
 max_version_pinning_duration_sec = 10800
 compactor_max_sst_key_count = 2097152
 compact_iter_recreate_timeout_ms = 600000
 compactor_max_sst_size = 536870912
-enable_fast_compaction = false
+enable_fast_compaction = true
 check_compaction_result = false
 max_preload_io_retry_times = 3
 compactor_fast_max_compact_delete_ratio = 40
@@ -147,30 +162,22 @@ algorithm = "Lru"
 dir = ""
 capacity_mb = 1024
 file_capacity_mb = 64
-device_align = 4096
-device_io_size = 16384
 flushers = 4
 reclaimers = 4
 recover_concurrency = 8
-lfu_window_to_cache_size_ratio = 1
-lfu_tiny_lru_capacity_ratio = 0.01
 insert_rate_limit_mb = 0
-catalog_bits = 6
+indexer_shards = 64
 compression = "none"
 
 [storage.meta_file_cache]
 dir = ""
 capacity_mb = 1024
 file_capacity_mb = 64
-device_align = 4096
-device_io_size = 16384
 flushers = 4
 reclaimers = 4
 recover_concurrency = 8
-lfu_window_to_cache_size_ratio = 1
-lfu_tiny_lru_capacity_ratio = 0.01
 insert_rate_limit_mb = 0
-catalog_bits = 6
+indexer_shards = 64
 compression = "none"
 
 [storage.cache_refill]
@@ -183,19 +190,33 @@ recent_filter_layers = 6
 recent_filter_rotate_interval_ms = 10000
 
 [storage.object_store]
-object_store_streaming_read_timeout_ms = 480000
-object_store_streaming_upload_timeout_ms = 480000
-object_store_upload_timeout_ms = 480000
-object_store_read_timeout_ms = 480000
 object_store_set_atomic_write_dir = false
 
+[storage.object_store.retry]
+req_backoff_interval_ms = 1000
+req_backoff_max_delay_ms = 10000
+req_backoff_factor = 2
+upload_attempt_timeout_ms = 8000
+upload_retry_attempts = 3
+streaming_upload_attempt_timeout_ms = 5000
+streaming_upload_retry_attempts = 3
+read_attempt_timeout_ms = 8000
+read_retry_attempts = 3
+streaming_read_attempt_timeout_ms = 3000
+streaming_read_retry_attempts = 3
+metadata_attempt_timeout_ms = 60000
+metadata_retry_attempts = 3
+delete_attempt_timeout_ms = 5000
+delete_retry_attempts = 3
+delete_objects_attempt_timeout_ms = 5000
+delete_objects_retry_attempts = 3
+list_attempt_timeout_ms = 600000
+list_retry_attempts = 3
+
 [storage.object_store.s3]
 object_store_keepalive_ms = 600000
 object_store_recv_buffer_size = 2097152
 object_store_nodelay = true
-object_store_req_retry_interval_ms = 20
-object_store_req_retry_max_delay_ms = 10000
-object_store_req_retry_max_attempts = 8
 retry_unknown_service_error = false
 identity_resolution_timeout_s = 5
 
diff --git a/src/connector/Cargo.toml b/src/connector/Cargo.toml
index 66c33335501d5..7cb6f23e5ec7e 100644
--- a/src/connector/Cargo.toml
+++ b/src/connector/Cargo.toml
@@ -26,12 +26,15 @@ arrow-row = { workspace = true }
 arrow-schema = { workspace = true }
 arrow-select = { workspace = true }
 assert_matches = "1"
-async-nats = "0.34"
+async-nats = "0.35"
 async-trait = "0.1"
 auto_enums = { workspace = true }
 auto_impl = "1"
+await-tree = { workspace = true }
 aws-config = { workspace = true }
 aws-credential-types = { workspace = true }
+aws-msk-iam-sasl-signer = "1.0.0"
+aws-sdk-dynamodb = "1"
 aws-sdk-kinesis = { workspace = true }
 aws-sdk-s3 = { workspace = true }
 aws-smithy-http = { workspace = true }
@@ -42,6 +45,7 @@ aws-types = { workspace = true }
 base64 = "0.22"
 byteorder = "1"
 bytes = { version = "1", features = ["serde"] }
+cfg-or-panic = "0.2"
 chrono = { version = "0.4", default-features = false, features = [
     "clock",
     "std",
@@ -51,19 +55,23 @@ clickhouse = { git = "https://github.com/risingwavelabs/clickhouse.rs", rev = "d
 ] }
 csv = "1.3"
 deltalake = { workspace = true }
-duration-str = "0.7.1"
+duration-str = "0.10.0"
 easy-ext = "1"
 enum-as-inner = "0.6"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 futures-async-stream = { workspace = true }
 gcp-bigquery-client = "0.18.0"
 glob = "0.3"
-google-cloud-pubsub = "0.24"
+google-cloud-bigquery = { version = "0.9.0", features = ["auth"] }
+google-cloud-gax = "0.17.0"
+google-cloud-googleapis = { version = "0.13", features = ["pubsub", "bigquery"] }
+google-cloud-pubsub = "0.25"
 http = "0.2"
 icelake = { workspace = true }
 indexmap = { version = "1.9.3", features = ["serde"] }
 itertools = { workspace = true }
 jni = { version = "0.21.1", features = ["invocation"] }
+jsonbb = { workspace = true }
 jsonwebtoken = "9.2.0"
 jst = { package = 'jsonschema-transpiler', git = "https://github.com/mozilla/jsonschema-transpiler", rev = "c1a89d720d118843d8bcca51084deb0ed223e4b4" }
 maplit = "1.0.2"
@@ -86,8 +94,8 @@ prometheus = { version = "0.13", features = ["process"] }
 prost = { version = "0.12", features = ["no-recursion-limit"] }
 prost-reflect = "0.13"
 prost-types = "0.12"
-protobuf-native = "0.2.1"
-pulsar = { version = "6.1", default-features = false, features = [
+protobuf-native = "0.2.2"
+pulsar = { version = "6.3", default-features = false, features = [
     "tokio-runtime",
     "telemetry",
     "auth-oauth2",
@@ -96,10 +104,7 @@ pulsar = { version = "6.1", default-features = false, features = [
 ] }
 rdkafka = { workspace = true, features = [
     "cmake-build",
-    # "ssl",
-    # FIXME: temporary workaround before we find an ideal solution.
-    # See why it's needed and why it's not ideal in https://github.com/risingwavelabs/risingwave/issues/9852
-    "ssl-vendored",
+    "ssl",
     "gssapi",
     "zstd",
 ] }
@@ -108,6 +113,7 @@ regex = "1.4"
 reqwest = { version = "0.12.2", features = ["json", "stream"] }
 risingwave_common = { workspace = true }
 risingwave_common_estimate_size = { workspace = true }
+risingwave_connector_codec = { workspace = true }
 risingwave_jni_core = { workspace = true }
 risingwave_object_store = { workspace = true }
 risingwave_pb = { workspace = true }
@@ -118,16 +124,19 @@ rustls-native-certs = "0.7"
 rustls-pemfile = "2"
 rustls-pki-types = "1"
 rw_futures_util = { workspace = true }
+sea-schema = { version = "0.14", features = ["default", "sqlx-postgres", "sqlx-mysql"] }
 serde = { version = "1", features = ["derive", "rc"] }
 serde_derive = "1"
 serde_json = "1"
 serde_with = { version = "3", features = ["json"] }
-simd-json = "0.13.3"
+simd-json = { version = "0.13.3", features = ["hints"] }
+sqlx = { workspace = true }
 strum = "0.26"
 strum_macros = "0.26"
 tempfile = "3"
 thiserror = "1"
 thiserror-ext = { workspace = true }
+tiberius = { version = "0.12", default-features = false, features = ["chrono", "time", "tds73", "rust_decimal", "bigdecimal", "rustls"] }
 time = "0.3.30"
 tokio = { version = "0.2", package = "madsim-tokio", features = [
     "rt",
@@ -175,7 +184,7 @@ prost-build = "0.12"
 protobuf-src = "1"
 
 [[bench]]
-name = "parser"
+name = "debezium_json_parser"
 harness = false
 
 [[bench]]
@@ -183,7 +192,11 @@ name = "nexmark_integration"
 harness = false
 
 [[bench]]
-name = "json_parser"
+name = "json_parser_case_insensitive"
+harness = false
+
+[[bench]]
+name = "json_vs_plain_parser"
 harness = false
 
 [lints]
diff --git a/src/connector/benches/json_parser.rs b/src/connector/benches/debezium_json_parser.rs
similarity index 51%
rename from src/connector/benches/json_parser.rs
rename to src/connector/benches/debezium_json_parser.rs
index 90c78044d6221..e448fa17ad1db 100644
--- a/src/connector/benches/json_parser.rs
+++ b/src/connector/benches/debezium_json_parser.rs
@@ -12,20 +12,28 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
+//! Benchmark for Debezium JSON records with `DebeziumParser`.
+
+mod json_common;
 
 use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
 use futures::executor::block_on;
+use json_common::*;
 use paste::paste;
-use rand::distributions::Alphanumeric;
-use rand::prelude::*;
-use risingwave_common::catalog::ColumnId;
-use risingwave_common::types::{DataType, Date, Timestamp};
-use risingwave_connector::parser::plain_parser::PlainParser;
-use risingwave_connector::parser::{
-    DebeziumParser, SourceStreamChunkBuilder, SpecificParserConfig,
-};
-use risingwave_connector::source::{SourceColumnDesc, SourceContext};
+use rand::Rng;
+use risingwave_connector::parser::{DebeziumParser, SourceStreamChunkBuilder};
+
+fn generate_debezium_json_row(rng: &mut impl Rng, change_event: &str) -> String {
+    let source = r#"{"version":"1.7.1.Final","connector":"mysql","name":"dbserver1","ts_ms":1639547113601,"snapshot":"true","db":"inventory","sequence":null,"table":"products","server_id":0,"gtid":null,"file":"mysql-bin.000003","pos":156,"row":0,"thread":null,"query":null}"#;
+    let (before, after) = match change_event {
+        "c" => ("null".to_string(), generate_json_row(rng)),
+        "r" => ("null".to_string(), generate_json_row(rng)),
+        "u" => (generate_json_row(rng), generate_json_row(rng)),
+        "d" => (generate_json_row(rng), "null".to_string()),
+        _ => unreachable!(),
+    };
+    format!("{{\"before\": {before}, \"after\": {after}, \"source\": {source}, \"op\": \"{change_event}\", \"ts_ms\":1639551564960, \"transaction\":null}}")
+}
 
 macro_rules! create_debezium_bench_helpers {
     ($op:ident, $op_sym:expr, $bench_function:expr) => {
@@ -68,100 +76,8 @@ create_debezium_bench_helpers!(read, "r", "bench_debezium_json_parser_read");
 create_debezium_bench_helpers!(update, "u", "bench_debezium_json_parser_update");
 create_debezium_bench_helpers!(delete, "d", "bench_debezium_json_parser_delete");
 
-const NUM_RECORDS: usize = 1 << 18; // ~ 250,000
-
-fn generate_json_row(rng: &mut impl Rng) -> String {
-    format!("{{\"i32\":{},\"bool\":{},\"i16\":{},\"i64\":{},\"f32\":{},\"f64\":{},\"varchar\":\"{}\",\"date\":\"{}\",\"timestamp\":\"{}\"}}",
-        rng.gen::<i32>(),
-        rng.gen::<bool>(),
-        rng.gen::<i16>(),
-        rng.gen::<i64>(),
-        rng.gen::<f32>(),
-        rng.gen::<f64>(),
-        rng.sample_iter(&Alphanumeric)
-            .take(7)
-            .map(char::from)
-            .collect::<String>(),
-        Date::from_num_days_from_ce_uncheck((rng.gen::<u32>() % (1 << 20)) as i32).0,
-        {
-            let datetime = Timestamp::from_timestamp_uncheck((rng.gen::<u32>() % (1u32 << 28)) as i64, 0).0;
-            format!("{:?} {:?}", datetime.date(), datetime.time())
-        }
-    )
-}
-
-fn generate_json_rows() -> Vec<Vec<u8>> {
-    let mut rng = rand::thread_rng();
-    let mut records = Vec::with_capacity(NUM_RECORDS);
-    for _ in 0..NUM_RECORDS {
-        records.push(generate_json_row(&mut rng).into_bytes());
-    }
-    records
-}
-
-fn generate_debezium_json_row(rng: &mut impl Rng, change_event: &str) -> String {
-    let source = r#"{"version":"1.7.1.Final","connector":"mysql","name":"dbserver1","ts_ms":1639547113601,"snapshot":"true","db":"inventory","sequence":null,"table":"products","server_id":0,"gtid":null,"file":"mysql-bin.000003","pos":156,"row":0,"thread":null,"query":null}"#;
-    let (before, after) = match change_event {
-        "c" => ("null".to_string(), generate_json_row(rng)),
-        "r" => ("null".to_string(), generate_json_row(rng)),
-        "u" => (generate_json_row(rng), generate_json_row(rng)),
-        "d" => (generate_json_row(rng), "null".to_string()),
-        _ => unreachable!(),
-    };
-    format!("{{\"before\": {before}, \"after\": {after}, \"source\": {source}, \"op\": \"{change_event}\", \"ts_ms\":1639551564960, \"transaction\":null}}")
-}
-
-fn get_descs() -> Vec<SourceColumnDesc> {
-    vec![
-        SourceColumnDesc::simple("i32", DataType::Int32, ColumnId::from(0)),
-        SourceColumnDesc::simple("bool", DataType::Boolean, ColumnId::from(2)),
-        SourceColumnDesc::simple("i16", DataType::Int16, ColumnId::from(3)),
-        SourceColumnDesc::simple("i64", DataType::Int64, ColumnId::from(4)),
-        SourceColumnDesc::simple("f32", DataType::Float32, ColumnId::from(5)),
-        SourceColumnDesc::simple("f64", DataType::Float64, ColumnId::from(6)),
-        SourceColumnDesc::simple("varchar", DataType::Varchar, ColumnId::from(7)),
-        SourceColumnDesc::simple("date", DataType::Date, ColumnId::from(8)),
-        SourceColumnDesc::simple("timestamp", DataType::Timestamp, ColumnId::from(9)),
-    ]
-}
-
-fn bench_json_parser(c: &mut Criterion) {
-    let descs = get_descs();
-    let rt = tokio::runtime::Builder::new_multi_thread()
-        .enable_all()
-        .build()
-        .unwrap();
-    let records = generate_json_rows();
-    let ctx = Arc::new(SourceContext::dummy());
-    c.bench_function("json_parser", |b| {
-        b.to_async(&rt).iter_batched(
-            || records.clone(),
-            |records| async {
-                let mut parser = rt
-                    .block_on(PlainParser::new(
-                        SpecificParserConfig::DEFAULT_PLAIN_JSON,
-                        descs.clone(),
-                        ctx.clone(),
-                    ))
-                    .unwrap();
-                let mut builder =
-                    SourceStreamChunkBuilder::with_capacity(descs.clone(), NUM_RECORDS);
-                for record in records {
-                    let writer = builder.row_writer();
-                    parser
-                        .parse_inner(None, Some(record), writer)
-                        .await
-                        .unwrap();
-                }
-            },
-            BatchSize::SmallInput,
-        )
-    });
-}
-
 criterion_group!(
     benches,
-    bench_json_parser,
     bench_debezium_json_parser_create,
     bench_debezium_json_parser_read,
     bench_debezium_json_parser_update,
diff --git a/src/connector/benches/json_common/mod.rs b/src/connector/benches/json_common/mod.rs
new file mode 100644
index 0000000000000..cb67c4cb3d547
--- /dev/null
+++ b/src/connector/benches/json_common/mod.rs
@@ -0,0 +1,57 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Common utilities shared by JSON parser benchmarks.
+
+use rand::distributions::Alphanumeric;
+use rand::prelude::*;
+use risingwave_common::catalog::ColumnId;
+use risingwave_common::types::{DataType, Date, Timestamp};
+use risingwave_connector::source::SourceColumnDesc;
+
+pub const NUM_RECORDS: usize = 1 << 18; // ~ 250,000
+
+pub fn generate_json_row(rng: &mut impl Rng) -> String {
+    format!("{{\"i32\":{},\"bool\":{},\"i16\":{},\"i64\":{},\"f32\":{},\"f64\":{},\"varchar\":\"{}\",\"date\":\"{}\",\"timestamp\":\"{}\"}}",
+        rng.gen::<i32>(),
+        rng.gen::<bool>(),
+        rng.gen::<i16>(),
+        rng.gen::<i64>(),
+        rng.gen::<f32>(),
+        rng.gen::<f64>(),
+        rng.sample_iter(&Alphanumeric)
+            .take(7)
+            .map(char::from)
+            .collect::<String>(),
+        Date::from_num_days_from_ce_uncheck((rng.gen::<u32>() % (1 << 20)) as i32).0,
+        {
+            let datetime = Timestamp::from_timestamp_uncheck((rng.gen::<u32>() % (1u32 << 28)) as i64, 0).0;
+            format!("{:?} {:?}", datetime.date(), datetime.time())
+        }
+    )
+}
+
+pub fn get_descs() -> Vec<SourceColumnDesc> {
+    vec![
+        SourceColumnDesc::simple("i32", DataType::Int32, ColumnId::from(0)),
+        SourceColumnDesc::simple("bool", DataType::Boolean, ColumnId::from(2)),
+        SourceColumnDesc::simple("i16", DataType::Int16, ColumnId::from(3)),
+        SourceColumnDesc::simple("i64", DataType::Int64, ColumnId::from(4)),
+        SourceColumnDesc::simple("f32", DataType::Float32, ColumnId::from(5)),
+        SourceColumnDesc::simple("f64", DataType::Float64, ColumnId::from(6)),
+        SourceColumnDesc::simple("varchar", DataType::Varchar, ColumnId::from(7)),
+        SourceColumnDesc::simple("date", DataType::Date, ColumnId::from(8)),
+        SourceColumnDesc::simple("timestamp", DataType::Timestamp, ColumnId::from(9)),
+    ]
+}
diff --git a/src/connector/benches/parser.rs b/src/connector/benches/json_parser_case_insensitive.rs
similarity index 73%
rename from src/connector/benches/parser.rs
rename to src/connector/benches/json_parser_case_insensitive.rs
index 21ce72dd1b2b1..17fd439e6ccc1 100644
--- a/src/connector/benches/parser.rs
+++ b/src/connector/benches/json_parser_case_insensitive.rs
@@ -12,24 +12,29 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//! Benchmarking JSON parsers for scenarios with exact key matches and case-insensitive key matches.
+
 use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion};
+use futures::StreamExt;
 use maplit::hashmap;
 use rand::Rng;
 use risingwave_common::types::DataType;
 use risingwave_connector::parser::{
-    EncodingProperties, JsonParser, JsonProperties, ProtocolProperties, SourceStreamChunkBuilder,
-    SpecificParserConfig,
+    ByteStreamSourceParserImpl, CommonParserConfig, ParserConfig, SpecificParserConfig,
 };
-use risingwave_connector::source::{SourceColumnDesc, SourceContext};
+use risingwave_connector::source::{SourceColumnDesc, SourceMessage};
 use serde_json::json;
 use tokio::runtime::Runtime;
 
-fn gen_input(mode: &str, chunk_size: usize, chunk_num: usize) -> Vec<Vec<Vec<u8>>> {
+type Input = Vec<Vec<SourceMessage>>;
+type Parser = ByteStreamSourceParserImpl;
+
+fn gen_input(mode: &str, chunk_size: usize, chunk_num: usize) -> Input {
     let mut input = Vec::with_capacity(chunk_num);
     for _ in 0..chunk_num {
         let mut input_inner = Vec::with_capacity(chunk_size);
         for _ in 0..chunk_size {
-            input_inner.push(match mode {
+            let payload = match mode {
                 "match" => r#"{"alpha": 1, "bravo": 2, "charlie": 3, "delta": 4}"#
                     .as_bytes()
                     .to_vec(),
@@ -55,6 +60,10 @@ fn gen_input(mode: &str, chunk_size: usize, chunk_num: usize) -> Vec<Vec<Vec<u8>
                     serde_json::to_string(&value).unwrap().as_bytes().to_vec()
                 }
                 _ => unreachable!(),
+            };
+            input_inner.push(SourceMessage {
+                payload: Some(payload),
+                ..SourceMessage::dummy()
             });
         }
         input.push(input_inner);
@@ -62,40 +71,27 @@ fn gen_input(mode: &str, chunk_size: usize, chunk_num: usize) -> Vec<Vec<Vec<u8>
     input
 }
 
-fn create_parser(
-    chunk_size: usize,
-    chunk_num: usize,
-    mode: &str,
-) -> (JsonParser, Vec<SourceColumnDesc>, Vec<Vec<Vec<u8>>>) {
+fn create_parser(chunk_size: usize, chunk_num: usize, mode: &str) -> (Parser, Input) {
     let desc = vec![
         SourceColumnDesc::simple("alpha", DataType::Int16, 0.into()),
         SourceColumnDesc::simple("bravo", DataType::Int32, 1.into()),
         SourceColumnDesc::simple("charlie", DataType::Int64, 2.into()),
         SourceColumnDesc::simple("delta", DataType::Int64, 3.into()),
     ];
-    let props = SpecificParserConfig {
-        key_encoding_config: None,
-        encoding_config: EncodingProperties::Json(JsonProperties {
-            use_schema_registry: false,
-            timestamptz_handling: None,
-        }),
-        protocol_config: ProtocolProperties::Plain,
+    let config = ParserConfig {
+        common: CommonParserConfig { rw_columns: desc },
+        specific: SpecificParserConfig::DEFAULT_PLAIN_JSON,
     };
-    let parser = JsonParser::new(props, desc.clone(), SourceContext::dummy().into()).unwrap();
+    let parser = ByteStreamSourceParserImpl::create_for_test(config).unwrap();
     let input = gen_input(mode, chunk_size, chunk_num);
-    (parser, desc, input)
+    (parser, input)
 }
 
-async fn parse(parser: JsonParser, column_desc: Vec<SourceColumnDesc>, input: Vec<Vec<Vec<u8>>>) {
-    for input_inner in input {
-        let mut builder =
-            SourceStreamChunkBuilder::with_capacity(column_desc.clone(), input_inner.len());
-        for payload in input_inner {
-            let row_writer = builder.row_writer();
-            parser.parse_inner(payload, row_writer).await.unwrap();
-        }
-        builder.finish();
-    }
+async fn parse(parser: Parser, input: Input) {
+    parser
+        .into_stream(futures::stream::iter(input.into_iter().map(Ok)).boxed())
+        .count() // consume the stream
+        .await;
 }
 
 fn do_bench(c: &mut Criterion, mode: &str) {
@@ -110,7 +106,7 @@ fn do_bench(c: &mut Criterion, mode: &str) {
                 let chunk_num = TOTAL_SIZE / chunk_size;
                 b.to_async(&rt).iter_batched(
                     || create_parser(chunk_size, chunk_num, mode),
-                    |(parser, column_desc, input)| parse(parser, column_desc, input),
+                    |(parser, input)| parse(parser, input),
                     BatchSize::SmallInput,
                 );
             },
diff --git a/src/connector/benches/json_vs_plain_parser.rs b/src/connector/benches/json_vs_plain_parser.rs
new file mode 100644
index 0000000000000..b9a1139dcb03b
--- /dev/null
+++ b/src/connector/benches/json_vs_plain_parser.rs
@@ -0,0 +1,172 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Benchmark for comparing the performance of parsing JSON records directly
+//! through the `JsonParser` versus indirectly through the `PlainParser`.
+
+mod json_common;
+
+use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
+use futures::executor::block_on;
+use json_common::*;
+use old_json_parser::JsonParser;
+use risingwave_connector::parser::plain_parser::PlainParser;
+use risingwave_connector::parser::{SourceStreamChunkBuilder, SpecificParserConfig};
+use risingwave_connector::source::SourceContext;
+
+// The original implementation used to parse JSON prior to #13707.
+mod old_json_parser {
+    use anyhow::Context as _;
+    use itertools::{Either, Itertools as _};
+    use risingwave_common::{bail, try_match_expand};
+    use risingwave_connector::error::ConnectorResult;
+    use risingwave_connector::parser::{
+        Access as _, EncodingProperties, JsonAccess, SourceStreamChunkRowWriter,
+    };
+    use risingwave_connector::source::{SourceColumnDesc, SourceContextRef};
+
+    use super::*;
+
+    /// Parser for JSON format
+    #[derive(Debug)]
+    pub struct JsonParser {
+        _rw_columns: Vec<SourceColumnDesc>,
+        _source_ctx: SourceContextRef,
+        // If schema registry is used, the starting index of payload is 5.
+        payload_start_idx: usize,
+    }
+
+    impl JsonParser {
+        pub fn new(
+            props: SpecificParserConfig,
+            rw_columns: Vec<SourceColumnDesc>,
+            source_ctx: SourceContextRef,
+        ) -> ConnectorResult<Self> {
+            let json_config = try_match_expand!(props.encoding_config, EncodingProperties::Json)?;
+            let payload_start_idx = if json_config.use_schema_registry {
+                5
+            } else {
+                0
+            };
+            Ok(Self {
+                _rw_columns: rw_columns,
+                _source_ctx: source_ctx,
+                payload_start_idx,
+            })
+        }
+
+        #[allow(clippy::unused_async)]
+        pub async fn parse_inner(
+            &self,
+            mut payload: Vec<u8>,
+            mut writer: SourceStreamChunkRowWriter<'_>,
+        ) -> ConnectorResult<()> {
+            let value = simd_json::to_borrowed_value(&mut payload[self.payload_start_idx..])
+                .context("failed to parse json payload")?;
+            let values = if let simd_json::BorrowedValue::Array(arr) = value {
+                Either::Left(arr.into_iter())
+            } else {
+                Either::Right(std::iter::once(value))
+            };
+
+            let mut errors = Vec::new();
+            for value in values {
+                let accessor = JsonAccess::new(value);
+                match writer.do_insert(|column| accessor.access(&[&column.name], &column.data_type))
+                {
+                    Ok(_) => {}
+                    Err(err) => errors.push(err),
+                }
+            }
+
+            if errors.is_empty() {
+                Ok(())
+            } else {
+                bail!(
+                    "failed to parse {} row(s) in a single json message: {}",
+                    errors.len(),
+                    errors.iter().format(", ")
+                );
+            }
+        }
+    }
+}
+
+fn generate_json_rows() -> Vec<Vec<u8>> {
+    let mut rng = rand::thread_rng();
+    let mut records = Vec::with_capacity(NUM_RECORDS);
+    for _ in 0..NUM_RECORDS {
+        records.push(generate_json_row(&mut rng).into_bytes());
+    }
+    records
+}
+
+fn bench_plain_parser_and_json_parser(c: &mut Criterion) {
+    let rt = tokio::runtime::Runtime::new().unwrap();
+    let records = generate_json_rows();
+
+    let mut group = c.benchmark_group("plain parser and json parser comparison");
+
+    group.bench_function("plain_parser", |b| {
+        b.to_async(&rt).iter_batched(
+            || {
+                let parser = block_on(PlainParser::new(
+                    SpecificParserConfig::DEFAULT_PLAIN_JSON,
+                    get_descs(),
+                    SourceContext::dummy().into(),
+                ))
+                .unwrap();
+                (parser, records.clone())
+            },
+            |(mut parser, records)| async move {
+                let mut builder = SourceStreamChunkBuilder::with_capacity(get_descs(), NUM_RECORDS);
+                for record in records {
+                    let writer = builder.row_writer();
+                    parser
+                        .parse_inner(None, Some(record), writer)
+                        .await
+                        .unwrap();
+                }
+            },
+            BatchSize::SmallInput,
+        )
+    });
+
+    group.bench_function("json_parser", |b| {
+        b.to_async(&rt).iter_batched(
+            || {
+                let parser = JsonParser::new(
+                    SpecificParserConfig::DEFAULT_PLAIN_JSON,
+                    get_descs(),
+                    SourceContext::dummy().into(),
+                )
+                .unwrap();
+                (parser, records.clone())
+            },
+            |(parser, records)| async move {
+                let mut builder = SourceStreamChunkBuilder::with_capacity(get_descs(), NUM_RECORDS);
+                for record in records {
+                    let writer = builder.row_writer();
+                    parser.parse_inner(record, writer).await.unwrap();
+                }
+            },
+            BatchSize::SmallInput,
+        )
+    });
+
+    group.finish();
+}
+
+criterion_group!(benches, bench_plain_parser_and_json_parser,);
+criterion_main!(benches);
diff --git a/src/connector/benches/nexmark_integration.rs b/src/connector/benches/nexmark_integration.rs
index 1c05147eeafbb..172931562efef 100644
--- a/src/connector/benches/nexmark_integration.rs
+++ b/src/connector/benches/nexmark_integration.rs
@@ -12,6 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//! Integration benchmark for parsing Nexmark events.
+//!
+//! To cover the code path in real-world scenarios, the parser is created through
+//! `ByteStreamSourceParserImpl::create` based on the given configuration, rather
+//! than depending on a specific internal implementation.
+
 #![feature(lazy_cell)]
 
 use std::sync::LazyLock;
@@ -20,27 +26,23 @@ use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
 use futures::{FutureExt, StreamExt, TryStreamExt};
 use itertools::Itertools;
 use risingwave_common::array::StreamChunk;
-use risingwave_common::catalog::ColumnId;
+use risingwave_common::catalog::{ColumnDesc, ColumnId};
 use risingwave_common::types::DataType;
 use risingwave_connector::parser::{
-    ByteStreamSourceParser, JsonParser, SourceParserIntoStreamExt, SpecificParserConfig,
+    ByteStreamSourceParserImpl, CommonParserConfig, ParserConfig, SpecificParserConfig,
 };
 use risingwave_connector::source::{
-    BoxChunkSourceStream, BoxSourceStream, SourceColumnDesc, SourceContext, SourceMessage,
-    SourceMeta,
+    BoxChunkSourceStream, BoxSourceStream, SourceColumnDesc, SourceMessage, SourceMeta,
 };
 use tracing::Level;
 use tracing_subscriber::prelude::*;
 
-static BATCH: LazyLock<Vec<SourceMessage>> = LazyLock::new(make_batch);
+static BATCH: LazyLock<Vec<SourceMessage>> = LazyLock::new(|| make_batch(false));
+static STRUCT_BATCH: LazyLock<Vec<SourceMessage>> = LazyLock::new(|| make_batch(true));
 
-fn make_batch() -> Vec<SourceMessage> {
+fn make_batch(use_struct: bool) -> Vec<SourceMessage> {
     let mut generator = nexmark::EventGenerator::default()
         .with_type_filter(nexmark::event::EventType::Bid)
-        .map(|e| match e {
-            nexmark::event::Event::Bid(bid) => bid, // extract the bid event
-            _ => unreachable!(),
-        })
         .enumerate();
 
     let message_base = SourceMessage {
@@ -53,9 +55,16 @@ fn make_batch() -> Vec<SourceMessage> {
 
     generator
         .by_ref()
-        .take(1024)
-        .map(|(i, e)| {
-            let payload = serde_json::to_vec(&e).unwrap();
+        .take(16384)
+        .map(|(i, event)| {
+            let payload = if use_struct {
+                serde_json::to_vec(&event).unwrap()
+            } else {
+                let nexmark::event::Event::Bid(bid) = event else {
+                    unreachable!()
+                };
+                serde_json::to_vec(&bid).unwrap()
+            };
             SourceMessage {
                 payload: Some(payload),
                 offset: i.to_string(),
@@ -65,14 +74,18 @@ fn make_batch() -> Vec<SourceMessage> {
         .collect_vec()
 }
 
-fn make_data_stream() -> BoxSourceStream {
-    futures::future::ready(Ok(BATCH.clone()))
-        .into_stream()
-        .boxed()
+fn make_data_stream(use_struct: bool) -> BoxSourceStream {
+    futures::future::ready(Ok(if use_struct {
+        STRUCT_BATCH.clone()
+    } else {
+        BATCH.clone()
+    }))
+    .into_stream()
+    .boxed()
 }
 
-fn make_parser() -> impl ByteStreamSourceParser {
-    let columns = [
+fn make_parser(use_struct: bool) -> ByteStreamSourceParserImpl {
+    let fields = vec![
         ("auction", DataType::Int64),
         ("bidder", DataType::Int64),
         ("price", DataType::Int64),
@@ -80,19 +93,36 @@ fn make_parser() -> impl ByteStreamSourceParser {
         ("url", DataType::Varchar),
         ("date_time", DataType::Timestamp),
         ("extra", DataType::Varchar),
-    ]
-    .into_iter()
-    .enumerate()
-    .map(|(i, (n, t))| SourceColumnDesc::simple(n, t, ColumnId::new(i as _)))
-    .collect_vec();
+    ];
+
+    let rw_columns = if use_struct {
+        let fields = fields
+            .into_iter()
+            .enumerate()
+            .map(|(i, (n, t))| ColumnDesc::named(n, ColumnId::new(i as _), t))
+            .collect();
+        let struct_col = ColumnDesc::new_struct("bid", 114514, "bid", fields);
+        vec![(&struct_col).into()]
+    } else {
+        fields
+            .into_iter()
+            .enumerate()
+            .map(|(i, (n, t))| SourceColumnDesc::simple(n, t, ColumnId::new(i as _)))
+            .collect_vec()
+    };
 
-    let props = SpecificParserConfig::DEFAULT_PLAIN_JSON;
+    let config = ParserConfig {
+        common: CommonParserConfig { rw_columns },
+        specific: SpecificParserConfig::DEFAULT_PLAIN_JSON,
+    };
 
-    JsonParser::new(props, columns, SourceContext::dummy().into()).unwrap()
+    ByteStreamSourceParserImpl::create_for_test(config).unwrap()
 }
 
-fn make_stream_iter() -> impl Iterator<Item = StreamChunk> {
-    let mut stream: BoxChunkSourceStream = make_parser().into_stream(make_data_stream()).boxed();
+fn make_stream_iter(use_struct: bool) -> impl Iterator<Item = StreamChunk> {
+    let mut stream: BoxChunkSourceStream = make_parser(use_struct)
+        .into_stream(make_data_stream(use_struct))
+        .boxed();
 
     std::iter::from_fn(move || {
         stream
@@ -108,7 +138,7 @@ fn make_stream_iter() -> impl Iterator<Item = StreamChunk> {
 fn bench(c: &mut Criterion) {
     c.bench_function("parse_nexmark", |b| {
         b.iter_batched(
-            make_stream_iter,
+            || make_stream_iter(false),
             |mut iter| iter.next().unwrap(),
             BatchSize::SmallInput,
         )
@@ -127,11 +157,19 @@ fn bench(c: &mut Criterion) {
             .into();
 
         b.iter_batched(
-            make_stream_iter,
+            || make_stream_iter(false),
             |mut iter| tracing::dispatcher::with_default(&dispatch, || iter.next().unwrap()),
             BatchSize::SmallInput,
         )
     });
+
+    c.bench_function("parse_nexmark_struct_type", |b| {
+        b.iter_batched(
+            || make_stream_iter(true),
+            |mut iter| iter.next().unwrap(),
+            BatchSize::SmallInput,
+        )
+    });
 }
 
 criterion_group!(benches, bench);
diff --git a/src/connector/codec/Cargo.toml b/src/connector/codec/Cargo.toml
new file mode 100644
index 0000000000000..ef12b325d446d
--- /dev/null
+++ b/src/connector/codec/Cargo.toml
@@ -0,0 +1,45 @@
+[package]
+name = "risingwave_connector_codec"
+description = "Encoding and decoding between external data formats and RisingWave datum"
+version = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+keywords = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+
+[package.metadata.cargo-machete]
+ignored = ["workspace-hack"]
+
+[package.metadata.cargo-udeps.ignore]
+normal = ["workspace-hack"]
+
+[dependencies]
+anyhow = "1"
+apache-avro = { git = "https://github.com/risingwavelabs/avro", rev = "5349b0c7b35940d117397edbd314ca9087cdb892", features = [
+    "snappy",
+    "zstandard",
+    "bzip",
+    "xz",
+] }
+chrono = { version = "0.4", default-features = false, features = [
+    "clock",
+    "std",
+] }
+easy-ext = "1"
+itertools = { workspace = true }
+jsonbb = { workspace = true }
+num-bigint = "0.4"
+risingwave_common = { workspace = true }
+risingwave_pb = { workspace = true }
+rust_decimal = "1"
+thiserror = "1"
+thiserror-ext = { workspace = true }
+time = "0.3.30"
+tracing = "0.1"
+
+[target.'cfg(not(madsim))'.dependencies]
+workspace-hack = { path = "../../workspace-hack" }
+
+[lints]
+workspace = true
diff --git a/src/connector/codec/src/decoder/avro/mod.rs b/src/connector/codec/src/decoder/avro/mod.rs
new file mode 100644
index 0000000000000..cdd9aea416c8f
--- /dev/null
+++ b/src/connector/codec/src/decoder/avro/mod.rs
@@ -0,0 +1,621 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod schema;
+use std::sync::LazyLock;
+
+use apache_avro::schema::{DecimalSchema, RecordSchema};
+use apache_avro::types::{Value, ValueKind};
+use apache_avro::{Decimal as AvroDecimal, Schema};
+use chrono::Datelike;
+use itertools::Itertools;
+use num_bigint::{BigInt, Sign};
+use risingwave_common::array::{ListValue, StructValue};
+use risingwave_common::bail;
+use risingwave_common::log::LogSuppresser;
+use risingwave_common::types::{
+    DataType, Date, DatumCow, Interval, JsonbVal, ScalarImpl, Time, Timestamp, Timestamptz,
+    ToOwnedDatum,
+};
+use risingwave_common::util::iter_util::ZipEqFast;
+
+pub use self::schema::{avro_schema_to_column_descs, MapHandling, ResolvedAvroSchema};
+use super::utils::extract_decimal;
+use super::{bail_uncategorized, uncategorized, Access, AccessError, AccessResult};
+
+#[derive(Clone)]
+/// Options for parsing an `AvroValue` into Datum, with an optional avro schema.
+pub struct AvroParseOptions<'a> {
+    /// Currently, this schema is only used for decimal.
+    ///
+    /// FIXME: In theory we should use resolved schema.
+    /// e.g., it's possible that a field is a reference to a decimal or a record containing a decimal field.
+    pub schema: Option<&'a Schema>,
+    /// Strict Mode
+    /// If strict mode is disabled, an int64 can be parsed from an `AvroInt` (int32) value.
+    pub relax_numeric: bool,
+}
+
+impl<'a> AvroParseOptions<'a> {
+    pub fn create(schema: &'a Schema) -> Self {
+        Self {
+            schema: Some(schema),
+            relax_numeric: true,
+        }
+    }
+}
+
+impl<'a> AvroParseOptions<'a> {
+    fn extract_inner_schema(&self, key: Option<&str>) -> Option<&'a Schema> {
+        self.schema
+            .map(|schema| avro_extract_field_schema(schema, key))
+            .transpose()
+            .map_err(|_err| {
+                static LOG_SUPPERSSER: LazyLock<LogSuppresser> =
+                    LazyLock::new(LogSuppresser::default);
+                if let Ok(suppressed_count) = LOG_SUPPERSSER.check() {
+                    tracing::error!(suppressed_count, "extract sub-schema");
+                }
+            })
+            .ok()
+            .flatten()
+    }
+
+    /// Parse an avro value into expected type.
+    ///
+    /// 3 kinds of type info are used to parsing:
+    /// - `type_expected`. The type that we expect the value is.
+    /// - value type. The type info together with the value argument.
+    /// - schema. The `AvroSchema` provided in option.
+    ///
+    /// Cases: (FIXME: Is this precise?)
+    /// - If both `type_expected` and schema are provided, it will check both strictly.
+    /// - If only `type_expected` is provided, it will try to match the value type and the
+    ///    `type_expected`, converting the value if possible.
+    /// - If only value is provided (without schema and `type_expected`),
+    ///     the `DataType` will be inferred.
+    pub fn convert_to_datum<'b>(
+        &self,
+        value: &'b Value,
+        type_expected: &DataType,
+    ) -> AccessResult<DatumCow<'b>>
+    where
+        'b: 'a,
+    {
+        let create_error = || AccessError::TypeError {
+            expected: format!("{:?}", type_expected),
+            got: format!("{:?}", value),
+            value: String::new(),
+        };
+
+        macro_rules! borrowed {
+            ($v:expr) => {
+                return Ok(DatumCow::Borrowed(Some($v.into())))
+            };
+        }
+
+        let v: ScalarImpl = match (type_expected, value) {
+            (_, Value::Null) => return Ok(DatumCow::NULL),
+            (_, Value::Union(_, v)) => {
+                let schema = self.extract_inner_schema(None);
+                return Self {
+                    schema,
+                    relax_numeric: self.relax_numeric,
+                }
+                .convert_to_datum(v, type_expected);
+            }
+            // ---- Boolean -----
+            (DataType::Boolean, Value::Boolean(b)) => (*b).into(),
+            // ---- Int16 -----
+            (DataType::Int16, Value::Int(i)) if self.relax_numeric => (*i as i16).into(),
+            (DataType::Int16, Value::Long(i)) if self.relax_numeric => (*i as i16).into(),
+
+            // ---- Int32 -----
+            (DataType::Int32, Value::Int(i)) => (*i).into(),
+            (DataType::Int32, Value::Long(i)) if self.relax_numeric => (*i as i32).into(),
+            // ---- Int64 -----
+            (DataType::Int64, Value::Long(i)) => (*i).into(),
+            (DataType::Int64, Value::Int(i)) if self.relax_numeric => (*i as i64).into(),
+            // ---- Float32 -----
+            (DataType::Float32, Value::Float(i)) => (*i).into(),
+            (DataType::Float32, Value::Double(i)) => (*i as f32).into(),
+            // ---- Float64 -----
+            (DataType::Float64, Value::Double(i)) => (*i).into(),
+            (DataType::Float64, Value::Float(i)) => (*i as f64).into(),
+            // ---- Decimal -----
+            (DataType::Decimal, Value::Decimal(avro_decimal)) => {
+                let (precision, scale) = match self.schema {
+                    Some(Schema::Decimal(DecimalSchema {
+                        precision, scale, ..
+                    })) => (*precision, *scale),
+                    _ => Err(create_error())?,
+                };
+                let decimal = avro_decimal_to_rust_decimal(avro_decimal.clone(), precision, scale)
+                    .map_err(|_| create_error())?;
+                ScalarImpl::Decimal(risingwave_common::types::Decimal::Normalized(decimal))
+            }
+            (DataType::Decimal, Value::Record(fields)) => {
+                // VariableScaleDecimal has fixed fields, scale(int) and value(bytes)
+                let find_in_records = |field_name: &str| {
+                    fields
+                        .iter()
+                        .find(|field| field.0 == field_name)
+                        .map(|field| &field.1)
+                        .ok_or_else(|| {
+                            uncategorized!("`{field_name}` field not found in VariableScaleDecimal")
+                        })
+                };
+                let scale = match find_in_records("scale")? {
+                    Value::Int(scale) => *scale,
+                    avro_value => bail_uncategorized!(
+                        "scale field in VariableScaleDecimal is not int, got {:?}",
+                        avro_value
+                    ),
+                };
+
+                let value: BigInt = match find_in_records("value")? {
+                    Value::Bytes(bytes) => BigInt::from_signed_bytes_be(bytes),
+                    avro_value => bail_uncategorized!(
+                        "value field in VariableScaleDecimal is not bytes, got {:?}",
+                        avro_value
+                    ),
+                };
+
+                let negative = value.sign() == Sign::Minus;
+                let (lo, mid, hi) = extract_decimal(value.to_bytes_be().1)?;
+                let decimal =
+                    rust_decimal::Decimal::from_parts(lo, mid, hi, negative, scale as u32);
+                ScalarImpl::Decimal(risingwave_common::types::Decimal::Normalized(decimal))
+            }
+            // ---- Time -----
+            (DataType::Time, Value::TimeMillis(ms)) => Time::with_milli(*ms as u32)
+                .map_err(|_| create_error())?
+                .into(),
+            (DataType::Time, Value::TimeMicros(us)) => Time::with_micro(*us as u64)
+                .map_err(|_| create_error())?
+                .into(),
+            // ---- Date -----
+            (DataType::Date, Value::Date(days)) => Date::with_days(days + unix_epoch_days())
+                .map_err(|_| create_error())?
+                .into(),
+            // ---- Varchar -----
+            (DataType::Varchar, Value::Enum(_, symbol)) => borrowed!(symbol.as_str()),
+            (DataType::Varchar, Value::String(s)) => borrowed!(s.as_str()),
+            // ---- Timestamp -----
+            (DataType::Timestamp, Value::LocalTimestampMillis(ms)) => Timestamp::with_millis(*ms)
+                .map_err(|_| create_error())?
+                .into(),
+            (DataType::Timestamp, Value::LocalTimestampMicros(us)) => Timestamp::with_micros(*us)
+                .map_err(|_| create_error())?
+                .into(),
+
+            // ---- TimestampTz -----
+            (DataType::Timestamptz, Value::TimestampMillis(ms)) => Timestamptz::from_millis(*ms)
+                .ok_or_else(|| {
+                    uncategorized!("timestamptz with milliseconds {ms} * 1000 is out of range")
+                })?
+                .into(),
+            (DataType::Timestamptz, Value::TimestampMicros(us)) => {
+                Timestamptz::from_micros(*us).into()
+            }
+
+            // ---- Interval -----
+            (DataType::Interval, Value::Duration(duration)) => {
+                let months = u32::from(duration.months()) as i32;
+                let days = u32::from(duration.days()) as i32;
+                let usecs = (u32::from(duration.millis()) as i64) * 1000; // never overflows
+                ScalarImpl::Interval(Interval::from_month_day_usec(months, days, usecs))
+            }
+            // ---- Struct -----
+            (DataType::Struct(struct_type_info), Value::Record(descs)) => StructValue::new(
+                struct_type_info
+                    .names()
+                    .zip_eq_fast(struct_type_info.types())
+                    .map(|(field_name, field_type)| {
+                        let maybe_value = descs.iter().find(|(k, _v)| k == field_name);
+                        if let Some((_, value)) = maybe_value {
+                            let schema = self.extract_inner_schema(Some(field_name));
+                            Ok(Self {
+                                schema,
+                                relax_numeric: self.relax_numeric,
+                            }
+                            .convert_to_datum(value, field_type)?
+                            .to_owned_datum())
+                        } else {
+                            Ok(None)
+                        }
+                    })
+                    .collect::<Result<_, AccessError>>()?,
+            )
+            .into(),
+            // ---- List -----
+            (DataType::List(item_type), Value::Array(array)) => ListValue::new({
+                let schema = self.extract_inner_schema(None);
+                let mut builder = item_type.create_array_builder(array.len());
+                for v in array {
+                    let value = Self {
+                        schema,
+                        relax_numeric: self.relax_numeric,
+                    }
+                    .convert_to_datum(v, item_type)?;
+                    builder.append(value);
+                }
+                builder.finish()
+            })
+            .into(),
+            // ---- Bytea -----
+            (DataType::Bytea, Value::Bytes(value)) => borrowed!(value.as_slice()),
+            // ---- Jsonb -----
+            (DataType::Jsonb, v @ Value::Map(_)) => {
+                let mut builder = jsonbb::Builder::default();
+                avro_to_jsonb(v, &mut builder)?;
+                let jsonb = builder.finish();
+                debug_assert!(jsonb.as_ref().is_object());
+                JsonbVal::from(jsonb).into()
+            }
+            (DataType::Varchar, Value::Uuid(uuid)) => {
+                uuid.as_hyphenated().to_string().into_boxed_str().into()
+            }
+
+            (_expected, _got) => Err(create_error())?,
+        };
+        Ok(DatumCow::Owned(Some(v)))
+    }
+}
+
+pub struct AvroAccess<'a> {
+    value: &'a Value,
+    options: AvroParseOptions<'a>,
+}
+
+impl<'a> AvroAccess<'a> {
+    pub fn new(value: &'a Value, options: AvroParseOptions<'a>) -> Self {
+        Self { value, options }
+    }
+}
+
+impl Access for AvroAccess<'_> {
+    fn access<'a>(&'a self, path: &[&str], type_expected: &DataType) -> AccessResult<DatumCow<'a>> {
+        let mut value = self.value;
+        let mut options: AvroParseOptions<'_> = self.options.clone();
+
+        let mut i = 0;
+        while i < path.len() {
+            let key = path[i];
+            let create_error = || AccessError::Undefined {
+                name: key.to_string(),
+                path: path.iter().take(i).join("."),
+            };
+            match value {
+                Value::Union(_, v) => {
+                    value = v;
+                    options.schema = options.extract_inner_schema(None);
+                    continue;
+                }
+                Value::Record(fields) => {
+                    if let Some((_, v)) = fields.iter().find(|(k, _)| k == key) {
+                        value = v;
+                        options.schema = options.extract_inner_schema(Some(key));
+                        i += 1;
+                        continue;
+                    }
+                }
+                _ => (),
+            }
+            Err(create_error())?;
+        }
+
+        options.convert_to_datum(value, type_expected)
+    }
+}
+
+pub(crate) fn avro_decimal_to_rust_decimal(
+    avro_decimal: AvroDecimal,
+    _precision: usize,
+    scale: usize,
+) -> AccessResult<rust_decimal::Decimal> {
+    let negative = !avro_decimal.is_positive();
+    let bytes = avro_decimal.to_vec_unsigned();
+
+    let (lo, mid, hi) = extract_decimal(bytes)?;
+    Ok(rust_decimal::Decimal::from_parts(
+        lo,
+        mid,
+        hi,
+        negative,
+        scale as u32,
+    ))
+}
+
+pub fn avro_schema_skip_union(schema: &Schema) -> anyhow::Result<&Schema> {
+    match schema {
+        Schema::Union(union_schema) => {
+            let inner_schema = union_schema
+                .variants()
+                .iter()
+                .find(|s| !matches!(s, &&Schema::Null))
+                .ok_or_else(|| {
+                    anyhow::format_err!("illegal avro record schema {:?}", union_schema)
+                })?;
+            Ok(inner_schema)
+        }
+        other => Ok(other),
+    }
+}
+
+// extract inner filed/item schema of record/array/union
+pub fn avro_extract_field_schema<'a>(
+    schema: &'a Schema,
+    name: Option<&str>,
+) -> anyhow::Result<&'a Schema> {
+    match schema {
+        Schema::Record(RecordSchema { fields, lookup, .. }) => {
+            let name =
+                name.ok_or_else(|| anyhow::format_err!("no name provided for a field in record"))?;
+            let index = lookup.get(name).ok_or_else(|| {
+                anyhow::format_err!("no field named '{}' in record: {:?}", name, schema)
+            })?;
+            let field = fields
+                .get(*index)
+                .ok_or_else(|| anyhow::format_err!("illegal avro record schema {:?}", schema))?;
+            Ok(&field.schema)
+        }
+        Schema::Array(schema) => Ok(schema),
+        Schema::Union(_) => avro_schema_skip_union(schema),
+        Schema::Map(schema) => Ok(schema),
+        _ => bail!("avro schema does not have inner item, schema: {:?}", schema),
+    }
+}
+
+pub(crate) fn unix_epoch_days() -> i32 {
+    Date::from_ymd_uncheck(1970, 1, 1).0.num_days_from_ce()
+}
+
+pub(crate) fn avro_to_jsonb(avro: &Value, builder: &mut jsonbb::Builder) -> AccessResult<()> {
+    match avro {
+        Value::Null => builder.add_null(),
+        Value::Boolean(b) => builder.add_bool(*b),
+        Value::Int(i) => builder.add_i64(*i as i64),
+        Value::String(s) => builder.add_string(s),
+        Value::Map(m) => {
+            builder.begin_object();
+            for (k, v) in m {
+                builder.add_string(k);
+                avro_to_jsonb(v, builder)?;
+            }
+            builder.end_object()
+        }
+        // same representation as map
+        Value::Record(r) => {
+            builder.begin_object();
+            for (k, v) in r {
+                builder.add_string(k);
+                avro_to_jsonb(v, builder)?;
+            }
+            builder.end_object()
+        }
+        Value::Array(a) => {
+            builder.begin_array();
+            for v in a {
+                avro_to_jsonb(v, builder)?;
+            }
+            builder.end_array()
+        }
+
+        // TODO: figure out where the following encoding is reasonable before enabling them.
+        // See discussions: https://github.com/risingwavelabs/risingwave/pull/16948
+
+        // jsonbb supports int64, but JSON spec does not allow it. How should we handle it?
+        // BTW, protobuf canonical JSON converts int64 to string.
+        // Value::Long(l) => builder.add_i64(*l),
+        // Value::Float(f) => {
+        //     if f.is_nan() || f.is_infinite() {
+        //         // XXX: pad null or return err here?
+        //         builder.add_null()
+        //     } else {
+        //         builder.add_f64(*f as f64)
+        //     }
+        // }
+        // Value::Double(f) => {
+        //     if f.is_nan() || f.is_infinite() {
+        //         // XXX: pad null or return err here?
+        //         builder.add_null()
+        //     } else {
+        //         builder.add_f64(*f)
+        //     }
+        // }
+        // // XXX: What encoding to use?
+        // // ToText is \x plus hex string.
+        // Value::Bytes(b) => builder.add_string(&ToText::to_text(&b.as_slice())),
+        // Value::Enum(_, symbol) => {
+        //     builder.add_string(&symbol);
+        // }
+        // Value::Uuid(id) => builder.add_string(&id.as_hyphenated().to_string()),
+        // // For Union, one concern is that the avro union is tagged (like rust enum) but json union is untagged (like c union).
+        // // When the union consists of multiple records, it is possible to distinguish which variant is active in avro, but in json they will all become jsonb objects and indistinguishable.
+        // Value::Union(_, v) => avro_to_jsonb(v, builder)?
+        // XXX: pad null or return err here?
+        v @ (Value::Long(_)
+        | Value::Float(_)
+        | Value::Double(_)
+        | Value::Bytes(_)
+        | Value::Enum(_, _)
+        | Value::Fixed(_, _)
+        | Value::Date(_)
+        | Value::Decimal(_)
+        | Value::TimeMillis(_)
+        | Value::TimeMicros(_)
+        | Value::TimestampMillis(_)
+        | Value::TimestampMicros(_)
+        | Value::LocalTimestampMillis(_)
+        | Value::LocalTimestampMicros(_)
+        | Value::Duration(_)
+        | Value::Uuid(_)
+        | Value::Union(_, _)) => {
+            bail_uncategorized!(
+                "unimplemented conversion from avro to jsonb: {:?}",
+                ValueKind::from(v)
+            )
+        }
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::str::FromStr;
+
+    use apache_avro::Decimal as AvroDecimal;
+    use risingwave_common::types::{Datum, Decimal};
+
+    use super::*;
+
+    #[test]
+    fn test_convert_decimal() {
+        // 280
+        let v = vec![1, 24];
+        let avro_decimal = AvroDecimal::from(v);
+        let rust_decimal = avro_decimal_to_rust_decimal(avro_decimal, 28, 0).unwrap();
+        assert_eq!(rust_decimal, rust_decimal::Decimal::from(280));
+
+        // 28.1
+        let v = vec![1, 25];
+        let avro_decimal = AvroDecimal::from(v);
+        let rust_decimal = avro_decimal_to_rust_decimal(avro_decimal, 28, 1).unwrap();
+        assert_eq!(rust_decimal, rust_decimal::Decimal::try_from(28.1).unwrap());
+
+        // 1.1234567891
+        let value = BigInt::from(11234567891_i64);
+        let negative = value.sign() == Sign::Minus;
+        let (lo, mid, hi) = extract_decimal(value.to_bytes_be().1).unwrap();
+        let decimal = rust_decimal::Decimal::from_parts(lo, mid, hi, negative, 10);
+        assert_eq!(
+            decimal,
+            rust_decimal::Decimal::try_from(1.1234567891).unwrap()
+        );
+
+        // 1.123456789123456789123456789
+        let v = vec![3, 161, 77, 58, 146, 180, 49, 220, 100, 4, 95, 21];
+        let avro_decimal = AvroDecimal::from(v);
+        let rust_decimal = avro_decimal_to_rust_decimal(avro_decimal, 28, 27).unwrap();
+        assert_eq!(
+            rust_decimal,
+            rust_decimal::Decimal::from_str("1.123456789123456789123456789").unwrap()
+        );
+    }
+
+    /// Convert Avro value to datum.For now, support the following [Avro type](https://avro.apache.org/docs/current/spec.html).
+    ///  - boolean
+    ///  - int : i32
+    ///  - long: i64
+    ///  - float: f32
+    ///  - double: f64
+    ///  - string: String
+    ///  - Date (the number of days from the unix epoch, 1970-1-1 UTC)
+    ///  - Timestamp (the number of milliseconds from the unix epoch,  1970-1-1 00:00:00.000 UTC)
+    fn from_avro_value(
+        value: Value,
+        value_schema: &Schema,
+        shape: &DataType,
+    ) -> anyhow::Result<Datum> {
+        Ok(AvroParseOptions::create(value_schema)
+            .convert_to_datum(&value, shape)?
+            .to_owned_datum())
+    }
+
+    #[test]
+    fn test_avro_timestamptz_micros() {
+        let v1 = Value::TimestampMicros(1620000000000000);
+        let v2 = Value::TimestampMillis(1620000000000);
+        let value_schema1 = Schema::TimestampMicros;
+        let value_schema2 = Schema::TimestampMillis;
+        let datum1 = from_avro_value(v1, &value_schema1, &DataType::Timestamptz).unwrap();
+        let datum2 = from_avro_value(v2, &value_schema2, &DataType::Timestamptz).unwrap();
+        assert_eq!(
+            datum1,
+            Some(ScalarImpl::Timestamptz(
+                Timestamptz::from_str("2021-05-03T00:00:00Z").unwrap()
+            ))
+        );
+        assert_eq!(
+            datum2,
+            Some(ScalarImpl::Timestamptz(
+                Timestamptz::from_str("2021-05-03T00:00:00Z").unwrap()
+            ))
+        );
+    }
+
+    #[test]
+    fn test_decimal_truncate() {
+        let schema = Schema::parse_str(
+            r#"
+            {
+                "type": "bytes",
+                "logicalType": "decimal",
+                "precision": 38,
+                "scale": 18
+            }
+            "#,
+        )
+        .unwrap();
+        let bytes = vec![0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f];
+        let value = Value::Decimal(AvroDecimal::from(bytes));
+        let options = AvroParseOptions::create(&schema);
+        let resp = options
+            .convert_to_datum(&value, &DataType::Decimal)
+            .unwrap()
+            .to_owned_datum();
+        assert_eq!(
+            resp,
+            Some(ScalarImpl::Decimal(Decimal::Normalized(
+                rust_decimal::Decimal::from_str("0.017802464409370431").unwrap()
+            )))
+        );
+    }
+
+    #[test]
+    fn test_variable_scale_decimal() {
+        let schema = Schema::parse_str(
+            r#"
+            {
+                "type": "record",
+                "name": "VariableScaleDecimal",
+                "namespace": "io.debezium.data",
+                "fields": [
+                    {
+                        "name": "scale",
+                        "type": "int"
+                    },
+                    {
+                        "name": "value",
+                        "type": "bytes"
+                    }
+                ]
+            }
+            "#,
+        )
+        .unwrap();
+        let value = Value::Record(vec![
+            ("scale".to_string(), Value::Int(0)),
+            ("value".to_string(), Value::Bytes(vec![0x01, 0x02, 0x03])),
+        ]);
+
+        let options = AvroParseOptions::create(&schema);
+        let resp = options
+            .convert_to_datum(&value, &DataType::Decimal)
+            .unwrap()
+            .to_owned_datum();
+        assert_eq!(resp, Some(ScalarImpl::Decimal(Decimal::from(66051))));
+    }
+}
diff --git a/src/connector/codec/src/decoder/avro/schema.rs b/src/connector/codec/src/decoder/avro/schema.rs
new file mode 100644
index 0000000000000..4bd0a469041ad
--- /dev/null
+++ b/src/connector/codec/src/decoder/avro/schema.rs
@@ -0,0 +1,280 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::{Arc, LazyLock};
+
+use apache_avro::schema::{DecimalSchema, RecordSchema, ResolvedSchema, Schema};
+use apache_avro::AvroResult;
+use itertools::Itertools;
+use risingwave_common::bail;
+use risingwave_common::log::LogSuppresser;
+use risingwave_common::types::{DataType, Decimal};
+use risingwave_pb::plan_common::{AdditionalColumn, ColumnDesc, ColumnDescVersion};
+
+/// Avro schema with `Ref` inlined. The newtype is used to indicate whether the schema is resolved.
+///
+/// TODO: Actually most of the place should use resolved schema, but currently they just happen to work (Some edge cases are not met yet).
+///
+/// TODO: refactor avro lib to use the feature there.
+#[derive(Debug)]
+pub struct ResolvedAvroSchema {
+    /// Should be used for parsing bytes into Avro value
+    pub original_schema: Arc<Schema>,
+    /// Should be used for type mapping from Avro value to RisingWave datum
+    pub resolved_schema: Schema,
+}
+
+impl ResolvedAvroSchema {
+    pub fn create(schema: Arc<Schema>) -> AvroResult<Self> {
+        let resolver = ResolvedSchema::try_from(schema.as_ref())?;
+        // todo: to_resolved may cause stackoverflow if there's a loop in the schema
+        let resolved_schema = resolver.to_resolved(schema.as_ref())?;
+        Ok(Self {
+            original_schema: schema,
+            resolved_schema,
+        })
+    }
+}
+
+/// How to convert the map type from the input encoding to RisingWave's datatype.
+///
+/// XXX: Should this be `avro.map.handling.mode`? Can it be shared between Avro and Protobuf?
+#[derive(Debug, Copy, Clone)]
+pub enum MapHandling {
+    Jsonb,
+    // TODO: <https://github.com/risingwavelabs/risingwave/issues/13387>
+    // Map
+}
+
+impl MapHandling {
+    pub const OPTION_KEY: &'static str = "map.handling.mode";
+
+    pub fn from_options(
+        options: &std::collections::BTreeMap<String, String>,
+    ) -> anyhow::Result<Option<Self>> {
+        let mode = match options.get(Self::OPTION_KEY).map(std::ops::Deref::deref) {
+            Some("jsonb") => Self::Jsonb,
+            Some(v) => bail!("unrecognized {} value {}", Self::OPTION_KEY, v),
+            None => return Ok(None),
+        };
+        Ok(Some(mode))
+    }
+}
+
+/// This function expects resolved schema (no `Ref`).
+/// FIXME: require passing resolved schema here.
+pub fn avro_schema_to_column_descs(
+    schema: &Schema,
+    map_handling: Option<MapHandling>,
+) -> anyhow::Result<Vec<ColumnDesc>> {
+    if let Schema::Record(RecordSchema { fields, .. }) = schema {
+        let mut index = 0;
+        let fields = fields
+            .iter()
+            .map(|field| {
+                avro_field_to_column_desc(&field.name, &field.schema, &mut index, map_handling)
+            })
+            .collect::<anyhow::Result<_>>()?;
+        Ok(fields)
+    } else {
+        bail!("schema invalid, record type required at top level of the schema.");
+    }
+}
+
+const DBZ_VARIABLE_SCALE_DECIMAL_NAME: &str = "VariableScaleDecimal";
+const DBZ_VARIABLE_SCALE_DECIMAL_NAMESPACE: &str = "io.debezium.data";
+
+fn avro_field_to_column_desc(
+    name: &str,
+    schema: &Schema,
+    index: &mut i32,
+    map_handling: Option<MapHandling>,
+) -> anyhow::Result<ColumnDesc> {
+    let data_type = avro_type_mapping(schema, map_handling)?;
+    match schema {
+        Schema::Record(RecordSchema {
+            name: schema_name,
+            fields,
+            ..
+        }) => {
+            let vec_column = fields
+                .iter()
+                .map(|f| avro_field_to_column_desc(&f.name, &f.schema, index, map_handling))
+                .collect::<anyhow::Result<_>>()?;
+            *index += 1;
+            Ok(ColumnDesc {
+                column_type: Some(data_type.to_protobuf()),
+                column_id: *index,
+                name: name.to_owned(),
+                field_descs: vec_column,
+                type_name: schema_name.to_string(),
+                generated_or_default_column: None,
+                description: None,
+                additional_column_type: 0, // deprecated
+                additional_column: Some(AdditionalColumn { column_type: None }),
+                version: ColumnDescVersion::Pr13707 as i32,
+            })
+        }
+        _ => {
+            *index += 1;
+            Ok(ColumnDesc {
+                column_type: Some(data_type.to_protobuf()),
+                column_id: *index,
+                name: name.to_owned(),
+                additional_column: Some(AdditionalColumn { column_type: None }),
+                version: ColumnDescVersion::Pr13707 as i32,
+                ..Default::default()
+            })
+        }
+    }
+}
+
+/// This function expects resolved schema (no `Ref`).
+fn avro_type_mapping(
+    schema: &Schema,
+    map_handling: Option<MapHandling>,
+) -> anyhow::Result<DataType> {
+    let data_type = match schema {
+        Schema::String => DataType::Varchar,
+        Schema::Int => DataType::Int32,
+        Schema::Long => DataType::Int64,
+        Schema::Boolean => DataType::Boolean,
+        Schema::Float => DataType::Float32,
+        Schema::Double => DataType::Float64,
+        Schema::Decimal(DecimalSchema { precision, .. }) => {
+            if *precision > Decimal::MAX_PRECISION.into() {
+                static LOG_SUPPERSSER: LazyLock<LogSuppresser> =
+                    LazyLock::new(LogSuppresser::default);
+                if let Ok(suppressed_count) = LOG_SUPPERSSER.check() {
+                    tracing::warn!(
+                        suppressed_count,
+                        "RisingWave supports decimal precision up to {}, but got {}. Will truncate.",
+                        Decimal::MAX_PRECISION,
+                        precision
+                    );
+                }
+            }
+            DataType::Decimal
+        }
+        Schema::Date => DataType::Date,
+        Schema::LocalTimestampMillis => DataType::Timestamp,
+        Schema::LocalTimestampMicros => DataType::Timestamp,
+        Schema::TimestampMillis => DataType::Timestamptz,
+        Schema::TimestampMicros => DataType::Timestamptz,
+        Schema::Duration => DataType::Interval,
+        Schema::Bytes => DataType::Bytea,
+        Schema::Enum { .. } => DataType::Varchar,
+        Schema::TimeMillis => DataType::Time,
+        Schema::TimeMicros => DataType::Time,
+        Schema::Record(RecordSchema { fields, name, .. }) => {
+            if name.name == DBZ_VARIABLE_SCALE_DECIMAL_NAME
+                && name.namespace == Some(DBZ_VARIABLE_SCALE_DECIMAL_NAMESPACE.into())
+            {
+                return Ok(DataType::Decimal);
+            }
+
+            let struct_fields = fields
+                .iter()
+                .map(|f| avro_type_mapping(&f.schema, map_handling))
+                .collect::<anyhow::Result<_>>()?;
+            let struct_names = fields.iter().map(|f| f.name.clone()).collect_vec();
+            DataType::new_struct(struct_fields, struct_names)
+        }
+        Schema::Array(item_schema) => {
+            let item_type = avro_type_mapping(item_schema.as_ref(), map_handling)?;
+            DataType::List(Box::new(item_type))
+        }
+        Schema::Union(union_schema) => {
+            // We only support using union to represent nullable fields, not general unions.
+            let variants = union_schema.variants();
+            if variants.len() != 2 || !variants.contains(&Schema::Null) {
+                bail!(
+                    "unsupported Avro type, only unions like [null, T] is supported: {:?}",
+                    schema
+                );
+            }
+            let nested_schema = variants
+                .iter()
+                .find_or_first(|s| !matches!(s, Schema::Null))
+                .unwrap();
+
+            avro_type_mapping(nested_schema, map_handling)?
+        }
+        Schema::Ref { name } => {
+            if name.name == DBZ_VARIABLE_SCALE_DECIMAL_NAME
+                && name.namespace == Some(DBZ_VARIABLE_SCALE_DECIMAL_NAMESPACE.into())
+            {
+                DataType::Decimal
+            } else {
+                bail!("unsupported Avro type: {:?}", schema);
+            }
+        }
+        Schema::Map(value_schema) => {
+            // TODO: support native map type
+            match map_handling {
+                Some(MapHandling::Jsonb) => {
+                    if supported_avro_to_json_type(value_schema) {
+                        DataType::Jsonb
+                    } else {
+                        bail!(
+                            "unsupported Avro type, cannot convert map to jsonb: {:?}",
+                            schema
+                        )
+                    }
+                }
+                None => {
+                    bail!("`map.handling.mode` not specified in ENCODE AVRO (...). Currently supported modes: `jsonb`")
+                }
+            }
+        }
+        Schema::Uuid => DataType::Varchar,
+        Schema::Null | Schema::Fixed(_) => {
+            bail!("unsupported Avro type: {:?}", schema)
+        }
+    };
+
+    Ok(data_type)
+}
+
+/// Check for [`super::avro_to_jsonb`]
+fn supported_avro_to_json_type(schema: &Schema) -> bool {
+    match schema {
+        Schema::Null | Schema::Boolean | Schema::Int | Schema::String => true,
+
+        Schema::Map(value_schema) | Schema::Array(value_schema) => {
+            supported_avro_to_json_type(value_schema)
+        }
+        Schema::Record(RecordSchema { fields, .. }) => fields
+            .iter()
+            .all(|f| supported_avro_to_json_type(&f.schema)),
+        Schema::Long
+        | Schema::Float
+        | Schema::Double
+        | Schema::Bytes
+        | Schema::Enum(_)
+        | Schema::Fixed(_)
+        | Schema::Decimal(_)
+        | Schema::Uuid
+        | Schema::Date
+        | Schema::TimeMillis
+        | Schema::TimeMicros
+        | Schema::TimestampMillis
+        | Schema::TimestampMicros
+        | Schema::LocalTimestampMillis
+        | Schema::LocalTimestampMicros
+        | Schema::Duration
+        | Schema::Ref { name: _ }
+        | Schema::Union(_) => false,
+    }
+}
diff --git a/src/connector/codec/src/decoder/mod.rs b/src/connector/codec/src/decoder/mod.rs
new file mode 100644
index 0000000000000..d71186815697e
--- /dev/null
+++ b/src/connector/codec/src/decoder/mod.rs
@@ -0,0 +1,79 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod avro;
+pub mod utils;
+
+use risingwave_common::types::{DataType, Datum, DatumCow, ToOwnedDatum};
+use thiserror::Error;
+use thiserror_ext::Macro;
+
+#[derive(Error, Debug, Macro)]
+#[thiserror_ext(macro(mangle, path = "crate::decoder"))]
+pub enum AccessError {
+    #[error("Undefined field `{name}` at `{path}`")]
+    Undefined { name: String, path: String },
+    #[error("Cannot parse value `{value}` with type `{got}` into expected type `{expected}`")]
+    TypeError {
+        expected: String,
+        got: String,
+        value: String,
+    },
+    #[error("Unsupported data type `{ty}`")]
+    UnsupportedType { ty: String },
+
+    #[error("Unsupported additional column `{name}`")]
+    UnsupportedAdditionalColumn { name: String },
+
+    /// Errors that are not categorized into variants above.
+    #[error("{message}")]
+    Uncategorized { message: String },
+}
+
+pub type AccessResult<T = Datum> = std::result::Result<T, AccessError>;
+
+/// Access to a field in the data structure.
+pub trait Access {
+    /// Accesses `path` in the data structure (*parsed* Avro/JSON/Protobuf data),
+    /// and then converts it to RisingWave `Datum`.
+    /// `type_expected` might or might not be used during the conversion depending on the implementation.
+    ///
+    /// # Path
+    ///
+    /// We usually expect the data is a record (struct), and `path` represents field path.
+    /// The data (or part of the data) represents the whole row (`Vec<Datum>`),
+    /// and we use different `path` to access one column at a time.
+    ///
+    /// e.g., for Avro, we access `["col_name"]`; for Debezium Avro, we access `["before", "col_name"]`.
+    ///
+    /// # Returns
+    ///
+    /// The implementation should prefer to return a borrowed [`DatumRef`](risingwave_common::types::DatumRef)
+    /// through [`DatumCow::Borrowed`] to avoid unnecessary allocation if possible, especially for fields
+    /// with string or bytes data. If that's not the case, it may return an owned [`Datum`] through
+    /// [`DatumCow::Owned`].
+    fn access<'a>(&'a self, path: &[&str], type_expected: &DataType) -> AccessResult<DatumCow<'a>>;
+}
+
+// Note: made an extension trait to disallow implementing or overriding `access_owned`.
+#[easy_ext::ext(AccessExt)]
+impl<A: Access> A {
+    /// Similar to `access`, but always returns an owned [`Datum`]. See [`Access::access`] for more details.
+    ///
+    /// Always prefer calling `access` directly if possible to avoid unnecessary allocation.
+    pub fn access_owned(&self, path: &[&str], type_expected: &DataType) -> AccessResult<Datum> {
+        self.access(path, type_expected)
+            .map(ToOwnedDatum::to_owned_datum)
+    }
+}
diff --git a/src/connector/codec/src/decoder/utils.rs b/src/connector/codec/src/decoder/utils.rs
new file mode 100644
index 0000000000000..d4ce4b5ed8970
--- /dev/null
+++ b/src/connector/codec/src/decoder/utils.rs
@@ -0,0 +1,52 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{bail_uncategorized, AccessResult};
+
+pub fn extract_decimal(bytes: Vec<u8>) -> AccessResult<(u32, u32, u32)> {
+    match bytes.len() {
+        len @ 0..=4 => {
+            let mut pad = vec![0; 4 - len];
+            pad.extend_from_slice(&bytes);
+            let lo = u32::from_be_bytes(pad.try_into().unwrap());
+            Ok((lo, 0, 0))
+        }
+        len @ 5..=8 => {
+            let zero_len = 8 - len;
+            let mid_end = 4 - zero_len;
+
+            let mut pad = vec![0; zero_len];
+            pad.extend_from_slice(&bytes[..mid_end]);
+            let mid = u32::from_be_bytes(pad.try_into().unwrap());
+
+            let lo = u32::from_be_bytes(bytes[mid_end..].to_owned().try_into().unwrap());
+            Ok((lo, mid, 0))
+        }
+        len @ 9..=12 => {
+            let zero_len = 12 - len;
+            let hi_end = 4 - zero_len;
+            let mid_end = hi_end + 4;
+
+            let mut pad = vec![0; zero_len];
+            pad.extend_from_slice(&bytes[..hi_end]);
+            let hi = u32::from_be_bytes(pad.try_into().unwrap());
+
+            let mid = u32::from_be_bytes(bytes[hi_end..mid_end].to_owned().try_into().unwrap());
+
+            let lo = u32::from_be_bytes(bytes[mid_end..].to_owned().try_into().unwrap());
+            Ok((lo, mid, hi))
+        }
+        _ => bail_uncategorized!("invalid decimal bytes length {}", bytes.len()),
+    }
+}
diff --git a/src/connector/codec/src/lib.rs b/src/connector/codec/src/lib.rs
new file mode 100644
index 0000000000000..67198c78516a0
--- /dev/null
+++ b/src/connector/codec/src/lib.rs
@@ -0,0 +1,44 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Encoding and decoding between external data formats and RisingWave datum (i.e., type mappings).
+
+#![allow(clippy::derive_partial_eq_without_eq)]
+#![feature(array_chunks)]
+#![feature(coroutines)]
+#![feature(proc_macro_hygiene)]
+#![feature(stmt_expr_attributes)]
+#![feature(box_patterns)]
+#![feature(trait_alias)]
+#![feature(lint_reasons)]
+#![feature(lazy_cell)]
+#![feature(let_chains)]
+#![feature(box_into_inner)]
+#![feature(type_alias_impl_trait)]
+#![feature(associated_type_defaults)]
+#![feature(impl_trait_in_assoc_type)]
+#![feature(iter_from_coroutine)]
+#![feature(if_let_guard)]
+#![feature(iterator_try_collect)]
+#![feature(try_blocks)]
+#![feature(error_generic_member_access)]
+#![feature(negative_impls)]
+#![feature(register_tool)]
+#![feature(assert_matches)]
+#![register_tool(rw)]
+#![recursion_limit = "256"]
+
+/// Converts JSON/AVRO/Protobuf data to RisingWave datum.
+/// The core API is [`decoder::Access`].
+pub mod decoder;
diff --git a/src/connector/src/connector_common/common.rs b/src/connector/src/connector_common/common.rs
index 302b68dd664a1..dfda61f6ce578 100644
--- a/src/connector/src/connector_common/common.rs
+++ b/src/connector/src/connector_common/common.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use std::borrow::Cow;
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::io::Write;
 use std::time::Duration;
 
@@ -44,6 +44,8 @@ use crate::source::nats::source::NatsOffset;
 pub const PRIVATE_LINK_BROKER_REWRITE_MAP_KEY: &str = "broker.rewrite.endpoints";
 pub const PRIVATE_LINK_TARGETS_KEY: &str = "privatelink.targets";
 
+const AWS_MSK_IAM_AUTH: &str = "AWS_MSK_IAM";
+
 #[derive(Debug, Clone, Deserialize)]
 pub struct AwsPrivateLinkItem {
     pub az_id: Option<String>,
@@ -59,16 +61,28 @@ use aws_types::SdkConfig;
 /// A flatten config map for aws auth.
 #[derive(Deserialize, Debug, Clone, WithOptions)]
 pub struct AwsAuthProps {
+    #[serde(rename = "aws.region", alias = "region")]
     pub region: Option<String>,
-    #[serde(alias = "endpoint_url")]
+
+    #[serde(
+        rename = "aws.endpoint_url",
+        alias = "endpoint_url",
+        alias = "endpoint"
+    )]
     pub endpoint: Option<String>,
+    #[serde(rename = "aws.credentials.access_key_id", alias = "access_key")]
     pub access_key: Option<String>,
+    #[serde(rename = "aws.credentials.secret_access_key", alias = "secret_key")]
     pub secret_key: Option<String>,
+    #[serde(rename = "aws.credentials.session_token", alias = "session_token")]
     pub session_token: Option<String>,
+    /// IAM role
+    #[serde(rename = "aws.credentials.role.arn", alias = "arn")]
     pub arn: Option<String>,
-    /// This field was added for kinesis. Not sure if it's useful for other connectors.
-    /// Please ignore it in the documentation for now.
+    /// external ID in IAM role trust policy
+    #[serde(rename = "aws.credentials.role.external_id", alias = "external_id")]
     pub external_id: Option<String>,
+    #[serde(rename = "aws.profile", alias = "profile")]
     pub profile: Option<String>,
 }
 
@@ -181,7 +195,7 @@ pub struct KafkaCommon {
     #[serde(rename = "properties.ssl.key.password")]
     ssl_key_password: Option<String>,
 
-    /// SASL mechanism if SASL is enabled. Currently support PLAIN, SCRAM and GSSAPI.
+    /// SASL mechanism if SASL is enabled. Currently support PLAIN, SCRAM, GSSAPI, and AWS_MSK_IAM.
     #[serde(rename = "properties.sasl.mechanism")]
     sasl_mechanism: Option<String>,
 
@@ -224,7 +238,7 @@ pub struct KafkaPrivateLinkCommon {
     /// This is generated from `private_link_targets` and `private_link_endpoint` in frontend, instead of given by users.
     #[serde(rename = "broker.rewrite.endpoints")]
     #[serde_as(as = "Option<JsonString>")]
-    pub broker_rewrite_map: Option<HashMap<String, String>>,
+    pub broker_rewrite_map: Option<BTreeMap<String, String>>,
 }
 
 const fn default_kafka_sync_call_timeout() -> Duration {
@@ -286,6 +300,13 @@ impl RdKafkaPropertiesCommon {
 
 impl KafkaCommon {
     pub(crate) fn set_security_properties(&self, config: &mut ClientConfig) {
+        // AWS_MSK_IAM
+        if self.is_aws_msk_iam() {
+            config.set("security.protocol", "SASL_SSL");
+            config.set("sasl.mechanism", "OAUTHBEARER");
+            return;
+        }
+
         // Security protocol
         if let Some(security_protocol) = self.security_protocol.as_ref() {
             config.set("security.protocol", security_protocol);
@@ -356,6 +377,16 @@ impl KafkaCommon {
         // Currently, we only support unsecured OAUTH.
         config.set("enable.sasl.oauthbearer.unsecure.jwt", "true");
     }
+
+    pub(crate) fn is_aws_msk_iam(&self) -> bool {
+        if let Some(sasl_mechanism) = self.sasl_mechanism.as_ref()
+            && sasl_mechanism == AWS_MSK_IAM_AUTH
+        {
+            true
+        } else {
+            false
+        }
+    }
 }
 
 #[derive(Clone, Debug, Deserialize, WithOptions)]
diff --git a/src/connector/src/error.rs b/src/connector/src/error.rs
index 925f90c20c967..ab4b3e7bc37b5 100644
--- a/src/connector/src/error.rs
+++ b/src/connector/src/error.rs
@@ -41,10 +41,12 @@ def_anyhow_newtype! {
     url::ParseError => "failed to parse url",
     serde_json::Error => "failed to parse json",
     csv::Error => "failed to parse csv",
+    uuid::Error => transparent, // believed to be self-explanatory
 
     // Connector errors
     opendal::Error => transparent, // believed to be self-explanatory
 
+    sqlx::Error => transparent, // believed to be self-explanatory
     mysql_async::Error => "MySQL error",
     tokio_postgres::Error => "Postgres error",
     apache_avro::Error => "Avro error",
diff --git a/src/connector/src/lib.rs b/src/connector/src/lib.rs
index f28f3cdce5e77..8c0ade401cb47 100644
--- a/src/connector/src/lib.rs
+++ b/src/connector/src/lib.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![expect(dead_code)]
 #![allow(clippy::derive_partial_eq_without_eq)]
 #![feature(array_chunks)]
 #![feature(coroutines)]
@@ -34,13 +33,14 @@
 #![feature(error_generic_member_access)]
 #![feature(negative_impls)]
 #![feature(register_tool)]
+#![feature(assert_matches)]
+#![feature(never_type)]
 #![register_tool(rw)]
 #![recursion_limit = "256"]
 
 use std::time::Duration;
 
 use duration_str::parse_std;
-use risingwave_pb::connector_service::SinkPayloadFormat;
 use serde::de;
 
 pub mod aws_utils;
@@ -63,19 +63,6 @@ pub use with_options::WithPropertiesExt;
 #[cfg(test)]
 mod with_options_test;
 
-#[derive(Clone, Debug, Default)]
-pub struct ConnectorParams {
-    pub sink_payload_format: SinkPayloadFormat,
-}
-
-impl ConnectorParams {
-    pub fn new(sink_payload_format: SinkPayloadFormat) -> Self {
-        Self {
-            sink_payload_format,
-        }
-    }
-}
-
 pub(crate) fn deserialize_u32_from_string<'de, D>(deserializer: D) -> Result<u32, D::Error>
 where
     D: de::Deserializer<'de>,
@@ -175,6 +162,8 @@ mod tests {
 
     /// Test some serde behavior we rely on.
     mod serde {
+        #![expect(dead_code)]
+
         use std::collections::BTreeMap;
 
         use expect_test::expect;
diff --git a/src/connector/src/macros.rs b/src/connector/src/macros.rs
index b369e6d8a11e3..48f4e16c747b1 100644
--- a/src/connector/src/macros.rs
+++ b/src/connector/src/macros.rs
@@ -94,7 +94,9 @@ macro_rules! dispatch_source_enum_inner {
         match $impl {
             $(
                 $enum_name::$source_variant($inner_name) => {
+                    #[allow(dead_code)]
                     type $prop_type_name = $prop_name;
+                    #[allow(dead_code)]
                     type $split_type_name = $split;
                     {
                         $body
@@ -238,6 +240,7 @@ macro_rules! impl_cdc_source_type {
             }
         )*
 
+        #[derive(Clone)]
         pub enum CdcSourceType {
             $(
                 $cdc_source_type,
diff --git a/src/connector/src/parser/additional_columns.rs b/src/connector/src/parser/additional_columns.rs
index 59c47b06ee8b8..253718a00a7df 100644
--- a/src/connector/src/parser/additional_columns.rs
+++ b/src/connector/src/parser/additional_columns.rs
@@ -22,12 +22,14 @@ use risingwave_pb::data::data_type::TypeName;
 use risingwave_pb::data::DataType as PbDataType;
 use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType;
 use risingwave_pb::plan_common::{
-    AdditionalColumn, AdditionalColumnFilename, AdditionalColumnHeader, AdditionalColumnHeaders,
-    AdditionalColumnKey, AdditionalColumnOffset, AdditionalColumnPartition,
-    AdditionalColumnTimestamp,
+    AdditionalCollectionName, AdditionalColumn, AdditionalColumnFilename, AdditionalColumnHeader,
+    AdditionalColumnHeaders, AdditionalColumnKey, AdditionalColumnOffset,
+    AdditionalColumnPartition, AdditionalColumnTimestamp, AdditionalDatabaseName,
+    AdditionalSchemaName, AdditionalTableName,
 };
 
 use crate::error::ConnectorResult;
+use crate::source::cdc::MONGODB_CDC_CONNECTOR;
 use crate::source::{
     GCS_CONNECTOR, KAFKA_CONNECTOR, KINESIS_CONNECTOR, OPENDAL_S3_CONNECTOR, PULSAR_CONNECTOR,
     S3_CONNECTOR,
@@ -55,9 +57,42 @@ pub static COMPATIBLE_ADDITIONAL_COLUMNS: LazyLock<HashMap<&'static str, HashSet
             (OPENDAL_S3_CONNECTOR, HashSet::from(["file", "offset"])),
             (S3_CONNECTOR, HashSet::from(["file", "offset"])),
             (GCS_CONNECTOR, HashSet::from(["file", "offset"])),
+            // mongodb-cdc doesn't support cdc backfill table
+            (
+                MONGODB_CDC_CONNECTOR,
+                HashSet::from([
+                    "timestamp",
+                    "partition",
+                    "offset",
+                    "database_name",
+                    "collection_name",
+                ]),
+            ),
         ])
     });
 
+// For CDC backfill table, the additional columns are added to the schema of `StreamCdcScan`
+pub static CDC_BACKFILL_TABLE_ADDITIONAL_COLUMNS: LazyLock<Option<HashSet<&'static str>>> =
+    LazyLock::new(|| {
+        Some(HashSet::from([
+            "timestamp",
+            "database_name",
+            "schema_name",
+            "table_name",
+        ]))
+    });
+
+pub fn get_supported_additional_columns(
+    connector_name: &str,
+    is_cdc_backfill: bool,
+) -> Option<&HashSet<&'static str>> {
+    if is_cdc_backfill {
+        CDC_BACKFILL_TABLE_ADDITIONAL_COLUMNS.as_ref()
+    } else {
+        COMPATIBLE_ADDITIONAL_COLUMNS.get(connector_name)
+    }
+}
+
 pub fn gen_default_addition_col_name(
     connector_name: &str,
     additional_col_type: &str,
@@ -87,9 +122,10 @@ pub fn build_additional_column_catalog(
     inner_field_name: Option<&str>,
     data_type: Option<&str>,
     reject_unknown_connector: bool,
+    is_cdc_backfill_table: bool,
 ) -> ConnectorResult<ColumnCatalog> {
     let compatible_columns = match (
-        COMPATIBLE_ADDITIONAL_COLUMNS.get(connector_name),
+        get_supported_additional_columns(connector_name, is_cdc_backfill_table),
         reject_unknown_connector,
     ) {
         (Some(compat_cols), _) => compat_cols,
@@ -179,6 +215,55 @@ pub fn build_additional_column_catalog(
             is_hidden: false,
         },
         "header" => build_header_catalog(column_id, &column_name, inner_field_name, data_type),
+        "database_name" => ColumnCatalog {
+            column_desc: ColumnDesc::named_with_additional_column(
+                column_name,
+                column_id,
+                DataType::Varchar,
+                AdditionalColumn {
+                    column_type: Some(AdditionalColumnType::DatabaseName(
+                        AdditionalDatabaseName {},
+                    )),
+                },
+            ),
+            is_hidden: false,
+        },
+        "schema_name" => ColumnCatalog {
+            column_desc: ColumnDesc::named_with_additional_column(
+                column_name,
+                column_id,
+                DataType::Varchar,
+                AdditionalColumn {
+                    column_type: Some(AdditionalColumnType::SchemaName(AdditionalSchemaName {})),
+                },
+            ),
+            is_hidden: false,
+        },
+
+        "table_name" => ColumnCatalog {
+            column_desc: ColumnDesc::named_with_additional_column(
+                column_name,
+                column_id,
+                DataType::Varchar,
+                AdditionalColumn {
+                    column_type: Some(AdditionalColumnType::TableName(AdditionalTableName {})),
+                },
+            ),
+            is_hidden: false,
+        },
+        "collection_name" => ColumnCatalog {
+            column_desc: ColumnDesc::named_with_additional_column(
+                column_name,
+                column_id,
+                DataType::Varchar,
+                AdditionalColumn {
+                    column_type: Some(AdditionalColumnType::CollectionName(
+                        AdditionalCollectionName {},
+                    )),
+                },
+            ),
+            is_hidden: false,
+        },
         _ => unreachable!(),
     };
 
@@ -190,7 +275,7 @@ pub fn build_additional_column_catalog(
 /// ## Returns
 /// - `columns_exist`: whether 1. `partition`/`file` and 2. `offset` columns are included in `columns`.
 /// - `additional_columns`: The `ColumnCatalog` for `partition`/`file` and `offset` columns.
-pub fn add_partition_offset_cols(
+pub fn source_add_partition_offset_cols(
     columns: &[ColumnCatalog],
     connector_name: &str,
 ) -> ([bool; 2], [ColumnCatalog; 2]) {
@@ -219,6 +304,7 @@ pub fn add_partition_offset_cols(
                             None,
                             None,
                             false,
+                            false,
                         )
                         .unwrap(),
                     )
diff --git a/src/connector/src/parser/avro/parser.rs b/src/connector/src/parser/avro/parser.rs
index 65601d9c3a3e6..4b73c6ac7c80f 100644
--- a/src/connector/src/parser/avro/parser.rs
+++ b/src/connector/src/parser/avro/parser.rs
@@ -21,13 +21,13 @@ use apache_avro::{from_avro_datum, Reader, Schema};
 use risingwave_common::{bail, try_match_expand};
 use risingwave_pb::plan_common::ColumnDesc;
 
-use super::schema_resolver::ConfluentSchemaResolver;
-use super::util::avro_schema_to_column_descs;
+use super::schema_resolver::ConfluentSchemaCache;
+use super::util::{avro_schema_to_column_descs, ResolvedAvroSchema};
 use crate::error::ConnectorResult;
 use crate::parser::unified::avro::{AvroAccess, AvroParseOptions};
 use crate::parser::unified::AccessImpl;
 use crate::parser::util::bytes_from_url;
-use crate::parser::{AccessBuilder, EncodingProperties, EncodingType};
+use crate::parser::{AccessBuilder, AvroProperties, EncodingProperties, EncodingType, MapHandling};
 use crate::schema::schema_registry::{
     extract_schema_id, get_subject_by_strategy, handle_sr_list, Client,
 };
@@ -35,17 +35,18 @@ use crate::schema::schema_registry::{
 // Default avro access builder
 #[derive(Debug)]
 pub struct AvroAccessBuilder {
-    schema: Arc<Schema>,
-    pub schema_resolver: Option<Arc<ConfluentSchemaResolver>>,
+    schema: Arc<ResolvedAvroSchema>,
+    /// Refer to [`AvroParserConfig::writer_schema_cache`].
+    pub writer_schema_cache: Option<Arc<ConfluentSchemaCache>>,
     value: Option<Value>,
 }
 
 impl AccessBuilder for AvroAccessBuilder {
-    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_, '_>> {
-        self.value = self.parse_avro_value(&payload, Some(&*self.schema)).await?;
+    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_>> {
+        self.value = self.parse_avro_value(&payload).await?;
         Ok(AccessImpl::Avro(AvroAccess::new(
             self.value.as_ref().unwrap(),
-            AvroParseOptions::default().with_schema(&self.schema),
+            AvroParseOptions::create(&self.schema.resolved_schema),
         )))
     }
 }
@@ -55,7 +56,7 @@ impl AvroAccessBuilder {
         let AvroParserConfig {
             schema,
             key_schema,
-            schema_resolver,
+            writer_schema_cache,
             ..
         } = config;
         Ok(Self {
@@ -63,118 +64,125 @@ impl AvroAccessBuilder {
                 EncodingType::Key => key_schema.context("Avro with empty key schema")?,
                 EncodingType::Value => schema,
             },
-            schema_resolver,
+            writer_schema_cache,
             value: None,
         })
     }
 
-    async fn parse_avro_value(
-        &self,
-        payload: &[u8],
-        reader_schema: Option<&Schema>,
-    ) -> ConnectorResult<Option<Value>> {
+    /// Note: we should use unresolved schema to parsing bytes into avro value.
+    /// Otherwise it's an invalid schema and parsing will fail. (Avro error: Two named schema defined for same fullname)
+    async fn parse_avro_value(&self, payload: &[u8]) -> ConnectorResult<Option<Value>> {
         // parse payload to avro value
         // if use confluent schema, get writer schema from confluent schema registry
-        if let Some(resolver) = &self.schema_resolver {
+        if let Some(resolver) = &self.writer_schema_cache {
             let (schema_id, mut raw_payload) = extract_schema_id(payload)?;
-            let writer_schema = resolver.get(schema_id).await?;
+            let writer_schema = resolver.get_by_id(schema_id).await?;
             Ok(Some(from_avro_datum(
                 writer_schema.as_ref(),
                 &mut raw_payload,
-                reader_schema,
+                Some(&self.schema.original_schema),
             )?))
-        } else if let Some(schema) = reader_schema {
-            let mut reader = Reader::with_schema(schema, payload)?;
+        } else {
+            let mut reader = Reader::with_schema(&self.schema.original_schema, payload)?;
             match reader.next() {
                 Some(Ok(v)) => Ok(Some(v)),
                 Some(Err(e)) => Err(e)?,
                 None => bail!("avro parse unexpected eof"),
             }
-        } else {
-            unreachable!("both schema_resolver and reader_schema not exist");
         }
     }
 }
 
 #[derive(Debug, Clone)]
 pub struct AvroParserConfig {
-    pub schema: Arc<Schema>,
-    pub key_schema: Option<Arc<Schema>>,
-    pub schema_resolver: Option<Arc<ConfluentSchemaResolver>>,
+    pub schema: Arc<ResolvedAvroSchema>,
+    pub key_schema: Option<Arc<ResolvedAvroSchema>>,
+    /// Writer schema is the schema used to write the data. When parsing Avro data, the exactly same schema
+    /// must be used to decode the message, and then convert it with the reader schema.
+    pub writer_schema_cache: Option<Arc<ConfluentSchemaCache>>,
+
+    pub map_handling: Option<MapHandling>,
 }
 
 impl AvroParserConfig {
     pub async fn new(encoding_properties: EncodingProperties) -> ConnectorResult<Self> {
-        let avro_config = try_match_expand!(encoding_properties, EncodingProperties::Avro)?;
-        let schema_location = &avro_config.row_schema_location;
-        let enable_upsert = avro_config.enable_upsert;
+        let AvroProperties {
+            use_schema_registry,
+            row_schema_location: schema_location,
+            client_config,
+            aws_auth_props,
+            topic,
+            enable_upsert,
+            record_name,
+            key_record_name,
+            name_strategy,
+            map_handling,
+        } = try_match_expand!(encoding_properties, EncodingProperties::Avro)?;
         let url = handle_sr_list(schema_location.as_str())?;
-        if avro_config.use_schema_registry {
-            let client = Client::new(url, &avro_config.client_config)?;
-            let resolver = ConfluentSchemaResolver::new(client);
+        if use_schema_registry {
+            let client = Client::new(url, &client_config)?;
+            let resolver = ConfluentSchemaCache::new(client);
 
             let subject_key = if enable_upsert {
                 Some(get_subject_by_strategy(
-                    &avro_config.name_strategy,
-                    avro_config.topic.as_str(),
-                    avro_config.key_record_name.as_deref(),
+                    &name_strategy,
+                    topic.as_str(),
+                    key_record_name.as_deref(),
                     true,
                 )?)
             } else {
-                if let Some(name) = &avro_config.key_record_name {
-                    bail!("key.message = {name} not used");
+                if let Some(name) = &key_record_name {
+                    bail!("unused FORMAT ENCODE option: key.message='{name}'");
                 }
                 None
             };
             let subject_value = get_subject_by_strategy(
-                &avro_config.name_strategy,
-                avro_config.topic.as_str(),
-                avro_config.record_name.as_deref(),
+                &name_strategy,
+                topic.as_str(),
+                record_name.as_deref(),
                 false,
             )?;
             tracing::debug!("infer key subject {subject_key:?}, value subject {subject_value}");
 
             Ok(Self {
-                schema: resolver.get_by_subject_name(&subject_value).await?,
+                schema: Arc::new(ResolvedAvroSchema::create(
+                    resolver.get_by_subject(&subject_value).await?,
+                )?),
                 key_schema: if let Some(subject_key) = subject_key {
-                    Some(resolver.get_by_subject_name(&subject_key).await?)
+                    Some(Arc::new(ResolvedAvroSchema::create(
+                        resolver.get_by_subject(&subject_key).await?,
+                    )?))
                 } else {
                     None
                 },
-                schema_resolver: Some(Arc::new(resolver)),
+                writer_schema_cache: Some(Arc::new(resolver)),
+                map_handling,
             })
         } else {
             if enable_upsert {
                 bail!("avro upsert without schema registry is not supported");
             }
             let url = url.first().unwrap();
-            let schema_content = bytes_from_url(url, avro_config.aws_auth_props.as_ref()).await?;
+            let schema_content = bytes_from_url(url, aws_auth_props.as_ref()).await?;
             let schema = Schema::parse_reader(&mut schema_content.as_slice())
                 .context("failed to parse avro schema")?;
             Ok(Self {
-                schema: Arc::new(schema),
+                schema: Arc::new(ResolvedAvroSchema::create(Arc::new(schema))?),
                 key_schema: None,
-                schema_resolver: None,
+                writer_schema_cache: None,
+                map_handling,
             })
         }
     }
 
-    pub fn extract_pks(&self) -> ConnectorResult<Vec<ColumnDesc>> {
-        avro_schema_to_column_descs(
-            self.key_schema
-                .as_deref()
-                .ok_or_else(|| anyhow::format_err!("key schema is required"))?,
-        )
-    }
-
     pub fn map_to_columns(&self) -> ConnectorResult<Vec<ColumnDesc>> {
-        avro_schema_to_column_descs(self.schema.as_ref())
+        avro_schema_to_column_descs(&self.schema.resolved_schema, self.map_handling)
+            .map_err(Into::into)
     }
 }
 
 #[cfg(test)]
 mod test {
-    use std::collections::HashMap;
     use std::env;
     use std::fs::OpenOptions;
     use std::io::Write;
@@ -182,25 +190,22 @@ mod test {
     use std::path::PathBuf;
 
     use apache_avro::schema::RecordSchema;
-    use apache_avro::types::{Record, Value};
-    use apache_avro::{Codec, Days, Duration, Millis, Months, Reader, Schema, Writer};
+    use apache_avro::types::Record;
+    use apache_avro::{Codec, Days, Duration, Millis, Months, Writer};
     use itertools::Itertools;
     use risingwave_common::array::Op;
     use risingwave_common::catalog::ColumnId;
     use risingwave_common::row::Row;
-    use risingwave_common::types::{DataType, Date, Interval, ScalarImpl, Timestamptz};
+    use risingwave_common::types::{DataType, Date};
+    use risingwave_common::util::iter_util::ZipEqFast;
     use risingwave_pb::catalog::StreamSourceInfo;
     use risingwave_pb::plan_common::{PbEncodeType, PbFormatType};
     use url::Url;
 
     use super::*;
     use crate::connector_common::AwsAuthProps;
-    use crate::error::ConnectorResult;
     use crate::parser::plain_parser::PlainParser;
-    use crate::parser::unified::avro::unix_epoch_days;
-    use crate::parser::{
-        AccessBuilderImpl, EncodingType, SourceStreamChunkBuilder, SpecificParserConfig,
-    };
+    use crate::parser::{AccessBuilderImpl, SourceStreamChunkBuilder, SpecificParserConfig};
     use crate::source::{SourceColumnDesc, SourceContext};
 
     fn test_data_path(file_name: &str) -> String {
@@ -265,7 +270,7 @@ mod test {
             row_encode: PbEncodeType::Avro.into(),
             ..Default::default()
         };
-        let parser_config = SpecificParserConfig::new(&info, &HashMap::new())?;
+        let parser_config = SpecificParserConfig::new(&info, &Default::default())?;
         AvroParserConfig::new(parser_config.encoding_config).await
     }
 
@@ -290,7 +295,7 @@ mod test {
             .await
             .unwrap();
         let builder = try_match_expand!(&parser.payload_builder, AccessBuilderImpl::Avro).unwrap();
-        let schema = builder.schema.clone();
+        let schema = builder.schema.original_schema.clone();
         let record = build_avro_data(&schema);
         assert_eq!(record.fields.len(), 11);
         let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Snappy);
@@ -310,62 +315,28 @@ mod test {
         let chunk = builder.finish();
         let (op, row) = chunk.rows().next().unwrap();
         assert_eq!(op, Op::Insert);
-        let row = row.into_owned_row();
-        for (i, field) in record.fields.iter().enumerate() {
-            let value = field.clone().1;
-            match value {
-                Value::String(str) | Value::Union(_, box Value::String(str)) => {
-                    assert_eq!(row[i], Some(ScalarImpl::Utf8(str.into_boxed_str())));
-                }
-                Value::Boolean(bool_val) => {
-                    assert_eq!(row[i], Some(ScalarImpl::Bool(bool_val)));
-                }
-                Value::Int(int_val) => {
-                    assert_eq!(row[i], Some(ScalarImpl::Int32(int_val)));
-                }
-                Value::Long(i64_val) => {
-                    assert_eq!(row[i], Some(ScalarImpl::Int64(i64_val)));
-                }
-                Value::Float(f32_val) => {
-                    assert_eq!(row[i], Some(ScalarImpl::Float32(f32_val.into())));
-                }
-                Value::Double(f64_val) => {
-                    assert_eq!(row[i], Some(ScalarImpl::Float64(f64_val.into())));
-                }
-                Value::Date(days) => {
-                    assert_eq!(
-                        row[i],
-                        Some(ScalarImpl::Date(
-                            Date::with_days(days + unix_epoch_days()).unwrap(),
-                        ))
-                    );
-                }
-                Value::TimestampMillis(millis) => {
-                    assert_eq!(
-                        row[i],
-                        Some(Timestamptz::from_millis(millis).unwrap().into())
-                    );
-                }
-                Value::TimestampMicros(micros) => {
-                    assert_eq!(row[i], Some(Timestamptz::from_micros(micros).into()));
-                }
-                Value::Bytes(bytes) => {
-                    assert_eq!(row[i], Some(ScalarImpl::Bytea(bytes.into_boxed_slice())));
-                }
-                Value::Duration(duration) => {
-                    let months = u32::from(duration.months()) as i32;
-                    let days = u32::from(duration.days()) as i32;
-                    let usecs = (u32::from(duration.millis()) as i64) * 1000; // never overflows
-                    assert_eq!(
-                        row[i],
-                        Some(Interval::from_month_day_usec(months, days, usecs).into())
-                    );
-                }
-                _ => {
-                    unreachable!()
-                }
-            }
-        }
+
+        expect_test::expect![[r#"
+            ("id", Int(32)) => Some(Int32(32))
+            ("sequence_id", Long(64)) => Some(Int64(64))
+            ("name", Union(1, String("str_value"))) => Some(Utf8("str_value"))
+            ("score", Float(32.0)) => Some(Float32(OrderedFloat(32.0)))
+            ("avg_score", Double(64.0)) => Some(Float64(OrderedFloat(64.0)))
+            ("is_lasted", Boolean(true)) => Some(Bool(true))
+            ("entrance_date", Date(0)) => Some(Date(Date(1970-01-01)))
+            ("birthday", TimestampMillis(0)) => Some(Timestamptz(Timestamptz(0)))
+            ("anniversary", TimestampMicros(0)) => Some(Timestamptz(Timestamptz(0)))
+            ("passed", Duration(Duration { months: Months(1), days: Days(1), millis: Millis(1000) })) => Some(Interval(Interval { months: 1, days: 1, usecs: 1000000 }))
+            ("bytes", Bytes([1, 2, 3, 4, 5])) => Some(Bytea([1, 2, 3, 4, 5]))"#]].assert_eq(&format!(
+            "{}",
+            record
+                .fields
+                .iter()
+                .zip_eq_fast(row.iter())
+                .format_with("\n", |(avro, datum), f| {
+                    f(&format_args!("{:?} => {:?}", avro, datum))
+                })
+        ));
     }
 
     fn build_rw_columns() -> Vec<SourceColumnDesc> {
@@ -483,7 +454,7 @@ mod test {
             .await
             .unwrap();
         let builder = try_match_expand!(&parser.payload_builder, AccessBuilderImpl::Avro).unwrap();
-        let schema = &builder.schema;
+        let schema = &builder.schema.original_schema;
         let mut null_record = Record::new(schema).unwrap();
         null_record.put("id", Value::Int(5));
         null_record.put("age", Value::Union(0, Box::new(Value::Null)));
@@ -548,24 +519,24 @@ mod test {
     }
 
     // run this script when updating `simple-schema.avsc`, the script will generate new value in
-    // `avro_bin.1`
+    // `avro_simple_schema_bin.1`
     #[ignore]
     #[tokio::test]
     async fn update_avro_payload() {
         let conf = new_avro_conf_from_local("simple-schema.avsc")
             .await
             .unwrap();
-        let mut writer = Writer::new(&conf.schema, Vec::new());
-        let record = build_avro_data(&conf.schema);
+        let mut writer = Writer::new(conf.schema.original_schema.as_ref(), Vec::new());
+        let record = build_avro_data(conf.schema.original_schema.as_ref());
         writer.append(record).unwrap();
         let encoded = writer.into_inner().unwrap();
-        println!("path = {:?}", e2e_file_path("avro_bin.1"));
+        println!("path = {:?}", e2e_file_path("avro_simple_schema_bin.1"));
         let mut file = OpenOptions::new()
             .read(true)
             .write(true)
             .create(true)
             .truncate(true)
-            .open(e2e_file_path("avro_bin.1"))
+            .open(e2e_file_path("avro_simple_schema_bin.1"))
             .unwrap();
         file.write_all(encoded.as_slice()).unwrap();
         println!(
diff --git a/src/connector/src/parser/avro/schema_resolver.rs b/src/connector/src/parser/avro/schema_resolver.rs
index cdc52de7accee..058f9bcbf7ea3 100644
--- a/src/connector/src/parser/avro/schema_resolver.rs
+++ b/src/connector/src/parser/avro/schema_resolver.rs
@@ -21,13 +21,20 @@ use moka::future::Cache;
 use crate::error::ConnectorResult;
 use crate::schema::schema_registry::{Client, ConfluentSchema};
 
+/// Fetch schemas from confluent schema registry and cache them.
+///
+/// Background: This is mainly used for Avro **writer schema** (during schema evolution): When decoding an Avro message,
+/// we must get the message's schema id, and use the *exactly same schema* to decode the message, and then
+/// convert it with the reader schema. (This is also why Avro has to be used with a schema registry instead of a static schema file.)
+///
+/// TODO: support protobuf (not sure if it's needed)
 #[derive(Debug)]
-pub struct ConfluentSchemaResolver {
+pub struct ConfluentSchemaCache {
     writer_schemas: Cache<i32, Arc<Schema>>,
     confluent_client: Client,
 }
 
-impl ConfluentSchemaResolver {
+impl ConfluentSchemaCache {
     async fn parse_and_cache_schema(
         &self,
         raw_schema: ConfluentSchema,
@@ -43,29 +50,23 @@ impl ConfluentSchemaResolver {
 
     /// Create a new `ConfluentSchemaResolver`
     pub fn new(client: Client) -> Self {
-        ConfluentSchemaResolver {
+        ConfluentSchemaCache {
             writer_schemas: Cache::new(u64::MAX),
             confluent_client: client,
         }
     }
 
-    pub async fn get_by_subject_name(&self, subject_name: &str) -> ConnectorResult<Arc<Schema>> {
-        let raw_schema = self.get_raw_schema_by_subject_name(subject_name).await?;
-        self.parse_and_cache_schema(raw_schema).await
-    }
-
-    pub async fn get_raw_schema_by_subject_name(
-        &self,
-        subject_name: &str,
-    ) -> ConnectorResult<ConfluentSchema> {
-        self.confluent_client
+    /// Gets the latest schema by subject name, which is used as *reader schema*.
+    pub async fn get_by_subject(&self, subject_name: &str) -> ConnectorResult<Arc<Schema>> {
+        let raw_schema = self
+            .confluent_client
             .get_schema_by_subject(subject_name)
-            .await
-            .map_err(Into::into)
+            .await?;
+        self.parse_and_cache_schema(raw_schema).await
     }
 
-    // get the writer schema by id
-    pub async fn get(&self, schema_id: i32) -> ConnectorResult<Arc<Schema>> {
+    /// Gets the a specific schema by id, which is used as *writer schema*.
+    pub async fn get_by_id(&self, schema_id: i32) -> ConnectorResult<Arc<Schema>> {
         // TODO: use `get_with`
         if let Some(schema) = self.writer_schemas.get(&schema_id).await {
             Ok(schema)
diff --git a/src/connector/src/parser/avro/util.rs b/src/connector/src/parser/avro/util.rs
index a58ad884fd886..58043daa08b0f 100644
--- a/src/connector/src/parser/avro/util.rs
+++ b/src/connector/src/parser/avro/util.rs
@@ -12,150 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::LazyLock;
-
-use apache_avro::schema::{DecimalSchema, RecordSchema, Schema};
-use itertools::Itertools;
-use risingwave_common::bail;
-use risingwave_common::log::LogSuppresser;
-use risingwave_common::types::{DataType, Decimal};
-use risingwave_pb::plan_common::{AdditionalColumn, ColumnDesc, ColumnDescVersion};
-
-use crate::error::ConnectorResult;
-
-pub fn avro_schema_to_column_descs(schema: &Schema) -> ConnectorResult<Vec<ColumnDesc>> {
-    if let Schema::Record(RecordSchema { fields, .. }) = schema {
-        let mut index = 0;
-        let fields = fields
-            .iter()
-            .map(|field| avro_field_to_column_desc(&field.name, &field.schema, &mut index))
-            .collect::<ConnectorResult<Vec<_>>>()?;
-        Ok(fields)
-    } else {
-        bail!("schema invalid, record type required at top level of the schema.");
-    }
-}
-
-const DBZ_VARIABLE_SCALE_DECIMAL_NAME: &str = "VariableScaleDecimal";
-const DBZ_VARIABLE_SCALE_DECIMAL_NAMESPACE: &str = "io.debezium.data";
-
-fn avro_field_to_column_desc(
-    name: &str,
-    schema: &Schema,
-    index: &mut i32,
-) -> ConnectorResult<ColumnDesc> {
-    let data_type = avro_type_mapping(schema)?;
-    match schema {
-        Schema::Record(RecordSchema {
-            name: schema_name,
-            fields,
-            ..
-        }) => {
-            let vec_column = fields
-                .iter()
-                .map(|f| avro_field_to_column_desc(&f.name, &f.schema, index))
-                .collect::<ConnectorResult<Vec<_>>>()?;
-            *index += 1;
-            Ok(ColumnDesc {
-                column_type: Some(data_type.to_protobuf()),
-                column_id: *index,
-                name: name.to_owned(),
-                field_descs: vec_column,
-                type_name: schema_name.to_string(),
-                generated_or_default_column: None,
-                description: None,
-                additional_column_type: 0, // deprecated
-                additional_column: Some(AdditionalColumn { column_type: None }),
-                version: ColumnDescVersion::Pr13707 as i32,
-            })
-        }
-        _ => {
-            *index += 1;
-            Ok(ColumnDesc {
-                column_type: Some(data_type.to_protobuf()),
-                column_id: *index,
-                name: name.to_owned(),
-                additional_column: Some(AdditionalColumn { column_type: None }),
-                version: ColumnDescVersion::Pr13707 as i32,
-                ..Default::default()
-            })
-        }
-    }
-}
-
-fn avro_type_mapping(schema: &Schema) -> ConnectorResult<DataType> {
-    let data_type = match schema {
-        Schema::String => DataType::Varchar,
-        Schema::Int => DataType::Int32,
-        Schema::Long => DataType::Int64,
-        Schema::Boolean => DataType::Boolean,
-        Schema::Float => DataType::Float32,
-        Schema::Double => DataType::Float64,
-        Schema::Decimal(DecimalSchema { precision, .. }) => {
-            if *precision > Decimal::MAX_PRECISION.into() {
-                static LOG_SUPPERSSER: LazyLock<LogSuppresser> =
-                    LazyLock::new(LogSuppresser::default);
-                if let Ok(suppressed_count) = LOG_SUPPERSSER.check() {
-                    tracing::warn!(
-                        suppressed_count,
-                        "RisingWave supports decimal precision up to {}, but got {}. Will truncate.",
-                        Decimal::MAX_PRECISION,
-                        precision
-                    );
-                }
-            }
-            DataType::Decimal
-        }
-        Schema::Date => DataType::Date,
-        Schema::LocalTimestampMillis => DataType::Timestamp,
-        Schema::LocalTimestampMicros => DataType::Timestamp,
-        Schema::TimestampMillis => DataType::Timestamptz,
-        Schema::TimestampMicros => DataType::Timestamptz,
-        Schema::Duration => DataType::Interval,
-        Schema::Bytes => DataType::Bytea,
-        Schema::Enum { .. } => DataType::Varchar,
-        Schema::TimeMillis => DataType::Time,
-        Schema::TimeMicros => DataType::Time,
-        Schema::Record(RecordSchema { fields, name, .. }) => {
-            if name.name == DBZ_VARIABLE_SCALE_DECIMAL_NAME
-                && name.namespace == Some(DBZ_VARIABLE_SCALE_DECIMAL_NAMESPACE.into())
-            {
-                return Ok(DataType::Decimal);
-            }
-
-            let struct_fields = fields
-                .iter()
-                .map(|f| avro_type_mapping(&f.schema))
-                .collect::<ConnectorResult<Vec<_>>>()?;
-            let struct_names = fields.iter().map(|f| f.name.clone()).collect_vec();
-            DataType::new_struct(struct_fields, struct_names)
-        }
-        Schema::Array(item_schema) => {
-            let item_type = avro_type_mapping(item_schema.as_ref())?;
-            DataType::List(Box::new(item_type))
-        }
-        Schema::Union(union_schema) => {
-            let nested_schema = union_schema
-                .variants()
-                .iter()
-                .find_or_first(|s| !matches!(s, Schema::Null))
-                .ok_or_else(|| {
-                    anyhow::format_err!("unsupported type in Avro: {:?}", union_schema)
-                })?;
-
-            avro_type_mapping(nested_schema)?
-        }
-        Schema::Ref { name } => {
-            if name.name == DBZ_VARIABLE_SCALE_DECIMAL_NAME
-                && name.namespace == Some(DBZ_VARIABLE_SCALE_DECIMAL_NAMESPACE.into())
-            {
-                DataType::Decimal
-            } else {
-                bail!("unsupported type in Avro: {:?}", schema);
-            }
-        }
-        _ => bail!("unsupported type in Avro: {:?}", schema),
-    };
-
-    Ok(data_type)
-}
+pub use risingwave_connector_codec::decoder::avro::{
+    avro_schema_to_column_descs, ResolvedAvroSchema,
+};
diff --git a/src/connector/src/parser/bytes_parser.rs b/src/connector/src/parser/bytes_parser.rs
index 255c3ef829c79..5df7fa28118d3 100644
--- a/src/connector/src/parser/bytes_parser.rs
+++ b/src/connector/src/parser/bytes_parser.rs
@@ -26,7 +26,7 @@ pub struct BytesAccessBuilder {
 
 impl AccessBuilder for BytesAccessBuilder {
     #[allow(clippy::unused_async)]
-    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_, '_>> {
+    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_>> {
         Ok(AccessImpl::Bytes(BytesAccess::new(
             &self.column_name,
             payload,
diff --git a/src/connector/src/parser/canal/simd_json_parser.rs b/src/connector/src/parser/canal/simd_json_parser.rs
index 9bc236392eb48..ac8a63a6457e7 100644
--- a/src/connector/src/parser/canal/simd_json_parser.rs
+++ b/src/connector/src/parser/canal/simd_json_parser.rs
@@ -141,7 +141,6 @@ mod tests {
 
     use super::*;
     use crate::parser::SourceStreamChunkBuilder;
-    use crate::source::SourceColumnDesc;
 
     #[tokio::test]
     async fn test_data_types() {
diff --git a/src/connector/src/parser/common.rs b/src/connector/src/parser/common.rs
index 71ce50b29793a..102014966db42 100644
--- a/src/connector/src/parser/common.rs
+++ b/src/connector/src/parser/common.rs
@@ -18,10 +18,10 @@ use simd_json::BorrowedValue;
 /// Get a value from a json object by key, case insensitive.
 ///
 /// Returns `None` if the given json value is not an object, or the key is not found.
-pub(crate) fn json_object_get_case_insensitive<'a, 'b>(
-    v: &'b simd_json::BorrowedValue<'a>,
-    key: &'b str,
-) -> Option<&'b BorrowedValue<'a>> {
+pub(crate) fn json_object_get_case_insensitive<'b>(
+    v: &'b simd_json::BorrowedValue<'b>,
+    key: &str,
+) -> Option<&'b BorrowedValue<'b>> {
     let obj = v.as_object()?;
     let value = obj.get(key);
     if value.is_some() {
diff --git a/src/connector/src/parser/csv_parser.rs b/src/connector/src/parser/csv_parser.rs
index eaf09ab839891..25948c273d3e1 100644
--- a/src/connector/src/parser/csv_parser.rs
+++ b/src/connector/src/parser/csv_parser.rs
@@ -112,7 +112,7 @@ impl CsvParser {
                 // The header row does not output a row, so we return early.
                 return Ok(());
             }
-            writer.insert(|desc| {
+            writer.do_insert(|desc| {
                 if let Some(i) = headers.iter().position(|name| name == &desc.name) {
                     let value = fields.get_mut(i).map(std::mem::take).unwrap_or_default();
                     if value.is_empty() {
@@ -125,7 +125,7 @@ impl CsvParser {
             })?;
         } else {
             fields.reverse();
-            writer.insert(|desc| {
+            writer.do_insert(|desc| {
                 if let Some(value) = fields.pop() {
                     if value.is_empty() {
                         return Ok(None);
diff --git a/src/connector/src/parser/debezium/avro_parser.rs b/src/connector/src/parser/debezium/avro_parser.rs
index 29d9139d221cf..430d5072a88db 100644
--- a/src/connector/src/parser/debezium/avro_parser.rs
+++ b/src/connector/src/parser/debezium/avro_parser.rs
@@ -22,8 +22,8 @@ use risingwave_pb::catalog::PbSchemaRegistryNameStrategy;
 use risingwave_pb::plan_common::ColumnDesc;
 
 use crate::error::ConnectorResult;
-use crate::parser::avro::schema_resolver::ConfluentSchemaResolver;
-use crate::parser::avro::util::avro_schema_to_column_descs;
+use crate::parser::avro::schema_resolver::ConfluentSchemaCache;
+use crate::parser::avro::util::{avro_schema_to_column_descs, ResolvedAvroSchema};
 use crate::parser::unified::avro::{
     avro_extract_field_schema, avro_schema_skip_union, AvroAccess, AvroParseOptions,
 };
@@ -33,15 +33,10 @@ use crate::schema::schema_registry::{
     extract_schema_id, get_subject_by_strategy, handle_sr_list, Client,
 };
 
-const BEFORE: &str = "before";
-const AFTER: &str = "after";
-const OP: &str = "op";
-const PAYLOAD: &str = "payload";
-
 #[derive(Debug)]
 pub struct DebeziumAvroAccessBuilder {
-    schema: Schema,
-    schema_resolver: Arc<ConfluentSchemaResolver>,
+    schema: ResolvedAvroSchema,
+    schema_resolver: Arc<ConfluentSchemaCache>,
     key_schema: Option<Arc<Schema>>,
     value: Option<Value>,
     encoding_type: EncodingType,
@@ -49,9 +44,9 @@ pub struct DebeziumAvroAccessBuilder {
 
 // TODO: reduce encodingtype match
 impl AccessBuilder for DebeziumAvroAccessBuilder {
-    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_, '_>> {
+    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_>> {
         let (schema_id, mut raw_payload) = extract_schema_id(&payload)?;
-        let schema = self.schema_resolver.get(schema_id).await?;
+        let schema = self.schema_resolver.get_by_id(schema_id).await?;
         self.value = Some(from_avro_datum(schema.as_ref(), &mut raw_payload, None)?);
         self.key_schema = match self.encoding_type {
             EncodingType::Key => Some(schema),
@@ -59,9 +54,10 @@ impl AccessBuilder for DebeziumAvroAccessBuilder {
         };
         Ok(AccessImpl::Avro(AvroAccess::new(
             self.value.as_mut().unwrap(),
-            AvroParseOptions::default().with_schema(match self.encoding_type {
+            // Assumption: Key will not contain reference, so unresolved schema can work here.
+            AvroParseOptions::create(match self.encoding_type {
                 EncodingType::Key => self.key_schema.as_mut().unwrap(),
-                EncodingType::Value => &self.schema,
+                EncodingType::Value => &self.schema.resolved_schema,
             }),
         )))
     }
@@ -78,11 +74,8 @@ impl DebeziumAvroAccessBuilder {
             ..
         } = config;
 
-        let resolver = apache_avro::schema::ResolvedSchema::try_from(&*outer_schema)?;
-        // todo: to_resolved may cause stackoverflow if there's a loop in the schema
-        let schema = resolver.to_resolved(&outer_schema)?;
         Ok(Self {
-            schema,
+            schema: ResolvedAvroSchema::create(outer_schema)?,
             schema_resolver,
             key_schema: None,
             value: None,
@@ -96,7 +89,7 @@ impl DebeziumAvroAccessBuilder {
 pub struct DebeziumAvroParserConfig {
     pub key_schema: Arc<Schema>,
     pub outer_schema: Arc<Schema>,
-    pub schema_resolver: Arc<ConfluentSchemaResolver>,
+    pub schema_resolver: Arc<ConfluentSchemaCache>,
 }
 
 impl DebeziumAvroParserConfig {
@@ -107,13 +100,13 @@ impl DebeziumAvroParserConfig {
         let kafka_topic = &avro_config.topic;
         let url = handle_sr_list(schema_location)?;
         let client = Client::new(url, client_config)?;
-        let resolver = ConfluentSchemaResolver::new(client);
+        let resolver = ConfluentSchemaCache::new(client);
 
         let name_strategy = &PbSchemaRegistryNameStrategy::Unspecified;
         let key_subject = get_subject_by_strategy(name_strategy, kafka_topic, None, true)?;
         let val_subject = get_subject_by_strategy(name_strategy, kafka_topic, None, false)?;
-        let key_schema = resolver.get_by_subject_name(&key_subject).await?;
-        let outer_schema = resolver.get_by_subject_name(&val_subject).await?;
+        let key_schema = resolver.get_by_subject(&key_subject).await?;
+        let outer_schema = resolver.get_by_subject(&val_subject).await?;
 
         Ok(Self {
             key_schema,
@@ -123,14 +116,27 @@ impl DebeziumAvroParserConfig {
     }
 
     pub fn extract_pks(&self) -> ConnectorResult<Vec<ColumnDesc>> {
-        avro_schema_to_column_descs(&self.key_schema)
+        avro_schema_to_column_descs(
+            &self.key_schema,
+            // TODO: do we need to support map type here?
+            None,
+        )
+        .map_err(Into::into)
     }
 
     pub fn map_to_columns(&self) -> ConnectorResult<Vec<ColumnDesc>> {
-        avro_schema_to_column_descs(avro_schema_skip_union(avro_extract_field_schema(
-            &self.outer_schema,
-            Some("before"),
-        )?)?)
+        avro_schema_to_column_descs(
+            avro_schema_skip_union(avro_extract_field_schema(
+                // FIXME: use resolved schema here.
+                // Currently it works because "after" refers to a subtree in "before",
+                // but in theory, inside "before" there could also be a reference.
+                &self.outer_schema,
+                Some("before"),
+            )?)?,
+            // TODO: do we need to support map type here?
+            None,
+        )
+        .map_err(Into::into)
     }
 }
 
@@ -139,9 +145,8 @@ mod tests {
     use std::io::Read;
     use std::path::PathBuf;
 
-    use apache_avro::Schema;
     use itertools::Itertools;
-    use maplit::{convert_args, hashmap};
+    use maplit::{btreemap, convert_args};
     use risingwave_common::array::Op;
     use risingwave_common::catalog::ColumnDesc as CatColumnDesc;
     use risingwave_common::row::{OwnedRow, Row};
@@ -151,9 +156,7 @@ mod tests {
     use risingwave_pb::plan_common::{PbEncodeType, PbFormatType};
 
     use super::*;
-    use crate::parser::{
-        DebeziumAvroParserConfig, DebeziumParser, SourceStreamChunkBuilder, SpecificParserConfig,
-    };
+    use crate::parser::{DebeziumParser, SourceStreamChunkBuilder, SpecificParserConfig};
     use crate::source::{SourceColumnDesc, SourceContext};
 
     const DEBEZIUM_AVRO_DATA: &[u8] = b"\x00\x00\x00\x00\x06\x00\x02\xd2\x0f\x0a\x53\x61\x6c\x6c\x79\x0c\x54\x68\x6f\x6d\x61\x73\x2a\x73\x61\x6c\x6c\x79\x2e\x74\x68\x6f\x6d\x61\x73\x40\x61\x63\x6d\x65\x2e\x63\x6f\x6d\x16\x32\x2e\x31\x2e\x32\x2e\x46\x69\x6e\x61\x6c\x0a\x6d\x79\x73\x71\x6c\x12\x64\x62\x73\x65\x72\x76\x65\x72\x31\xc0\xb4\xe8\xb7\xc9\x61\x00\x30\x66\x69\x72\x73\x74\x5f\x69\x6e\x5f\x64\x61\x74\x61\x5f\x63\x6f\x6c\x6c\x65\x63\x74\x69\x6f\x6e\x12\x69\x6e\x76\x65\x6e\x74\x6f\x72\x79\x00\x02\x12\x63\x75\x73\x74\x6f\x6d\x65\x72\x73\x00\x00\x20\x6d\x79\x73\x71\x6c\x2d\x62\x69\x6e\x2e\x30\x30\x30\x30\x30\x33\x8c\x06\x00\x00\x00\x02\x72\x02\x92\xc3\xe8\xb7\xc9\x61\x00";
@@ -245,7 +248,7 @@ mod tests {
 }
 "#;
         let key_schema = Schema::parse_str(key_schema_str).unwrap();
-        let names: Vec<String> = avro_schema_to_column_descs(&key_schema)
+        let names: Vec<String> = avro_schema_to_column_descs(&key_schema, None)
             .unwrap()
             .drain(..)
             .map(|d| d.name)
@@ -301,7 +304,7 @@ mod tests {
 }
 "#;
         let schema = Schema::parse_str(test_schema_str).unwrap();
-        let columns = avro_schema_to_column_descs(&schema).unwrap();
+        let columns = avro_schema_to_column_descs(&schema, None).unwrap();
         for col in &columns {
             let dtype = col.column_type.as_ref().unwrap();
             println!("name = {}, type = {:?}", col.name, dtype.type_name);
@@ -319,6 +322,7 @@ mod tests {
                 avro_extract_field_schema(&outer_schema, Some("before")).unwrap(),
             )
             .unwrap(),
+            None,
         )
         .unwrap()
         .into_iter()
@@ -350,7 +354,7 @@ mod tests {
     #[ignore]
     #[tokio::test]
     async fn test_debezium_avro_parser() -> crate::error::ConnectorResult<()> {
-        let props = convert_args!(hashmap!(
+        let props = convert_args!(btreemap!(
             "kafka.topic" => "dbserver1.inventory.customers"
         ));
         let info = StreamSourceInfo {
diff --git a/src/connector/src/parser/debezium/debezium_parser.rs b/src/connector/src/parser/debezium/debezium_parser.rs
index 52d1e4e4a15a2..817c2a788f2be 100644
--- a/src/connector/src/parser/debezium/debezium_parser.rs
+++ b/src/connector/src/parser/debezium/debezium_parser.rs
@@ -198,6 +198,11 @@ mod tests {
     use std::sync::Arc;
 
     use risingwave_common::catalog::{ColumnCatalog, ColumnDesc, ColumnId};
+    use risingwave_common::row::Row;
+    use risingwave_common::types::Timestamptz;
+    use risingwave_pb::plan_common::{
+        additional_column, AdditionalColumn, AdditionalColumnTimestamp,
+    };
 
     use super::*;
     use crate::parser::{JsonProperties, SourceStreamChunkBuilder, TransactionControl};
@@ -266,4 +271,67 @@ mod tests {
             _ => panic!("unexpected parse result: {:?}", res),
         }
     }
+
+    #[tokio::test]
+    async fn test_parse_additional_columns() {
+        let columns = vec![
+            ColumnDesc::named("O_ORDERKEY", ColumnId::new(1), DataType::Int64),
+            ColumnDesc::named("O_CUSTKEY", ColumnId::new(2), DataType::Int64),
+            ColumnDesc::named("O_ORDERSTATUS", ColumnId::new(3), DataType::Varchar),
+            ColumnDesc::named("O_TOTALPRICE", ColumnId::new(4), DataType::Decimal),
+            ColumnDesc::named("O_ORDERDATE", ColumnId::new(5), DataType::Date),
+            ColumnDesc::named_with_additional_column(
+                "commit_ts",
+                ColumnId::new(6),
+                DataType::Timestamptz,
+                AdditionalColumn {
+                    column_type: Some(additional_column::ColumnType::Timestamp(
+                        AdditionalColumnTimestamp {},
+                    )),
+                },
+            ),
+        ];
+
+        let columns = columns
+            .iter()
+            .map(SourceColumnDesc::from)
+            .collect::<Vec<_>>();
+
+        let props = SpecificParserConfig {
+            key_encoding_config: None,
+            encoding_config: EncodingProperties::Json(JsonProperties {
+                use_schema_registry: false,
+                timestamptz_handling: None,
+            }),
+            protocol_config: ProtocolProperties::Debezium(DebeziumProps::default()),
+        };
+        let source_ctx = SourceContext {
+            connector_props: ConnectorProperties::PostgresCdc(Box::default()),
+            ..SourceContext::dummy()
+        };
+        let mut parser = DebeziumParser::new(props, columns.clone(), Arc::new(source_ctx))
+            .await
+            .unwrap();
+        let mut builder = SourceStreamChunkBuilder::with_capacity(columns, 0);
+
+        let payload = r#"{ "payload": { "before": null, "after": { "O_ORDERKEY": 5, "O_CUSTKEY": 44485, "O_ORDERSTATUS": "F", "O_TOTALPRICE": "144659.20", "O_ORDERDATE": "1994-07-30" }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002", "ts_ms": 1695277757000, "snapshot": "last", "db": "mydb", "sequence": null, "table": "orders_new", "server_id": 0, "gtid": null, "file": "binlog.000008", "pos": 3693, "row": 0, "thread": null, "query": null }, "op": "c", "ts_ms": 1695277757017, "transaction": null } }"#;
+
+        let res = parser
+            .parse_one_with_txn(
+                None,
+                Some(payload.as_bytes().to_vec()),
+                builder.row_writer(),
+            )
+            .await;
+        match res {
+            Ok(ParseResult::Rows) => {
+                let chunk = builder.finish();
+                for (_, row) in chunk.rows() {
+                    let commit_ts = row.datum_at(5).unwrap().into_timestamptz();
+                    assert_eq!(commit_ts, Timestamptz::from_millis(1695277757000).unwrap());
+                }
+            }
+            _ => panic!("unexpected parse result: {:?}", res),
+        }
+    }
 }
diff --git a/src/connector/src/parser/debezium/mod.rs b/src/connector/src/parser/debezium/mod.rs
index 2150af053ff2b..5b5416e647268 100644
--- a/src/connector/src/parser/debezium/mod.rs
+++ b/src/connector/src/parser/debezium/mod.rs
@@ -17,7 +17,6 @@
 mod avro_parser;
 mod debezium_parser;
 mod mongo_json_parser;
-mod operators;
 pub mod simd_json_parser;
 pub use avro_parser::*;
 pub use debezium_parser::*;
diff --git a/src/connector/src/parser/debezium/mongo_json_parser.rs b/src/connector/src/parser/debezium/mongo_json_parser.rs
index 2de46a9aa5460..4e3a679c31816 100644
--- a/src/connector/src/parser/debezium/mongo_json_parser.rs
+++ b/src/connector/src/parser/debezium/mongo_json_parser.rs
@@ -23,7 +23,7 @@ use crate::parser::simd_json_parser::DebeziumMongoJsonAccessBuilder;
 use crate::parser::unified::debezium::DebeziumChangeEvent;
 use crate::parser::unified::util::apply_row_operation_on_stream_chunk_writer;
 use crate::parser::{
-    AccessBuilderImpl, ByteStreamSourceParser, EncodingProperties, JsonProperties, ParserFormat,
+    AccessBuilderImpl, ByteStreamSourceParser, EncodingProperties, ParserFormat,
     SourceStreamChunkRowWriter,
 };
 use crate::source::{SourceColumnDesc, SourceContext, SourceContextRef};
@@ -31,8 +31,6 @@ use crate::source::{SourceColumnDesc, SourceContext, SourceContextRef};
 #[derive(Debug)]
 pub struct DebeziumMongoJsonParser {
     pub(crate) rw_columns: Vec<SourceColumnDesc>,
-    id_column: SourceColumnDesc,
-    payload_column: SourceColumnDesc,
     source_ctx: SourceContextRef,
     key_builder: AccessBuilderImpl,
     payload_builder: AccessBuilderImpl,
@@ -40,7 +38,7 @@ pub struct DebeziumMongoJsonParser {
 
 fn build_accessor_builder(config: EncodingProperties) -> anyhow::Result<AccessBuilderImpl> {
     match config {
-        EncodingProperties::MongoJson(_) => Ok(AccessBuilderImpl::DebeziumMongoJson(
+        EncodingProperties::MongoJson => Ok(AccessBuilderImpl::DebeziumMongoJson(
             DebeziumMongoJsonAccessBuilder::new()?,
         )),
         _ => bail!("unsupported encoding for DEBEZIUM_MONGO format"),
@@ -52,7 +50,7 @@ impl DebeziumMongoJsonParser {
         rw_columns: Vec<SourceColumnDesc>,
         source_ctx: SourceContextRef,
     ) -> ConnectorResult<Self> {
-        let id_column = rw_columns
+        let _id_column = rw_columns
             .iter()
             .find(|desc| {
                 desc.name == "_id"
@@ -65,27 +63,28 @@ impl DebeziumMongoJsonParser {
                     )
             })
             .context("Debezium Mongo needs a `_id` column with supported types (Varchar Jsonb int32 int64) in table")?.clone();
-        let payload_column = rw_columns
+        let _payload_column = rw_columns
             .iter()
             .find(|desc| desc.name == "payload" && matches!(desc.data_type, DataType::Jsonb))
             .context("Debezium Mongo needs a `payload` column with supported types Jsonb in table")?
             .clone();
 
         // _rw_{connector}_file/partition & _rw_{connector}_offset are created automatically.
-        if rw_columns.iter().filter(|desc| desc.is_visible()).count() != 2 {
-            bail!("Debezium Mongo needs no more columns except `_id` and `payload` in table");
+        if rw_columns
+            .iter()
+            .filter(|desc| desc.is_visible() && desc.additional_column.column_type.is_none())
+            .count()
+            != 2
+        {
+            bail!("Debezium Mongo needs no more data columns except `_id` and `payload` in table");
         }
 
         // encodings are fixed to MongoJson
-        let key_builder =
-            build_accessor_builder(EncodingProperties::MongoJson(JsonProperties::default()))?;
-        let payload_builder =
-            build_accessor_builder(EncodingProperties::MongoJson(JsonProperties::default()))?;
+        let key_builder = build_accessor_builder(EncodingProperties::MongoJson)?;
+        let payload_builder = build_accessor_builder(EncodingProperties::MongoJson)?;
 
         Ok(Self {
             rw_columns,
-            id_column,
-            payload_column,
             source_ctx,
             key_builder,
             payload_builder,
diff --git a/src/connector/src/parser/debezium/operators.rs b/src/connector/src/parser/debezium/operators.rs
deleted file mode 100644
index 8bb6230e9486f..0000000000000
--- a/src/connector/src/parser/debezium/operators.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-pub const DEBEZIUM_READ_OP: &str = "r";
-pub const DEBEZIUM_CREATE_OP: &str = "c";
-pub const DEBEZIUM_UPDATE_OP: &str = "u";
-pub const DEBEZIUM_DELETE_OP: &str = "d";
diff --git a/src/connector/src/parser/debezium/simd_json_parser.rs b/src/connector/src/parser/debezium/simd_json_parser.rs
index 63cb939d81238..85399ed772768 100644
--- a/src/connector/src/parser/debezium/simd_json_parser.rs
+++ b/src/connector/src/parser/debezium/simd_json_parser.rs
@@ -41,7 +41,7 @@ impl DebeziumJsonAccessBuilder {
 
 impl AccessBuilder for DebeziumJsonAccessBuilder {
     #[allow(clippy::unused_async)]
-    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_, '_>> {
+    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_>> {
         self.value = Some(payload);
         let mut event: BorrowedValue<'_> =
             simd_json::to_borrowed_value(self.value.as_mut().unwrap())
@@ -79,7 +79,7 @@ impl DebeziumMongoJsonAccessBuilder {
 
 impl AccessBuilder for DebeziumMongoJsonAccessBuilder {
     #[allow(clippy::unused_async)]
-    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_, '_>> {
+    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_>> {
         self.value = Some(payload);
         let mut event: BorrowedValue<'_> =
             simd_json::to_borrowed_value(self.value.as_mut().unwrap())
@@ -99,8 +99,6 @@ impl AccessBuilder for DebeziumMongoJsonAccessBuilder {
 
 #[cfg(test)]
 mod tests {
-    use std::convert::TryInto;
-
     use chrono::{NaiveDate, NaiveTime};
     use risingwave_common::array::{Op, StructValue};
     use risingwave_common::catalog::ColumnId;
@@ -532,7 +530,7 @@ mod tests {
                     // For other overflow, the parsing succeeds but the type conversion fails
                     // The errors are ignored and logged.
                     res.unwrap();
-                    assert!(logs_contain("Expected type"), "{i}");
+                    assert!(logs_contain("expected type"), "{i}");
                 } else {
                     // For f64 overflow, the parsing fails
                     let e = res.unwrap_err();
diff --git a/src/connector/src/parser/json_parser.rs b/src/connector/src/parser/json_parser.rs
index f010b8e6b7df6..89f118eb1022f 100644
--- a/src/connector/src/parser/json_parser.rs
+++ b/src/connector/src/parser/json_parser.rs
@@ -12,29 +12,31 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+// Note on this file:
+//
+// There's no struct named `JsonParser` anymore since #13707. `ENCODE JSON` will be
+// dispatched to `PlainParser` or `UpsertParser` with `JsonAccessBuilder` instead.
+//
+// This file now only contains utilities and tests for JSON parsing. Also, to avoid
+// rely on the internal implementation and allow that to be changed, the tests use
+// `ByteStreamSourceParserImpl` to create a parser instance.
+
+use std::collections::BTreeMap;
 
 use anyhow::Context as _;
 use apache_avro::Schema;
-use itertools::{Either, Itertools};
 use jst::{convert_avro, Context};
-use risingwave_common::{bail, try_match_expand};
+use risingwave_connector_codec::decoder::avro::MapHandling;
 use risingwave_pb::plan_common::ColumnDesc;
 
-use super::avro::schema_resolver::ConfluentSchemaResolver;
 use super::util::{bytes_from_url, get_kafka_topic};
-use super::{EncodingProperties, JsonProperties, SchemaRegistryAuth, SpecificParserConfig};
+use super::{JsonProperties, SchemaRegistryAuth};
 use crate::error::ConnectorResult;
-use crate::only_parse_payload;
 use crate::parser::avro::util::avro_schema_to_column_descs;
 use crate::parser::unified::json::{JsonAccess, JsonParseOptions};
-use crate::parser::unified::util::apply_row_accessor_on_stream_chunk_writer;
 use crate::parser::unified::AccessImpl;
-use crate::parser::{
-    AccessBuilder, ByteStreamSourceParser, ParserFormat, SourceStreamChunkRowWriter,
-};
+use crate::parser::AccessBuilder;
 use crate::schema::schema_registry::{handle_sr_list, Client};
-use crate::source::{SourceColumnDesc, SourceContext, SourceContextRef};
 
 #[derive(Debug)]
 pub struct JsonAccessBuilder {
@@ -45,7 +47,8 @@ pub struct JsonAccessBuilder {
 
 impl AccessBuilder for JsonAccessBuilder {
     #[allow(clippy::unused_async)]
-    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_, '_>> {
+    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_>> {
+        // XXX: When will we enter this branch?
         if payload.is_empty() {
             self.value = Some("{}".into());
         } else {
@@ -78,126 +81,39 @@ impl JsonAccessBuilder {
     }
 }
 
-/// Parser for JSON format
-#[derive(Debug)]
-pub struct JsonParser {
-    rw_columns: Vec<SourceColumnDesc>,
-    source_ctx: SourceContextRef,
-    // If schema registry is used, the starting index of payload is 5.
-    payload_start_idx: usize,
-}
-
-impl JsonParser {
-    pub fn new(
-        props: SpecificParserConfig,
-        rw_columns: Vec<SourceColumnDesc>,
-        source_ctx: SourceContextRef,
-    ) -> ConnectorResult<Self> {
-        let json_config = try_match_expand!(props.encoding_config, EncodingProperties::Json)?;
-        let payload_start_idx = if json_config.use_schema_registry {
-            5
-        } else {
-            0
-        };
-        Ok(Self {
-            rw_columns,
-            source_ctx,
-            payload_start_idx,
-        })
-    }
-
-    #[cfg(test)]
-    pub fn new_for_test(rw_columns: Vec<SourceColumnDesc>) -> ConnectorResult<Self> {
-        Ok(Self {
-            rw_columns,
-            source_ctx: SourceContext::dummy().into(),
-            payload_start_idx: 0,
-        })
-    }
-
-    #[allow(clippy::unused_async)]
-    pub async fn parse_inner(
-        &self,
-        mut payload: Vec<u8>,
-        mut writer: SourceStreamChunkRowWriter<'_>,
-    ) -> ConnectorResult<()> {
-        let value = simd_json::to_borrowed_value(&mut payload[self.payload_start_idx..])
-            .context("failed to parse json payload")?;
-        let values = if let simd_json::BorrowedValue::Array(arr) = value {
-            Either::Left(arr.into_iter())
-        } else {
-            Either::Right(std::iter::once(value))
-        };
-
-        let mut errors = Vec::new();
-        for value in values {
-            let accessor = JsonAccess::new(value);
-            match apply_row_accessor_on_stream_chunk_writer(accessor, &mut writer) {
-                Ok(_) => {}
-                Err(err) => errors.push(err),
-            }
-        }
-
-        if errors.is_empty() {
-            Ok(())
-        } else {
-            // TODO(error-handling): multiple errors
-            bail!(
-                "failed to parse {} row(s) in a single json message: {}",
-                errors.len(),
-                errors.iter().format(", ")
-            );
-        }
-    }
-}
-
-pub async fn schema_to_columns(
+pub async fn fetch_json_schema_and_map_to_columns(
     schema_location: &str,
     schema_registry_auth: Option<SchemaRegistryAuth>,
-    props: &HashMap<String, String>,
+    props: &BTreeMap<String, String>,
 ) -> ConnectorResult<Vec<ColumnDesc>> {
     let url = handle_sr_list(schema_location)?;
     let json_schema = if let Some(schema_registry_auth) = schema_registry_auth {
         let client = Client::new(url, &schema_registry_auth)?;
         let topic = get_kafka_topic(props)?;
-        let resolver = ConfluentSchemaResolver::new(client);
-        let content = resolver
-            .get_raw_schema_by_subject_name(&format!("{}-value", topic))
-            .await?
-            .content;
-        serde_json::from_str(&content)?
+        let schema = client
+            .get_schema_by_subject(&format!("{}-value", topic))
+            .await?;
+        serde_json::from_str(&schema.content)?
     } else {
         let url = url.first().unwrap();
         let bytes = bytes_from_url(url, None).await?;
         serde_json::from_slice(&bytes)?
     };
-    let context = Context::default();
-    let avro_schema = convert_avro(&json_schema, context).to_string();
-    let schema = Schema::parse_str(&avro_schema).context("failed to parse avro schema")?;
-    avro_schema_to_column_descs(&schema)
+    json_schema_to_columns(&json_schema)
 }
 
-impl ByteStreamSourceParser for JsonParser {
-    fn columns(&self) -> &[SourceColumnDesc] {
-        &self.rw_columns
-    }
-
-    fn source_ctx(&self) -> &SourceContext {
-        &self.source_ctx
-    }
-
-    fn parser_format(&self) -> ParserFormat {
-        ParserFormat::Json
-    }
-
-    async fn parse_one<'a>(
-        &'a mut self,
-        _key: Option<Vec<u8>>,
-        payload: Option<Vec<u8>>,
-        writer: SourceStreamChunkRowWriter<'a>,
-    ) -> ConnectorResult<()> {
-        only_parse_payload!(self, payload, writer)
-    }
+/// FIXME: when the JSON schema is invalid, it will panic.
+///
+/// ## Notes on type conversion
+/// Map will be used when an object doesn't have `properties` but has `additionalProperties`.
+/// When an object has `properties` and `additionalProperties`, the latter will be ignored.
+/// <https://github.com/mozilla/jsonschema-transpiler/blob/fb715c7147ebd52427e0aea09b2bba2d539850b1/src/jsonschema.rs#L228-L280>
+///
+/// TODO: examine other stuff like `oneOf`, `patternProperties`, etc.
+fn json_schema_to_columns(json_schema: &serde_json::Value) -> ConnectorResult<Vec<ColumnDesc>> {
+    let avro_schema = convert_avro(json_schema, Context::default()).to_string();
+    let schema = Schema::parse_str(&avro_schema).context("failed to parse avro schema")?;
+    avro_schema_to_column_descs(&schema, Some(MapHandling::Jsonb)).map_err(Into::into)
 }
 
 #[cfg(test)]
@@ -213,13 +129,31 @@ mod tests {
     use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType;
     use risingwave_pb::plan_common::{AdditionalColumn, AdditionalColumnKey};
 
-    use super::JsonParser;
-    use crate::parser::upsert_parser::UpsertParser;
+    use crate::parser::test_utils::ByteStreamSourceParserImplTestExt as _;
     use crate::parser::{
-        EncodingProperties, JsonProperties, ProtocolProperties, SourceColumnDesc,
-        SourceStreamChunkBuilder, SpecificParserConfig,
+        ByteStreamSourceParserImpl, CommonParserConfig, ParserConfig, ProtocolProperties,
+        SourceColumnDesc, SpecificParserConfig,
     };
-    use crate::source::{SourceColumnType, SourceContext};
+    use crate::source::SourceColumnType;
+
+    fn make_parser(rw_columns: Vec<SourceColumnDesc>) -> ByteStreamSourceParserImpl {
+        ByteStreamSourceParserImpl::create_for_test(ParserConfig {
+            common: CommonParserConfig { rw_columns },
+            specific: SpecificParserConfig::DEFAULT_PLAIN_JSON,
+        })
+        .unwrap()
+    }
+
+    fn make_upsert_parser(rw_columns: Vec<SourceColumnDesc>) -> ByteStreamSourceParserImpl {
+        ByteStreamSourceParserImpl::create_for_test(ParserConfig {
+            common: CommonParserConfig { rw_columns },
+            specific: SpecificParserConfig {
+                protocol_config: ProtocolProperties::Upsert,
+                ..SpecificParserConfig::DEFAULT_PLAIN_JSON
+            },
+        })
+        .unwrap()
+    }
 
     fn get_payload() -> Vec<Vec<u8>> {
         vec![
@@ -249,21 +183,8 @@ mod tests {
             SourceColumnDesc::simple("interval", DataType::Interval, 11.into()),
         ];
 
-        let parser = JsonParser::new(
-            SpecificParserConfig::DEFAULT_PLAIN_JSON,
-            descs.clone(),
-            SourceContext::dummy().into(),
-        )
-        .unwrap();
-
-        let mut builder = SourceStreamChunkBuilder::with_capacity(descs, 2);
-
-        for payload in get_payload() {
-            let writer = builder.row_writer();
-            parser.parse_inner(payload, writer).await.unwrap();
-        }
-
-        let chunk = builder.finish();
+        let parser = make_parser(descs);
+        let chunk = parser.parse(get_payload()).await;
 
         let mut rows = chunk.rows();
 
@@ -359,38 +280,20 @@ mod tests {
             SourceColumnDesc::simple("v2", DataType::Int16, 1.into()),
             SourceColumnDesc::simple("v3", DataType::Varchar, 2.into()),
         ];
-        let parser = JsonParser::new(
-            SpecificParserConfig::DEFAULT_PLAIN_JSON,
-            descs.clone(),
-            SourceContext::dummy().into(),
-        )
-        .unwrap();
-        let mut builder = SourceStreamChunkBuilder::with_capacity(descs, 3);
-
-        // Parse a correct record.
-        {
-            let writer = builder.row_writer();
-            let payload = br#"{"v1": 1, "v2": 2, "v3": "3"}"#.to_vec();
-            parser.parse_inner(payload, writer).await.unwrap();
-        }
 
-        // Parse an incorrect record.
-        {
-            let writer = builder.row_writer();
+        let parser = make_parser(descs);
+        let payloads = vec![
+            // Parse a correct record.
+            br#"{"v1": 1, "v2": 2, "v3": "3"}"#.to_vec(),
+            // Parse an incorrect record.
             // `v2` overflowed.
-            let payload = br#"{"v1": 1, "v2": 65536, "v3": "3"}"#.to_vec();
             // ignored the error, and fill None at v2.
-            parser.parse_inner(payload, writer).await.unwrap();
-        }
-
-        // Parse a correct record.
-        {
-            let writer = builder.row_writer();
-            let payload = br#"{"v1": 1, "v2": 2, "v3": "3"}"#.to_vec();
-            parser.parse_inner(payload, writer).await.unwrap();
-        }
+            br#"{"v1": 1, "v2": 65536, "v3": "3"}"#.to_vec(),
+            // Parse a correct record.
+            br#"{"v1": 1, "v2": 2, "v3": "3"}"#.to_vec(),
+        ];
+        let chunk = parser.parse(payloads).await;
 
-        let chunk = builder.finish();
         assert!(chunk.valid());
         assert_eq!(chunk.cardinality(), 3);
 
@@ -430,12 +333,7 @@ mod tests {
         .map(SourceColumnDesc::from)
         .collect_vec();
 
-        let parser = JsonParser::new(
-            SpecificParserConfig::DEFAULT_PLAIN_JSON,
-            descs.clone(),
-            SourceContext::dummy().into(),
-        )
-        .unwrap();
+        let parser = make_parser(descs);
         let payload = br#"
         {
             "data": {
@@ -454,12 +352,8 @@ mod tests {
             "VarcharCastToI64": "1598197865760800768"
         }
         "#.to_vec();
-        let mut builder = SourceStreamChunkBuilder::with_capacity(descs, 1);
-        {
-            let writer = builder.row_writer();
-            parser.parse_inner(payload, writer).await.unwrap();
-        }
-        let chunk = builder.finish();
+        let chunk = parser.parse(vec![payload]).await;
+
         let (op, row) = chunk.rows().next().unwrap();
         assert_eq!(op, Op::Insert);
         let row = row.into_owned_row().into_inner();
@@ -502,24 +396,15 @@ mod tests {
         .map(SourceColumnDesc::from)
         .collect_vec();
 
-        let parser = JsonParser::new(
-            SpecificParserConfig::DEFAULT_PLAIN_JSON,
-            descs.clone(),
-            SourceContext::dummy().into(),
-        )
-        .unwrap();
+        let parser = make_parser(descs);
         let payload = br#"
         {
             "struct": "{\"varchar\": \"varchar\", \"boolean\": true}"
         }
         "#
         .to_vec();
-        let mut builder = SourceStreamChunkBuilder::with_capacity(descs, 1);
-        {
-            let writer = builder.row_writer();
-            parser.parse_inner(payload, writer).await.unwrap();
-        }
-        let chunk = builder.finish();
+        let chunk = parser.parse(vec![payload]).await;
+
         let (op, row) = chunk.rows().next().unwrap();
         assert_eq!(op, Op::Insert);
         let row = row.into_owned_row().into_inner();
@@ -548,12 +433,7 @@ mod tests {
         .map(SourceColumnDesc::from)
         .collect_vec();
 
-        let parser = JsonParser::new(
-            SpecificParserConfig::DEFAULT_PLAIN_JSON,
-            descs.clone(),
-            SourceContext::dummy().into(),
-        )
-        .unwrap();
+        let parser = make_parser(descs);
         let payload = br#"
         {
             "struct": {
@@ -562,12 +442,8 @@ mod tests {
         }
         "#
         .to_vec();
-        let mut builder = SourceStreamChunkBuilder::with_capacity(descs, 1);
-        {
-            let writer = builder.row_writer();
-            parser.parse_inner(payload, writer).await.unwrap();
-        }
-        let chunk = builder.finish();
+        let chunk = parser.parse(vec![payload]).await;
+
         let (op, row) = chunk.rows().next().unwrap();
         assert_eq!(op, Op::Insert);
         let row = row.into_owned_row().into_inner();
@@ -589,7 +465,10 @@ mod tests {
             (r#"{"a":2}"#, r#"{"a":2,"b":2}"#),
             (r#"{"a":2}"#, r#""#),
         ]
-        .to_vec();
+        .into_iter()
+        .map(|(k, v)| (k.as_bytes().to_vec(), v.as_bytes().to_vec()))
+        .collect_vec();
+
         let key_column_desc = SourceColumnDesc {
             name: "rw_key".into(),
             data_type: DataType::Bytea,
@@ -607,34 +486,9 @@ mod tests {
             SourceColumnDesc::simple("b", DataType::Int32, 1.into()),
             key_column_desc,
         ];
-        let props = SpecificParserConfig {
-            key_encoding_config: None,
-            encoding_config: EncodingProperties::Json(JsonProperties {
-                use_schema_registry: false,
-                timestamptz_handling: None,
-            }),
-            protocol_config: ProtocolProperties::Upsert,
-        };
-        let mut parser = UpsertParser::new(props, descs.clone(), SourceContext::dummy().into())
-            .await
-            .unwrap();
-        let mut builder = SourceStreamChunkBuilder::with_capacity(descs, 4);
-        for item in items {
-            parser
-                .parse_inner(
-                    Some(item.0.as_bytes().to_vec()),
-                    if !item.1.is_empty() {
-                        Some(item.1.as_bytes().to_vec())
-                    } else {
-                        None
-                    },
-                    builder.row_writer(),
-                )
-                .await
-                .unwrap();
-        }
 
-        let chunk = builder.finish();
+        let parser = make_upsert_parser(descs);
+        let chunk = parser.parse_upsert(items).await;
 
         // expected chunk
         // +---+---+---+------------------+
diff --git a/src/connector/src/parser/maxwell/mod.rs b/src/connector/src/parser/maxwell/mod.rs
index bcbd0cc160e42..53b845260c421 100644
--- a/src/connector/src/parser/maxwell/mod.rs
+++ b/src/connector/src/parser/maxwell/mod.rs
@@ -13,6 +13,5 @@
 // limitations under the License.
 
 mod maxwell_parser;
-mod operators;
 pub use maxwell_parser::*;
 mod simd_json_parser;
diff --git a/src/connector/src/parser/maxwell/operators.rs b/src/connector/src/parser/maxwell/operators.rs
deleted file mode 100644
index 231399febc740..0000000000000
--- a/src/connector/src/parser/maxwell/operators.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-pub const MAXWELL_INSERT_OP: &str = "insert";
-pub const MAXWELL_UPDATE_OP: &str = "update";
-pub const MAXWELL_DELETE_OP: &str = "delete";
diff --git a/src/connector/src/parser/mod.rs b/src/connector/src/parser/mod.rs
index ddcbc3558b909..f7667a66a3747 100644
--- a/src/connector/src/parser/mod.rs
+++ b/src/connector/src/parser/mod.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::fmt::Debug;
 use std::sync::LazyLock;
 
@@ -30,9 +30,10 @@ use risingwave_common::bail;
 use risingwave_common::catalog::{KAFKA_TIMESTAMP_COLUMN_NAME, TABLE_NAME_COLUMN_NAME};
 use risingwave_common::log::LogSuppresser;
 use risingwave_common::metrics::GLOBAL_ERROR_METRICS;
-use risingwave_common::types::{Datum, Scalar, ScalarImpl};
+use risingwave_common::types::{Datum, DatumCow, DatumRef, ScalarRefImpl};
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::tracing::InstrumentStream;
+use risingwave_connector_codec::decoder::avro::MapHandling;
 use risingwave_pb::catalog::{
     SchemaRegistryNameStrategy as PbSchemaRegistryNameStrategy, StreamSourceInfo,
 };
@@ -45,7 +46,8 @@ pub use self::mysql::mysql_row_to_owned_row;
 use self::plain_parser::PlainParser;
 pub use self::postgres::postgres_row_to_owned_row;
 use self::simd_json_parser::DebeziumJsonAccessBuilder;
-pub use self::unified::json::TimestamptzHandling;
+pub use self::unified::json::{JsonAccess, TimestamptzHandling};
+pub use self::unified::Access;
 use self::unified::AccessImpl;
 use self::upsert_parser::UpsertParser;
 use self::util::get_kafka_topic;
@@ -54,7 +56,8 @@ use crate::error::{ConnectorError, ConnectorResult};
 use crate::parser::maxwell::MaxwellParser;
 use crate::parser::simd_json_parser::DebeziumMongoJsonAccessBuilder;
 use crate::parser::util::{
-    extract_header_inner_from_meta, extract_headers_from_meta, extreact_timestamp_from_meta,
+    extract_cdc_meta_column, extract_header_inner_from_meta, extract_headers_from_meta,
+    extreact_timestamp_from_meta,
 };
 use crate::schema::schema_registry::SchemaRegistryAuth;
 use crate::source::monitor::GLOBAL_SOURCE_METRICS;
@@ -75,7 +78,9 @@ mod maxwell;
 mod mysql;
 pub mod plain_parser;
 mod postgres;
+
 mod protobuf;
+pub mod scalar_adapter;
 mod unified;
 mod upsert_parser;
 mod util;
@@ -89,6 +94,7 @@ pub struct SourceStreamChunkBuilder {
     descs: Vec<SourceColumnDesc>,
     builders: Vec<ArrayBuilderImpl>,
     op_builder: Vec<Op>,
+    vis_builder: BitmapBuilder,
 }
 
 impl SourceStreamChunkBuilder {
@@ -102,6 +108,7 @@ impl SourceStreamChunkBuilder {
             descs,
             builders,
             op_builder: Vec::with_capacity(cap),
+            vis_builder: BitmapBuilder::with_capacity(cap),
         }
     }
 
@@ -110,18 +117,21 @@ impl SourceStreamChunkBuilder {
             descs: &self.descs,
             builders: &mut self.builders,
             op_builder: &mut self.op_builder,
+            vis_builder: &mut self.vis_builder,
+            visible: true, // write visible rows by default
             row_meta: None,
         }
     }
 
     /// Consumes the builder and returns a [`StreamChunk`].
     pub fn finish(self) -> StreamChunk {
-        StreamChunk::new(
+        StreamChunk::with_visibility(
             self.op_builder,
             self.builders
                 .into_iter()
                 .map(|builder| builder.finish().into())
                 .collect(),
+            self.vis_builder.finish(),
         )
     }
 
@@ -129,12 +139,12 @@ impl SourceStreamChunkBuilder {
     /// the builders of the next [`StreamChunk`].
     #[must_use]
     pub fn take(&mut self, next_cap: usize) -> StreamChunk {
-        let descs = std::mem::take(&mut self.descs);
+        let descs = std::mem::take(&mut self.descs); // we don't use `descs` in `finish`
         let builder = std::mem::replace(self, Self::with_capacity(descs, next_cap));
         builder.finish()
     }
 
-    pub fn op_num(&self) -> usize {
+    pub fn len(&self) -> usize {
         self.op_builder.len()
     }
 
@@ -143,63 +153,6 @@ impl SourceStreamChunkBuilder {
     }
 }
 
-/// A builder for building a [`StreamChunk`] that contains only heartbeat rows.
-/// Some connectors may emit heartbeat messages to the downstream, and the cdc source
-/// rely on the heartbeat messages to keep the source offset up-to-date with upstream.
-pub struct HeartbeatChunkBuilder {
-    builder: SourceStreamChunkBuilder,
-}
-
-impl HeartbeatChunkBuilder {
-    fn with_capacity(descs: Vec<SourceColumnDesc>, cap: usize) -> Self {
-        let builders = descs
-            .iter()
-            .map(|desc| desc.data_type.create_array_builder(cap))
-            .collect();
-
-        Self {
-            builder: SourceStreamChunkBuilder {
-                descs,
-                builders,
-                op_builder: Vec::with_capacity(cap),
-            },
-        }
-    }
-
-    fn row_writer(&mut self) -> SourceStreamChunkRowWriter<'_> {
-        self.builder.row_writer()
-    }
-
-    /// Consumes the builder and returns a [`StreamChunk`] with all rows marked as invisible
-    fn finish(self) -> StreamChunk {
-        // heartbeat chunk should be invisible
-        let builder = self.builder;
-        let visibility = BitmapBuilder::zeroed(builder.op_builder.len());
-        StreamChunk::with_visibility(
-            builder.op_builder,
-            builder
-                .builders
-                .into_iter()
-                .map(|builder| builder.finish().into())
-                .collect(),
-            visibility.finish(),
-        )
-    }
-
-    /// Resets the builder and returns a [`StreamChunk`], while reserving `next_cap` capacity for
-    /// the builders of the next [`StreamChunk`].
-    #[must_use]
-    fn take(&mut self, next_cap: usize) -> StreamChunk {
-        let descs = std::mem::take(&mut self.builder.descs);
-        let builder = std::mem::replace(self, Self::with_capacity(descs, next_cap));
-        builder.finish()
-    }
-
-    fn is_empty(&self) -> bool {
-        self.builder.is_empty()
-    }
-}
-
 /// `SourceStreamChunkRowWriter` is responsible to write one or more records to the [`StreamChunk`],
 /// where each contains either one row (Insert/Delete) or two rows (Update) that can be written atomically.
 ///
@@ -213,6 +166,10 @@ pub struct SourceStreamChunkRowWriter<'a> {
     descs: &'a [SourceColumnDesc],
     builders: &'a mut [ArrayBuilderImpl],
     op_builder: &'a mut Vec<Op>,
+    vis_builder: &'a mut BitmapBuilder,
+
+    /// Whether the rows written by this writer should be visible in output `StreamChunk`.
+    visible: bool,
 
     /// An optional meta data of the original message.
     ///
@@ -220,6 +177,22 @@ pub struct SourceStreamChunkRowWriter<'a> {
     row_meta: Option<MessageMeta<'a>>,
 }
 
+impl<'a> SourceStreamChunkRowWriter<'a> {
+    /// Set the meta data of the original message for this row writer.
+    ///
+    /// This should always be called except for tests.
+    fn with_meta(mut self, row_meta: MessageMeta<'a>) -> Self {
+        self.row_meta = Some(row_meta);
+        self
+    }
+
+    /// Convert the row writer to invisible row writer.
+    fn invisible(mut self) -> Self {
+        self.visible = false;
+        self
+    }
+}
+
 /// The meta data of the original message for a row writer.
 ///
 /// Extracted from the `SourceMessage`.
@@ -230,17 +203,17 @@ pub struct MessageMeta<'a> {
     offset: &'a str,
 }
 
-impl MessageMeta<'_> {
+impl<'a> MessageMeta<'a> {
     /// Extract the value for the given column.
     ///
     /// Returns `None` if the column is not a meta column.
-    fn value_for_column(self, desc: &SourceColumnDesc) -> Option<Datum> {
-        match desc.column_type {
+    fn value_for_column(self, desc: &SourceColumnDesc) -> Option<DatumRef<'a>> {
+        let datum: DatumRef<'_> = match desc.column_type {
             // Row id columns are filled with `NULL` here and will be filled with the real
             // row id generated by `RowIdGenExecutor` later.
-            SourceColumnType::RowId => Datum::None.into(),
+            SourceColumnType::RowId => None,
             // Extract the offset from the meta data.
-            SourceColumnType::Offset => Datum::Some(self.offset.into()).into(),
+            SourceColumnType::Offset => Some(self.offset.into()),
             // Extract custom meta data per connector.
             SourceColumnType::Meta if let SourceMeta::Kafka(kafka_meta) = self.meta => {
                 assert_eq!(
@@ -248,14 +221,11 @@ impl MessageMeta<'_> {
                     KAFKA_TIMESTAMP_COLUMN_NAME,
                     "unexpected kafka meta column name"
                 );
-                kafka_meta
-                    .timestamp
-                    .map(|ts| {
-                        risingwave_common::cast::i64_to_timestamptz(ts)
-                            .unwrap()
-                            .to_scalar_value()
-                    })
-                    .into()
+                kafka_meta.timestamp.map(|ts| {
+                    risingwave_common::cast::i64_to_timestamptz(ts)
+                        .unwrap()
+                        .into()
+                })
             }
             SourceColumnType::Meta if let SourceMeta::DebeziumCdc(cdc_meta) = self.meta => {
                 assert_eq!(
@@ -263,21 +233,23 @@ impl MessageMeta<'_> {
                     TABLE_NAME_COLUMN_NAME,
                     "unexpected cdc meta column name"
                 );
-                Datum::Some(cdc_meta.full_table_name.as_str().into()).into()
+                Some(cdc_meta.full_table_name.as_str().into())
             }
 
             // For other cases, return `None`.
-            SourceColumnType::Meta | SourceColumnType::Normal => None,
-        }
+            SourceColumnType::Meta | SourceColumnType::Normal => return None,
+        };
+
+        Some(datum)
     }
 }
 
 trait OpAction {
-    type Output;
+    type Output<'a>;
 
-    fn output_for(datum: Datum) -> Self::Output;
+    fn output_for<'a>(datum: impl Into<DatumCow<'a>>) -> Self::Output<'a>;
 
-    fn apply(builder: &mut ArrayBuilderImpl, output: Self::Output);
+    fn apply(builder: &mut ArrayBuilderImpl, output: Self::Output<'_>);
 
     fn rollback(builder: &mut ArrayBuilderImpl);
 
@@ -287,16 +259,16 @@ trait OpAction {
 struct OpActionInsert;
 
 impl OpAction for OpActionInsert {
-    type Output = Datum;
+    type Output<'a> = DatumCow<'a>;
 
     #[inline(always)]
-    fn output_for(datum: Datum) -> Self::Output {
-        datum
+    fn output_for<'a>(datum: impl Into<DatumCow<'a>>) -> Self::Output<'a> {
+        datum.into()
     }
 
     #[inline(always)]
-    fn apply(builder: &mut ArrayBuilderImpl, output: Datum) {
-        builder.append(&output)
+    fn apply(builder: &mut ArrayBuilderImpl, output: DatumCow<'_>) {
+        builder.append(output)
     }
 
     #[inline(always)]
@@ -306,23 +278,23 @@ impl OpAction for OpActionInsert {
 
     #[inline(always)]
     fn finish(writer: &mut SourceStreamChunkRowWriter<'_>) {
-        writer.op_builder.push(Op::Insert);
+        writer.append_op(Op::Insert);
     }
 }
 
 struct OpActionDelete;
 
 impl OpAction for OpActionDelete {
-    type Output = Datum;
+    type Output<'a> = DatumCow<'a>;
 
     #[inline(always)]
-    fn output_for(datum: Datum) -> Self::Output {
-        datum
+    fn output_for<'a>(datum: impl Into<DatumCow<'a>>) -> Self::Output<'a> {
+        datum.into()
     }
 
     #[inline(always)]
-    fn apply(builder: &mut ArrayBuilderImpl, output: Datum) {
-        builder.append(&output)
+    fn apply(builder: &mut ArrayBuilderImpl, output: DatumCow<'_>) {
+        builder.append(output)
     }
 
     #[inline(always)]
@@ -332,24 +304,25 @@ impl OpAction for OpActionDelete {
 
     #[inline(always)]
     fn finish(writer: &mut SourceStreamChunkRowWriter<'_>) {
-        writer.op_builder.push(Op::Delete);
+        writer.append_op(Op::Delete);
     }
 }
 
 struct OpActionUpdate;
 
 impl OpAction for OpActionUpdate {
-    type Output = (Datum, Datum);
+    type Output<'a> = (DatumCow<'a>, DatumCow<'a>);
 
     #[inline(always)]
-    fn output_for(datum: Datum) -> Self::Output {
+    fn output_for<'a>(datum: impl Into<DatumCow<'a>>) -> Self::Output<'a> {
+        let datum = datum.into();
         (datum.clone(), datum)
     }
 
     #[inline(always)]
-    fn apply(builder: &mut ArrayBuilderImpl, output: (Datum, Datum)) {
-        builder.append(&output.0);
-        builder.append(&output.1);
+    fn apply(builder: &mut ArrayBuilderImpl, output: (DatumCow<'_>, DatumCow<'_>)) {
+        builder.append(output.0);
+        builder.append(output.1);
     }
 
     #[inline(always)]
@@ -360,28 +333,50 @@ impl OpAction for OpActionUpdate {
 
     #[inline(always)]
     fn finish(writer: &mut SourceStreamChunkRowWriter<'_>) {
-        writer.op_builder.push(Op::UpdateDelete);
-        writer.op_builder.push(Op::UpdateInsert);
+        writer.append_op(Op::UpdateDelete);
+        writer.append_op(Op::UpdateInsert);
     }
 }
 
-impl<'a> SourceStreamChunkRowWriter<'a> {
-    /// Set the meta data of the original message for this row writer.
-    ///
-    /// This should always be called except for tests.
-    fn with_meta(self, row_meta: MessageMeta<'a>) -> Self {
-        Self {
-            row_meta: Some(row_meta),
-            ..self
-        }
+impl SourceStreamChunkRowWriter<'_> {
+    fn append_op(&mut self, op: Op) {
+        self.op_builder.push(op);
+        self.vis_builder.append(self.visible);
     }
-}
 
-impl SourceStreamChunkRowWriter<'_> {
-    fn do_action<A: OpAction>(
-        &mut self,
-        mut f: impl FnMut(&SourceColumnDesc) -> AccessResult<A::Output>,
+    fn do_action<'a, A: OpAction>(
+        &'a mut self,
+        mut f: impl FnMut(&SourceColumnDesc) -> AccessResult<A::Output<'a>>,
     ) -> AccessResult<()> {
+        let mut parse_field = |desc: &SourceColumnDesc| {
+            match f(desc) {
+                Ok(output) => Ok(output),
+
+                // Throw error for failed access to primary key columns.
+                Err(e) if desc.is_pk => Err(e),
+                // Ignore error for other columns and fill in `NULL` instead.
+                Err(error) => {
+                    // TODO: figure out a way to fill in not-null default value if user specifies one
+                    // TODO: decide whether the error should not be ignored (e.g., even not a valid Debezium message)
+                    // TODO: not using tracing span to provide `split_id` and `offset` due to performance concern,
+                    //       see #13105
+                    static LOG_SUPPERSSER: LazyLock<LogSuppresser> =
+                        LazyLock::new(LogSuppresser::default);
+                    if let Ok(suppressed_count) = LOG_SUPPERSSER.check() {
+                        tracing::warn!(
+                            error = %error.as_report(),
+                            split_id = self.row_meta.as_ref().map(|m| m.split_id),
+                            offset = self.row_meta.as_ref().map(|m| m.offset),
+                            column = desc.name,
+                            suppressed_count,
+                            "failed to parse non-pk column, padding with `NULL`"
+                        );
+                    }
+                    Ok(A::output_for(Datum::None))
+                }
+            }
+        };
+
         let mut wrapped_f = |desc: &SourceColumnDesc| {
             match (&desc.column_type, &desc.additional_column.column_type) {
                 (&SourceColumnType::Offset | &SourceColumnType::RowId, _) => {
@@ -407,20 +402,45 @@ impl SourceStreamChunkRowWriter<'_> {
                             .unwrap(), // handled all match cases in internal match, unwrap is safe
                     ));
                 }
-                (_, &Some(AdditionalColumnType::Timestamp(_))) => {
-                    return Ok(A::output_for(
-                        self.row_meta
-                            .as_ref()
-                            .and_then(|ele| extreact_timestamp_from_meta(ele.meta))
-                            .unwrap_or(None),
-                    ))
+
+                (
+                    _, // for cdc tables
+                    &Some(ref col @ AdditionalColumnType::DatabaseName(_))
+                    | &Some(ref col @ AdditionalColumnType::TableName(_)),
+                ) => {
+                    match self.row_meta {
+                        Some(row_meta) => {
+                            if let SourceMeta::DebeziumCdc(cdc_meta) = row_meta.meta {
+                                Ok(A::output_for(extract_cdc_meta_column(
+                                    cdc_meta,
+                                    col,
+                                    desc.name.as_str(),
+                                )?))
+                            } else {
+                                Err(AccessError::Uncategorized {
+                                    message: "CDC metadata not found in the message".to_string(),
+                                })
+                            }
+                        }
+                        None => parse_field(desc), // parse from payload
+                    }
+                }
+                (_, &Some(AdditionalColumnType::Timestamp(_))) => match self.row_meta {
+                    Some(row_meta) => Ok(A::output_for(
+                        extreact_timestamp_from_meta(row_meta.meta).unwrap_or(None),
+                    )),
+                    None => parse_field(desc), // parse from payload
+                },
+                (_, &Some(AdditionalColumnType::CollectionName(_))) => {
+                    // collection name for `mongodb-cdc` should be parsed from the message payload
+                    parse_field(desc)
                 }
                 (_, &Some(AdditionalColumnType::Partition(_))) => {
                     // the meta info does not involve spec connector
                     return Ok(A::output_for(
                         self.row_meta
                             .as_ref()
-                            .map(|ele| ScalarImpl::Utf8(ele.split_id.to_string().into())),
+                            .map(|ele| ScalarRefImpl::Utf8(ele.split_id)),
                     ));
                 }
                 (_, &Some(AdditionalColumnType::Offset(_))) => {
@@ -428,7 +448,7 @@ impl SourceStreamChunkRowWriter<'_> {
                     return Ok(A::output_for(
                         self.row_meta
                             .as_ref()
-                            .map(|ele| ScalarImpl::Utf8(ele.offset.to_string().into())),
+                            .map(|ele| ScalarRefImpl::Utf8(ele.offset)),
                     ));
                 }
                 (_, &Some(AdditionalColumnType::HeaderInner(ref header_inner))) => {
@@ -442,7 +462,7 @@ impl SourceStreamChunkRowWriter<'_> {
                                     header_inner.data_type.as_ref(),
                                 )
                             })
-                            .unwrap_or(None),
+                            .unwrap_or(Datum::None.into()),
                     ))
                 }
                 (_, &Some(AdditionalColumnType::Headers(_))) => {
@@ -458,50 +478,25 @@ impl SourceStreamChunkRowWriter<'_> {
                     return Ok(A::output_for(
                         self.row_meta
                             .as_ref()
-                            .map(|ele| ScalarImpl::Utf8(ele.split_id.to_string().into())),
+                            .map(|ele| ScalarRefImpl::Utf8(ele.split_id)),
                     ));
                 }
                 (_, _) => {
                     // For normal columns, call the user provided closure.
-                    match f(desc) {
-                        Ok(output) => Ok(output),
-
-                        // Throw error for failed access to primary key columns.
-                        Err(e) if desc.is_pk => Err(e),
-                        // Ignore error for other columns and fill in `NULL` instead.
-                        Err(error) => {
-                            // TODO: figure out a way to fill in not-null default value if user specifies one
-                            // TODO: decide whether the error should not be ignored (e.g., even not a valid Debezium message)
-                            // TODO: not using tracing span to provide `split_id` and `offset` due to performance concern,
-                            //       see #13105
-                            static LOG_SUPPERSSER: LazyLock<LogSuppresser> =
-                                LazyLock::new(LogSuppresser::default);
-                            if let Ok(suppressed_count) = LOG_SUPPERSSER.check() {
-                                tracing::warn!(
-                                    error = %error.as_report(),
-                                    split_id = self.row_meta.as_ref().map(|m| m.split_id),
-                                    offset = self.row_meta.as_ref().map(|m| m.offset),
-                                    column = desc.name,
-                                    suppressed_count,
-                                    "failed to parse non-pk column, padding with `NULL`"
-                                );
-                            }
-                            Ok(A::output_for(Datum::None))
-                        }
-                    }
+                    parse_field(desc)
                 }
             }
         };
 
         // Columns that changes have been applied to. Used to rollback when an error occurs.
-        let mut applied_columns = Vec::with_capacity(self.descs.len());
+        let mut applied_columns = 0;
 
         let result = (self.descs.iter())
             .zip_eq_fast(self.builders.iter_mut())
             .try_for_each(|(desc, builder)| {
                 wrapped_f(desc).map(|output| {
                     A::apply(builder, output);
-                    applied_columns.push(builder);
+                    applied_columns += 1;
                 })
             });
 
@@ -511,8 +506,8 @@ impl SourceStreamChunkRowWriter<'_> {
                 Ok(())
             }
             Err(e) => {
-                for builder in applied_columns {
-                    A::rollback(builder);
+                for i in 0..applied_columns {
+                    A::rollback(&mut self.builders[i]);
                 }
                 Err(e)
             }
@@ -523,33 +518,46 @@ impl SourceStreamChunkRowWriter<'_> {
     /// produces one [`Datum`] by corresponding [`SourceColumnDesc`].
     ///
     /// See the [struct-level documentation](SourceStreamChunkRowWriter) for more details.
-    pub fn insert(
+    #[inline(always)]
+    pub fn do_insert<'a, D>(
         &mut self,
-        f: impl FnMut(&SourceColumnDesc) -> AccessResult<Datum>,
-    ) -> AccessResult<()> {
-        self.do_action::<OpActionInsert>(f)
+        mut f: impl FnMut(&SourceColumnDesc) -> AccessResult<D>,
+    ) -> AccessResult<()>
+    where
+        D: Into<DatumCow<'a>>,
+    {
+        self.do_action::<OpActionInsert>(|desc| f(desc).map(Into::into))
     }
 
     /// Write a `Delete` record to the [`StreamChunk`], with the given fallible closure that
     /// produces one [`Datum`] by corresponding [`SourceColumnDesc`].
     ///
     /// See the [struct-level documentation](SourceStreamChunkRowWriter) for more details.
-    pub fn delete(
+    #[inline(always)]
+    pub fn do_delete<'a, D>(
         &mut self,
-        f: impl FnMut(&SourceColumnDesc) -> AccessResult<Datum>,
-    ) -> AccessResult<()> {
-        self.do_action::<OpActionDelete>(f)
+        mut f: impl FnMut(&SourceColumnDesc) -> AccessResult<D>,
+    ) -> AccessResult<()>
+    where
+        D: Into<DatumCow<'a>>,
+    {
+        self.do_action::<OpActionDelete>(|desc| f(desc).map(Into::into))
     }
 
     /// Write a `Update` record to the [`StreamChunk`], with the given fallible closure that
     /// produces two [`Datum`]s as old and new value by corresponding [`SourceColumnDesc`].
     ///
     /// See the [struct-level documentation](SourceStreamChunkRowWriter) for more details.
-    pub fn update(
+    #[inline(always)]
+    pub fn do_update<'a, D1, D2>(
         &mut self,
-        f: impl FnMut(&SourceColumnDesc) -> AccessResult<(Datum, Datum)>,
-    ) -> AccessResult<()> {
-        self.do_action::<OpActionUpdate>(f)
+        mut f: impl FnMut(&SourceColumnDesc) -> AccessResult<(D1, D2)>,
+    ) -> AccessResult<()>
+    where
+        D1: Into<DatumCow<'a>>,
+        D2: Into<DatumCow<'a>>,
+    {
+        self.do_action::<OpActionUpdate>(|desc| f(desc).map(|(old, new)| (old.into(), new.into())))
     }
 }
 
@@ -581,8 +589,10 @@ pub enum ParserFormat {
     Plain,
 }
 
-/// `ByteStreamSourceParser` is a new message parser, the parser should consume
-/// the input data stream and return a stream of parsed msgs.
+/// `ByteStreamSourceParser` is the entrypoint abstraction for parsing messages.
+/// It consumes bytes of one individual message and produces parsed records.
+///
+/// It's used by [`ByteStreamSourceParserImpl::into_stream`]. `pub` is for benchmark only.
 pub trait ByteStreamSourceParser: Send + Debug + Sized + 'static {
     /// The column descriptors of the output chunk.
     fn columns(&self) -> &[SourceColumnDesc];
@@ -621,7 +631,7 @@ pub trait ByteStreamSourceParser: Send + Debug + Sized + 'static {
     }
 
     fn emit_empty_row<'a>(&'a mut self, mut writer: SourceStreamChunkRowWriter<'a>) {
-        _ = writer.insert(|_column| Ok(None));
+        _ = writer.do_insert(|_column| Ok(Datum::None));
     }
 }
 
@@ -664,7 +674,7 @@ impl<P: ByteStreamSourceParser> P {
 
         // The parser stream will be long-lived. We use `instrument_with` here to create
         // a new span for the polling of each chunk.
-        into_chunk_stream(self, data_stream)
+        into_chunk_stream_inner(self, data_stream)
             .instrument_with(move || tracing::info_span!("source_parse_chunk", actor_id, source_id))
     }
 }
@@ -676,10 +686,13 @@ const MAX_ROWS_FOR_TRANSACTION: usize = 4096;
 // TODO: when upsert is disabled, how to filter those empty payload
 // Currently, an err is returned for non upsert with empty payload
 #[try_stream(ok = StreamChunk, error = crate::error::ConnectorError)]
-async fn into_chunk_stream<P: ByteStreamSourceParser>(mut parser: P, data_stream: BoxSourceStream) {
+async fn into_chunk_stream_inner<P: ByteStreamSourceParser>(
+    mut parser: P,
+    data_stream: BoxSourceStream,
+) {
     let columns = parser.columns().to_vec();
 
-    let mut heartbeat_builder = HeartbeatChunkBuilder::with_capacity(columns.clone(), 0);
+    let mut heartbeat_builder = SourceStreamChunkBuilder::with_capacity(columns.clone(), 0);
     let mut builder = SourceStreamChunkBuilder::with_capacity(columns, 0);
 
     struct Transaction {
@@ -687,13 +700,13 @@ async fn into_chunk_stream<P: ByteStreamSourceParser>(mut parser: P, data_stream
         len: usize,
     }
     let mut current_transaction = None;
-    let mut yield_asap = false; // whether we should yield the chunk as soon as possible (txn commits)
 
     #[for_await]
     for batch in data_stream {
         let batch = batch?;
         let batch_len = batch.len();
 
+        let mut last_batch_not_yielded = false;
         if let Some(Transaction { len, id }) = &mut current_transaction {
             // Dirty state. The last batch is not yielded due to uncommitted transaction.
             if *len > MAX_ROWS_FOR_TRANSACTION {
@@ -704,17 +717,13 @@ async fn into_chunk_stream<P: ByteStreamSourceParser>(mut parser: P, data_stream
                     "transaction is larger than {MAX_ROWS_FOR_TRANSACTION} rows, force commit"
                 );
                 *len = 0; // reset `len` while keeping `id`
-                yield_asap = false;
                 yield builder.take(batch_len);
             } else {
-                // Normal transaction. After the transaction is committed, we should yield the last
-                // batch immediately, so set `yield_asap` to true.
-                yield_asap = true;
+                last_batch_not_yielded = true
             }
         } else {
             // Clean state. Reserve capacity for the builder.
             assert!(builder.is_empty());
-            assert!(!yield_asap);
             let _ = builder.take(batch_len);
         }
 
@@ -725,11 +734,14 @@ async fn into_chunk_stream<P: ByteStreamSourceParser>(mut parser: P, data_stream
                     offset = msg.offset,
                     "got a empty message, could be a heartbeat"
                 );
-                parser.emit_empty_row(heartbeat_builder.row_writer().with_meta(MessageMeta {
-                    meta: &msg.meta,
-                    split_id: &msg.split_id,
-                    offset: &msg.offset,
-                }));
+                // Emit an empty invisible row for the heartbeat message.
+                parser.emit_empty_row(heartbeat_builder.row_writer().invisible().with_meta(
+                    MessageMeta {
+                        meta: &msg.meta,
+                        split_id: &msg.split_id,
+                        offset: &msg.offset,
+                    },
+                ));
                 continue;
             }
 
@@ -743,7 +755,7 @@ async fn into_chunk_stream<P: ByteStreamSourceParser>(mut parser: P, data_stream
                     .observe(lag_ms as f64);
             }
 
-            let old_op_num = builder.op_num();
+            let old_len = builder.len();
             match parser
                 .parse_one_with_txn(
                     msg.key,
@@ -759,12 +771,10 @@ async fn into_chunk_stream<P: ByteStreamSourceParser>(mut parser: P, data_stream
                 // It's possible that parsing multiple rows in a single message PARTIALLY failed.
                 // We still have to maintain the row number in this case.
                 res @ (Ok(ParseResult::Rows) | Err(_)) => {
-                    // The number of rows added to the builder.
-                    let num = builder.op_num() - old_op_num;
-
-                    // Aggregate the number of rows in the current transaction.
+                    // Aggregate the number of new rows into the current transaction.
                     if let Some(Transaction { len, .. }) = &mut current_transaction {
-                        *len += num;
+                        let n_new_rows = builder.len() - old_len;
+                        *len += n_new_rows;
                     }
 
                     if let Err(error) = res {
@@ -793,32 +803,28 @@ async fn into_chunk_stream<P: ByteStreamSourceParser>(mut parser: P, data_stream
                     }
                 }
 
-                Ok(ParseResult::TransactionControl(txn_ctl)) => {
-                    match txn_ctl {
-                        TransactionControl::Begin { id } => {
-                            if let Some(Transaction { id: current_id, .. }) = &current_transaction {
-                                tracing::warn!(current_id, id, "already in transaction");
-                            }
-                            tracing::debug!("begin upstream transaction: id={}", id);
-                            current_transaction = Some(Transaction { id, len: 0 });
-                        }
-                        TransactionControl::Commit { id } => {
-                            let current_id = current_transaction.as_ref().map(|t| &t.id);
-                            if current_id != Some(&id) {
-                                tracing::warn!(?current_id, id, "transaction id mismatch");
-                            }
-                            tracing::debug!("commit upstream transaction: id={}", id);
-                            current_transaction = None;
+                Ok(ParseResult::TransactionControl(txn_ctl)) => match txn_ctl {
+                    TransactionControl::Begin { id } => {
+                        if let Some(Transaction { id: current_id, .. }) = &current_transaction {
+                            tracing::warn!(current_id, id, "already in transaction");
                         }
+                        tracing::debug!(id, "begin upstream transaction");
+                        current_transaction = Some(Transaction { id, len: 0 });
                     }
+                    TransactionControl::Commit { id } => {
+                        let current_id = current_transaction.as_ref().map(|t| &t.id);
+                        if current_id != Some(&id) {
+                            tracing::warn!(?current_id, id, "transaction id mismatch");
+                        }
+                        tracing::debug!(id, "commit upstream transaction");
+                        current_transaction = None;
 
-                    // Not in a transaction anymore and `yield_asap` is set, so we should yield the
-                    // chunk now.
-                    if current_transaction.is_none() && yield_asap {
-                        yield_asap = false;
-                        yield builder.take(batch_len - (i + 1));
+                        if last_batch_not_yielded {
+                            yield builder.take(batch_len - (i + 1));
+                            last_batch_not_yielded = false;
+                        }
                     }
-                }
+                },
             }
         }
 
@@ -831,15 +837,13 @@ async fn into_chunk_stream<P: ByteStreamSourceParser>(mut parser: P, data_stream
 
         // If we are not in a transaction, we should yield the chunk now.
         if current_transaction.is_none() {
-            yield_asap = false;
-
             yield builder.take(0);
         }
     }
 }
 
 pub trait AccessBuilder {
-    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_, '_>>;
+    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_>>;
 }
 
 #[derive(Debug)]
@@ -884,10 +888,7 @@ impl AccessBuilderImpl {
         Ok(accessor)
     }
 
-    pub async fn generate_accessor(
-        &mut self,
-        payload: Vec<u8>,
-    ) -> ConnectorResult<AccessImpl<'_, '_>> {
+    pub async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_>> {
         let accessor = match self {
             Self::Avro(builder) => builder.generate_accessor(payload).await?,
             Self::Protobuf(builder) => builder.generate_accessor(payload).await?,
@@ -901,10 +902,11 @@ impl AccessBuilderImpl {
     }
 }
 
+/// The entrypoint of parsing. It parses [`SourceMessage`] stream (byte stream) into [`StreamChunk`] stream.
+/// Used by [`crate::source::into_chunk_stream`].
 #[derive(Debug)]
 pub enum ByteStreamSourceParserImpl {
     Csv(CsvParser),
-    Json(JsonParser),
     Debezium(DebeziumParser),
     Plain(PlainParser),
     Upsert(UpsertParser),
@@ -913,15 +915,12 @@ pub enum ByteStreamSourceParserImpl {
     CanalJson(CanalJsonParser),
 }
 
-pub type ParsedStreamImpl = impl ChunkSourceStream + Unpin;
-
 impl ByteStreamSourceParserImpl {
-    /// Converts this `SourceMessage` stream into a stream of [`StreamChunk`].
-    pub fn into_stream(self, msg_stream: BoxSourceStream) -> ParsedStreamImpl {
+    /// Converts [`SourceMessage`] stream into [`StreamChunk`] stream.
+    pub fn into_stream(self, msg_stream: BoxSourceStream) -> impl ChunkSourceStream + Unpin {
         #[auto_enum(futures03::Stream)]
         let stream = match self {
             Self::Csv(parser) => parser.into_stream(msg_stream),
-            Self::Json(parser) => parser.into_stream(msg_stream),
             Self::Debezium(parser) => parser.into_stream(msg_stream),
             Self::DebeziumMongoJson(parser) => parser.into_stream(msg_stream),
             Self::Maxwell(parser) => parser.into_stream(msg_stream),
@@ -975,6 +974,58 @@ impl ByteStreamSourceParserImpl {
             _ => unreachable!(),
         }
     }
+
+    /// Create a parser for testing purposes.
+    pub fn create_for_test(parser_config: ParserConfig) -> ConnectorResult<Self> {
+        futures::executor::block_on(Self::create(parser_config, SourceContext::dummy().into()))
+    }
+}
+
+/// Test utilities for [`ByteStreamSourceParserImpl`].
+#[cfg(test)]
+pub mod test_utils {
+    use futures::StreamExt as _;
+    use itertools::Itertools as _;
+
+    use super::*;
+
+    #[easy_ext::ext(ByteStreamSourceParserImplTestExt)]
+    pub(crate) impl ByteStreamSourceParserImpl {
+        /// Parse the given payloads into a [`StreamChunk`].
+        async fn parse(self, payloads: Vec<Vec<u8>>) -> StreamChunk {
+            let source_messages = payloads
+                .into_iter()
+                .map(|p| SourceMessage {
+                    payload: (!p.is_empty()).then_some(p),
+                    ..SourceMessage::dummy()
+                })
+                .collect_vec();
+
+            self.into_stream(futures::stream::once(async { Ok(source_messages) }).boxed())
+                .next()
+                .await
+                .unwrap()
+                .unwrap()
+        }
+
+        /// Parse the given key-value pairs into a [`StreamChunk`].
+        async fn parse_upsert(self, kvs: Vec<(Vec<u8>, Vec<u8>)>) -> StreamChunk {
+            let source_messages = kvs
+                .into_iter()
+                .map(|(k, v)| SourceMessage {
+                    key: (!k.is_empty()).then_some(k),
+                    payload: (!v.is_empty()).then_some(v),
+                    ..SourceMessage::dummy()
+                })
+                .collect_vec();
+
+            self.into_stream(futures::stream::once(async { Ok(source_messages) }).boxed())
+                .next()
+                .await
+                .unwrap()
+                .unwrap()
+        }
+    }
 }
 
 #[derive(Debug, Clone, Default)]
@@ -1024,6 +1075,7 @@ pub struct AvroProperties {
     pub record_name: Option<String>,
     pub key_record_name: Option<String>,
     pub name_strategy: PbSchemaRegistryNameStrategy,
+    pub map_handling: Option<MapHandling>,
 }
 
 #[derive(Debug, Default, Clone)]
@@ -1062,7 +1114,7 @@ pub enum EncodingProperties {
     Protobuf(ProtobufProperties),
     Csv(CsvProperties),
     Json(JsonProperties),
-    MongoJson(JsonProperties),
+    MongoJson,
     Bytes(BytesProperties),
     Native,
     /// Encoding can't be specified because the source will determines it. Now only used in Iceberg.
@@ -1090,7 +1142,7 @@ impl SpecificParserConfig {
     // The validity of (format, encode) is ensured by `extract_format_encode`
     pub fn new(
         info: &StreamSourceInfo,
-        with_properties: &HashMap<String, String>,
+        with_properties: &BTreeMap<String, String>,
     ) -> ConnectorResult<Self> {
         let source_struct = extract_source_struct(info)?;
         let format = source_struct.format;
@@ -1130,6 +1182,7 @@ impl SpecificParserConfig {
                         .unwrap(),
                     use_schema_registry: info.use_schema_registry,
                     row_schema_location: info.row_schema_location.clone(),
+                    map_handling: MapHandling::from_options(&info.format_encode_options)?,
                     ..Default::default()
                 };
                 if format == SourceFormat::Upsert {
diff --git a/src/connector/src/parser/plain_parser.rs b/src/connector/src/parser/plain_parser.rs
index c526366905938..f34fd29837174 100644
--- a/src/connector/src/parser/plain_parser.rs
+++ b/src/connector/src/parser/plain_parser.rs
@@ -15,6 +15,7 @@
 use risingwave_common::bail;
 
 use super::unified::json::TimestamptzHandling;
+use super::unified::kv_event::KvEvent;
 use super::{
     AccessBuilderImpl, ByteStreamSourceParser, EncodingProperties, EncodingType,
     SourceStreamChunkRowWriter, SpecificParserConfig,
@@ -23,13 +24,12 @@ use crate::error::ConnectorResult;
 use crate::parser::bytes_parser::BytesAccessBuilder;
 use crate::parser::simd_json_parser::DebeziumJsonAccessBuilder;
 use crate::parser::unified::debezium::parse_transaction_meta;
-use crate::parser::unified::upsert::UpsertChangeEvent;
-use crate::parser::unified::util::apply_row_operation_on_stream_chunk_writer_with_op;
-use crate::parser::unified::{AccessImpl, ChangeEventOperation};
+use crate::parser::unified::AccessImpl;
 use crate::parser::upsert_parser::get_key_column_name;
 use crate::parser::{BytesProperties, ParseResult, ParserFormat};
 use crate::source::{SourceColumnDesc, SourceContext, SourceContextRef, SourceMeta};
 
+/// Parser for `FORMAT PLAIN`, i.e., append-only source.
 #[derive(Debug)]
 pub struct PlainParser {
     pub key_builder: Option<AccessBuilderImpl>,
@@ -102,30 +102,22 @@ impl PlainParser {
             };
         }
 
-        // reuse upsert component but always insert
-        let mut row_op: UpsertChangeEvent<AccessImpl<'_, '_>, AccessImpl<'_, '_>> =
-            UpsertChangeEvent::default();
-        let change_event_op = ChangeEventOperation::Upsert;
+        let mut row_op: KvEvent<AccessImpl<'_>, AccessImpl<'_>> = KvEvent::default();
 
         if let Some(data) = key
             && let Some(key_builder) = self.key_builder.as_mut()
         {
             // key is optional in format plain
-            row_op = row_op.with_key(key_builder.generate_accessor(data).await?);
+            row_op.with_key(key_builder.generate_accessor(data).await?);
         }
         if let Some(data) = payload {
             // the data part also can be an empty vec
-            row_op = row_op.with_value(self.payload_builder.generate_accessor(data).await?);
+            row_op.with_value(self.payload_builder.generate_accessor(data).await?);
         }
 
-        Ok(
-            apply_row_operation_on_stream_chunk_writer_with_op(
-                row_op,
-                &mut writer,
-                change_event_op,
-            )
-            .map(|_| ParseResult::Rows)?,
-        )
+        writer.do_insert(|column: &SourceColumnDesc| row_op.access_field(column))?;
+
+        Ok(ParseResult::Rows)
     }
 }
 
@@ -210,7 +202,7 @@ mod tests {
         let mut transactional = false;
         // for untransactional source, we expect emit a chunk for each message batch
         let message_stream = source_message_stream(transactional);
-        let chunk_stream = crate::parser::into_chunk_stream(parser, message_stream.boxed());
+        let chunk_stream = crate::parser::into_chunk_stream_inner(parser, message_stream.boxed());
         let output: std::result::Result<Vec<_>, _> = block_on(chunk_stream.collect::<Vec<_>>())
             .into_iter()
             .collect();
@@ -247,7 +239,7 @@ mod tests {
         // for transactional source, we expect emit a single chunk for the transaction
         transactional = true;
         let message_stream = source_message_stream(transactional);
-        let chunk_stream = crate::parser::into_chunk_stream(parser, message_stream.boxed());
+        let chunk_stream = crate::parser::into_chunk_stream_inner(parser, message_stream.boxed());
         let output: std::result::Result<Vec<_>, _> = block_on(chunk_stream.collect::<Vec<_>>())
             .into_iter()
             .collect();
@@ -286,11 +278,11 @@ mod tests {
             if i == 0 {
                 // put begin message at first
                 source_msg_batch.push(SourceMessage {
-                    meta: SourceMeta::DebeziumCdc(DebeziumCdcMeta {
-                        full_table_name: "orders".to_string(),
-                        source_ts_ms: 0,
-                        is_transaction_meta: transactional,
-                    }),
+                    meta: SourceMeta::DebeziumCdc(DebeziumCdcMeta::new(
+                        "orders".to_string(),
+                        0,
+                        transactional,
+                    )),
                     split_id: SplitId::from("1001"),
                     offset: "0".into(),
                     key: None,
@@ -300,11 +292,11 @@ mod tests {
             // put data messages
             for data_msg in batch {
                 source_msg_batch.push(SourceMessage {
-                    meta: SourceMeta::DebeziumCdc(DebeziumCdcMeta {
-                        full_table_name: "orders".to_string(),
-                        source_ts_ms: 0,
-                        is_transaction_meta: false,
-                    }),
+                    meta: SourceMeta::DebeziumCdc(DebeziumCdcMeta::new(
+                        "orders".to_string(),
+                        0,
+                        false,
+                    )),
                     split_id: SplitId::from("1001"),
                     offset: "0".into(),
                     key: None,
@@ -314,11 +306,11 @@ mod tests {
             if i == data_batches.len() - 1 {
                 // put commit message at last
                 source_msg_batch.push(SourceMessage {
-                    meta: SourceMeta::DebeziumCdc(DebeziumCdcMeta {
-                        full_table_name: "orders".to_string(),
-                        source_ts_ms: 0,
-                        is_transaction_meta: transactional,
-                    }),
+                    meta: SourceMeta::DebeziumCdc(DebeziumCdcMeta::new(
+                        "orders".to_string(),
+                        0,
+                        transactional,
+                    )),
                     split_id: SplitId::from("1001"),
                     offset: "0".into(),
                     key: None,
@@ -363,11 +355,7 @@ mod tests {
         let begin_msg = r#"{"schema":null,"payload":{"status":"BEGIN","id":"3E11FA47-71CA-11E1-9E33-C80AA9429562:23","event_count":null,"data_collections":null,"ts_ms":1704269323180}}"#;
         let commit_msg = r#"{"schema":null,"payload":{"status":"END","id":"3E11FA47-71CA-11E1-9E33-C80AA9429562:23","event_count":11,"data_collections":[{"data_collection":"public.orders_tx","event_count":5},{"data_collection":"public.person","event_count":6}],"ts_ms":1704269323180}}"#;
 
-        let cdc_meta = SourceMeta::DebeziumCdc(DebeziumCdcMeta {
-            full_table_name: "orders".to_string(),
-            source_ts_ms: 0,
-            is_transaction_meta: true,
-        });
+        let cdc_meta = SourceMeta::DebeziumCdc(DebeziumCdcMeta::new("orders".to_string(), 0, true));
         let msg_meta = MessageMeta {
             meta: &cdc_meta,
             split_id: "1001",
diff --git a/src/connector/src/parser/postgres.rs b/src/connector/src/parser/postgres.rs
index efeb27b1ee4f1..da17ea256ba3c 100644
--- a/src/connector/src/parser/postgres.rs
+++ b/src/connector/src/parser/postgres.rs
@@ -12,59 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::str::FromStr;
 use std::sync::LazyLock;
 
-use bytes::BytesMut;
-use chrono::{NaiveDate, Utc};
-use pg_bigdecimal::PgNumeric;
 use risingwave_common::catalog::Schema;
 use risingwave_common::log::LogSuppresser;
 use risingwave_common::row::OwnedRow;
-use risingwave_common::types::{
-    DataType, Date, Decimal, Int256, Interval, JsonbVal, ListValue, ScalarImpl, Time, Timestamp,
-    Timestamptz,
-};
-use rust_decimal::Decimal as RustDecimal;
+use risingwave_common::types::{DataType, Decimal, ScalarImpl};
 use thiserror_ext::AsReport;
-use tokio_postgres::types::{to_sql_checked, FromSql, IsNull, Kind, ToSql, Type};
 
+use crate::parser::scalar_adapter::ScalarAdapter;
 use crate::parser::util::log_error;
 
 static LOG_SUPPERSSER: LazyLock<LogSuppresser> = LazyLock::new(LogSuppresser::default);
 
-macro_rules! handle_list_data_type {
-    ($row:expr, $i:expr, $name:expr, $type:ty, $builder:expr) => {
-        let res = $row.try_get::<_, Option<Vec<$type>>>($i);
-        match res {
-            Ok(val) => {
-                if let Some(v) = val {
-                    v.into_iter()
-                        .for_each(|val| $builder.append(Some(ScalarImpl::from(val))))
-                }
-            }
-            Err(err) => {
-                log_error!($name, err, "parse column failed");
-            }
-        }
-    };
-    ($row:expr, $i:expr, $name:expr, $type:ty, $builder:expr, $rw_type:ty) => {
-        let res = $row.try_get::<_, Option<Vec<$type>>>($i);
-        match res {
-            Ok(val) => {
-                if let Some(v) = val {
-                    v.into_iter().for_each(|val| {
-                        $builder.append(Some(ScalarImpl::from(<$rw_type>::from(val))))
-                    })
-                }
-            }
-            Err(err) => {
-                log_error!($name, err, "parse column failed");
-            }
-        }
-    };
-}
-
 macro_rules! handle_data_type {
     ($row:expr, $i:expr, $name:expr, $type:ty) => {{
         let res = $row.try_get::<_, Option<$type>>($i);
@@ -76,16 +36,6 @@ macro_rules! handle_data_type {
             }
         }
     }};
-    ($row:expr, $i:expr, $name:expr, $type:ty, $rw_type:ty) => {{
-        let res = $row.try_get::<_, Option<$type>>($i);
-        match res {
-            Ok(val) => val.map(|v| ScalarImpl::from(<$rw_type>::from(v))),
-            Err(err) => {
-                log_error!($name, err, "parse column failed");
-                None
-            }
-        }
-    }};
 }
 
 pub fn postgres_row_to_owned_row(row: tokio_postgres::Row, schema: &Schema) -> OwnedRow {
@@ -93,390 +43,83 @@ pub fn postgres_row_to_owned_row(row: tokio_postgres::Row, schema: &Schema) -> O
     for i in 0..schema.fields.len() {
         let rw_field = &schema.fields[i];
         let name = rw_field.name.as_str();
-        let datum = {
-            match &rw_field.data_type {
-                DataType::Boolean => {
-                    handle_data_type!(row, i, name, bool)
-                }
-                DataType::Int16 => {
-                    handle_data_type!(row, i, name, i16)
-                }
-                DataType::Int32 => {
-                    handle_data_type!(row, i, name, i32)
-                }
-                DataType::Int64 => {
-                    handle_data_type!(row, i, name, i64)
-                }
-                DataType::Float32 => {
-                    handle_data_type!(row, i, name, f32)
-                }
-                DataType::Float64 => {
-                    handle_data_type!(row, i, name, f64)
-                }
-                DataType::Decimal => {
-                    handle_data_type!(row, i, name, RustDecimal, Decimal)
-                }
-                DataType::Int256 => {
-                    // Currently in order to handle the decimal beyond RustDecimal,
-                    // we use the PgNumeric type to convert the decimal to a string.
-                    // Then we convert the string to Int256.
-                    // Note: It's only used to map the numeric type in upstream Postgres to RisingWave's rw_int256.
-                    let res = row.try_get::<_, Option<PgNumeric>>(i);
-                    match res {
-                        Ok(val) => pg_numeric_to_rw_int256(val, name),
-                        Err(err) => {
-                            log_error!(name, err, "parse numeric column as pg_numeric failed");
-                            None
-                        }
-                    }
-                }
-                DataType::Varchar => {
-                    if let Kind::Enum(_) = row.columns()[i].type_().kind() {
-                        // enum type needs to be handled separately
-                        let res = row.try_get::<_, Option<EnumString>>(i);
-                        match res {
-                            Ok(val) => val.map(|v| ScalarImpl::from(v.0)),
-                            Err(err) => {
-                                log_error!(name, err, "parse enum column failed");
-                                None
-                            }
-                        }
-                    } else {
-                        match *row.columns()[i].type_() {
-                            // Since we don't support UUID natively, adapt it to a VARCHAR column
-                            Type::UUID => {
-                                let res = row.try_get::<_, Option<uuid::Uuid>>(i);
-                                match res {
-                                    Ok(val) => val.map(|v| ScalarImpl::from(v.to_string())),
-                                    Err(err) => {
-                                        log_error!(name, err, "parse uuid column failed");
-                                        None
-                                    }
-                                }
-                            }
-                            // we support converting NUMERIC to VARCHAR implicitly
-                            Type::NUMERIC => {
-                                // Currently in order to handle the decimal beyond RustDecimal,
-                                // we use the PgNumeric type to convert the decimal to a string.
-                                // Note: It's only used to map the numeric type in upstream Postgres to RisingWave's varchar.
-                                let res = row.try_get::<_, Option<PgNumeric>>(i);
-                                match res {
-                                    Ok(val) => pg_numeric_to_varchar(val),
-                                    Err(err) => {
-                                        log_error!(
-                                            name,
-                                            err,
-                                            "parse numeric column as pg_numeric failed"
-                                        );
-                                        None
-                                    }
-                                }
-                            }
-                            _ => {
-                                handle_data_type!(row, i, name, String)
-                            }
-                        }
-                    }
-                }
-                DataType::Date => {
-                    handle_data_type!(row, i, name, NaiveDate, Date)
-                }
-                DataType::Time => {
-                    handle_data_type!(row, i, name, chrono::NaiveTime, Time)
-                }
-                DataType::Timestamp => {
-                    handle_data_type!(row, i, name, chrono::NaiveDateTime, Timestamp)
-                }
-                DataType::Timestamptz => {
-                    handle_data_type!(row, i, name, chrono::DateTime<Utc>, Timestamptz)
-                }
-                DataType::Bytea => {
-                    let res = row.try_get::<_, Option<Vec<u8>>>(i);
-                    match res {
-                        Ok(val) => val.map(|v| ScalarImpl::from(v.into_boxed_slice())),
-                        Err(err) => {
-                            log_error!(name, err, "parse column failed");
-                            None
-                        }
-                    }
-                }
-                DataType::Jsonb => {
-                    handle_data_type!(row, i, name, serde_json::Value, JsonbVal)
-                }
-                DataType::Interval => {
-                    handle_data_type!(row, i, name, Interval)
-                }
-                DataType::List(dtype) => {
-                    let mut builder = dtype.create_array_builder(0);
-                    // enum list needs to be handled separately
-                    if let Kind::Array(item_type) = row.columns()[i].type_().kind()
-                        && let Kind::Enum(_) = item_type.kind()
-                    {
-                        let res = row.try_get::<_, Option<Vec<EnumString>>>(i);
-                        match res {
-                            Ok(val) => {
-                                if let Some(v) = val {
-                                    v.into_iter().for_each(|val| {
-                                        builder.append(Some(ScalarImpl::from(val.0)))
-                                    });
-                                }
-                            }
-                            Err(err) => {
-                                log_error!(name, err, "parse enum column failed");
-                            }
-                        }
-                    } else {
-                        match **dtype {
-                            DataType::Boolean => {
-                                handle_list_data_type!(row, i, name, bool, builder);
-                            }
-                            DataType::Int16 => {
-                                handle_list_data_type!(row, i, name, i16, builder);
-                            }
-                            DataType::Int32 => {
-                                handle_list_data_type!(row, i, name, i32, builder);
-                            }
-                            DataType::Int64 => {
-                                handle_list_data_type!(row, i, name, i64, builder);
-                            }
-                            DataType::Float32 => {
-                                handle_list_data_type!(row, i, name, f32, builder);
-                            }
-                            DataType::Float64 => {
-                                handle_list_data_type!(row, i, name, f64, builder);
-                            }
-                            DataType::Decimal => {
-                                handle_list_data_type!(row, i, name, RustDecimal, builder, Decimal);
-                            }
-                            DataType::Date => {
-                                handle_list_data_type!(row, i, name, NaiveDate, builder, Date);
-                            }
-                            DataType::Varchar => {
-                                match *row.columns()[i].type_() {
-                                    // Since we don't support UUID natively, adapt it to a VARCHAR column
-                                    Type::UUID_ARRAY => {
-                                        let res = row.try_get::<_, Option<Vec<uuid::Uuid>>>(i);
-                                        match res {
-                                            Ok(val) => {
-                                                if let Some(v) = val {
-                                                    v.into_iter().for_each(|val| {
-                                                        builder.append(Some(ScalarImpl::from(
-                                                            val.to_string(),
-                                                        )))
-                                                    });
-                                                }
-                                            }
-                                            Err(err) => {
-                                                log_error!(name, err, "parse uuid column failed");
-                                            }
-                                        };
-                                    }
-                                    Type::NUMERIC_ARRAY => {
-                                        let res = row.try_get::<_, Option<Vec<PgNumeric>>>(i);
-                                        match res {
-                                            Ok(val) => {
-                                                if let Some(v) = val {
-                                                    v.into_iter().for_each(|val| {
-                                                        builder.append(pg_numeric_to_varchar(Some(
-                                                            val,
-                                                        )))
-                                                    });
-                                                }
-                                            }
-                                            Err(err) => {
-                                                log_error!(
-                                                    name,
-                                                    err,
-                                                    "parse numeric list column as pg_numeric list failed"
-                                                );
-                                            }
-                                        };
-                                    }
-                                    _ => {
-                                        handle_list_data_type!(row, i, name, String, builder);
-                                    }
-                                }
-                            }
-                            DataType::Time => {
-                                handle_list_data_type!(
-                                    row,
-                                    i,
-                                    name,
-                                    chrono::NaiveTime,
-                                    builder,
-                                    Time
-                                );
-                            }
-                            DataType::Timestamp => {
-                                handle_list_data_type!(
-                                    row,
-                                    i,
-                                    name,
-                                    chrono::NaiveDateTime,
-                                    builder,
-                                    Timestamp
-                                );
-                            }
-                            DataType::Timestamptz => {
-                                handle_list_data_type!(
-                                    row,
-                                    i,
-                                    name,
-                                    chrono::DateTime<Utc>,
-                                    builder,
-                                    Timestamptz
-                                );
-                            }
-                            DataType::Interval => {
-                                handle_list_data_type!(row, i, name, Interval, builder);
-                            }
-                            DataType::Jsonb => {
-                                handle_list_data_type!(
-                                    row,
-                                    i,
-                                    name,
-                                    serde_json::Value,
-                                    builder,
-                                    JsonbVal
-                                );
-                            }
-                            DataType::Bytea => {
-                                let res = row.try_get::<_, Option<Vec<Vec<u8>>>>(i);
-                                match res {
-                                    Ok(val) => {
-                                        if let Some(v) = val {
-                                            v.into_iter().for_each(|val| {
-                                                builder.append(Some(ScalarImpl::from(
-                                                    val.into_boxed_slice(),
-                                                )))
-                                            })
-                                        }
-                                    }
-                                    Err(err) => {
-                                        log_error!(name, err, "parse column failed");
-                                    }
-                                }
-                            }
-                            DataType::Int256 => {
-                                let res = row.try_get::<_, Option<Vec<PgNumeric>>>(i);
-                                match res {
-                                    Ok(val) => {
-                                        if let Some(v) = val {
-                                            v.into_iter().for_each(|val| {
-                                                builder.append(pg_numeric_to_rw_int256(
-                                                    Some(val),
-                                                    name,
-                                                ))
-                                            });
-                                        }
-                                    }
-                                    Err(err) => {
-                                        log_error!(
-                                            name,
-                                            err,
-                                            "parse numeric list column as pg_numeric list failed"
-                                        );
-                                    }
-                                };
-                            }
-                            DataType::Struct(_) | DataType::List(_) | DataType::Serial => {
-                                tracing::warn!(
-                                    "unsupported List data type {:?}, set the List to empty",
-                                    **dtype
-                                );
-                            }
-                        };
-                    }
-                    Some(ScalarImpl::from(ListValue::new(builder.finish())))
-                }
-                DataType::Struct(_) | DataType::Serial => {
-                    // Interval, Struct, List are not supported
-                    tracing::warn!(rw_field.name, ?rw_field.data_type, "unsupported data type, set to null");
-                    None
-                }
-            }
-        };
+        let datum = postgres_cell_to_scalar_impl(&row, &rw_field.data_type, i, name);
         datums.push(datum);
     }
     OwnedRow::new(datums)
 }
 
-fn pg_numeric_to_rw_int256(val: Option<PgNumeric>, name: &str) -> Option<ScalarImpl> {
-    let string = pg_numeric_to_string(val)?;
-    match Int256::from_str(string.as_str()) {
-        Ok(num) => Some(ScalarImpl::from(num)),
-        Err(err) => {
-            log_error!(name, err, "parse numeric string as rw_int256 failed");
-            None
+fn postgres_cell_to_scalar_impl(
+    row: &tokio_postgres::Row,
+    data_type: &DataType,
+    i: usize,
+    name: &str,
+) -> Option<ScalarImpl> {
+    // We observe several incompatibility issue in Debezium's Postgres connector. We summarize them here:
+    // Issue #1. The null of enum list is not supported in Debezium. An enum list contains `NULL` will fallback to `NULL`.
+    // Issue #2. In our parser, when there's inf, -inf, nan or invalid item in a list, the whole list will fallback null.
+    match data_type {
+        DataType::Boolean
+        | DataType::Int16
+        | DataType::Int32
+        | DataType::Int64
+        | DataType::Float32
+        | DataType::Float64
+        | DataType::Date
+        | DataType::Time
+        | DataType::Timestamp
+        | DataType::Timestamptz
+        | DataType::Jsonb
+        | DataType::Interval
+        | DataType::Bytea => {
+            // ScalarAdapter is also fine. But ScalarImpl is more efficient
+            let res = row.try_get::<_, Option<ScalarImpl>>(i);
+            match res {
+                Ok(val) => val,
+                Err(err) => {
+                    log_error!(name, err, "parse column failed");
+                    None
+                }
+            }
         }
-    }
-}
-
-fn pg_numeric_to_varchar(val: Option<PgNumeric>) -> Option<ScalarImpl> {
-    pg_numeric_to_string(val).map(ScalarImpl::from)
-}
-
-fn pg_numeric_to_string(val: Option<PgNumeric>) -> Option<String> {
-    if let Some(pg_numeric) = val {
-        // TODO(kexiang): NEGATIVE_INFINITY -> -Infinity, POSITIVE_INFINITY -> Infinity, NAN -> NaN
-        // The current implementation is to ensure consistency with the behavior of cdc event parsor.
-        match pg_numeric {
-            PgNumeric::NegativeInf => Some(String::from("NEGATIVE_INFINITY")),
-            PgNumeric::Normalized(big_decimal) => Some(big_decimal.to_string()),
-            PgNumeric::PositiveInf => Some(String::from("POSITIVE_INFINITY")),
-            PgNumeric::NaN => Some(String::from("NAN")),
+        DataType::Decimal => {
+            // Decimal is more efficient than PgNumeric in ScalarAdapter
+            handle_data_type!(row, i, name, Decimal)
         }
-    } else {
-        // NULL
-        None
-    }
-}
-
-#[derive(Clone, Debug)]
-struct EnumString(String);
-
-impl<'a> FromSql<'a> for EnumString {
-    fn from_sql(
-        _ty: &Type,
-        raw: &'a [u8],
-    ) -> Result<Self, Box<dyn std::error::Error + 'static + Sync + Send>> {
-        Ok(EnumString(String::from_utf8_lossy(raw).into_owned()))
-    }
-
-    fn accepts(ty: &Type) -> bool {
-        matches!(ty.kind(), Kind::Enum(_))
-    }
-}
-
-impl ToSql for EnumString {
-    to_sql_checked!();
-
-    fn accepts(ty: &Type) -> bool {
-        matches!(ty.kind(), Kind::Enum(_))
-    }
-
-    fn to_sql(
-        &self,
-        ty: &Type,
-        out: &mut BytesMut,
-    ) -> Result<IsNull, Box<dyn std::error::Error + Sync + Send>>
-    where
-        Self: Sized,
-    {
-        match ty.kind() {
-            Kind::Enum(e) => {
-                if e.contains(&self.0) {
-                    out.extend_from_slice(self.0.as_bytes());
-                    Ok(IsNull::No)
-                } else {
-                    Err(format!(
-                        "EnumString value {} is not in the enum type {:?}",
-                        self.0, e
-                    )
-                    .into())
+        DataType::Varchar | DataType::Int256 => {
+            let res = row.try_get::<_, Option<ScalarAdapter>>(i);
+            match res {
+                Ok(val) => val.and_then(|v| v.into_scalar(data_type)),
+                Err(err) => {
+                    log_error!(name, err, "parse column failed");
+                    None
                 }
             }
-            _ => Err("EnumString can only be used with ENUM types".into()),
+        }
+        DataType::List(dtype) => match **dtype {
+            // TODO(Kexiang): allow DataType::List(_)
+            DataType::Struct(_) | DataType::List(_) | DataType::Serial => {
+                tracing::warn!(
+                    "unsupported List data type {:?}, set the List to empty",
+                    **dtype
+                );
+                None
+            }
+            _ => {
+                let res = row.try_get::<_, Option<ScalarAdapter>>(i);
+                match res {
+                    Ok(val) => val.and_then(|v| v.into_scalar(data_type)),
+                    Err(err) => {
+                        log_error!(name, err, "parse list column failed");
+                        None
+                    }
+                }
+            }
+        },
+        DataType::Struct(_) | DataType::Serial => {
+            // Struct and Serial are not supported
+            tracing::warn!(name, ?data_type, "unsupported data type, set to null");
+            None
         }
     }
 }
@@ -485,7 +128,7 @@ impl ToSql for EnumString {
 mod tests {
     use tokio_postgres::NoTls;
 
-    use crate::parser::postgres::EnumString;
+    use crate::parser::scalar_adapter::EnumString;
     const DB: &str = "postgres";
     const USER: &str = "kexiang";
 
diff --git a/src/connector/src/parser/protobuf/parser.rs b/src/connector/src/parser/protobuf/parser.rs
index e9ae317ca5ebc..8be25074f6295 100644
--- a/src/connector/src/parser/protobuf/parser.rs
+++ b/src/connector/src/parser/protobuf/parser.rs
@@ -21,7 +21,10 @@ use prost_reflect::{
     MessageDescriptor, ReflectMessage, Value,
 };
 use risingwave_common::array::{ListValue, StructValue};
-use risingwave_common::types::{DataType, Datum, Decimal, JsonbVal, ScalarImpl, F32, F64};
+use risingwave_common::types::{
+    DataType, Datum, DatumCow, Decimal, JsonbRef, JsonbVal, ScalarImpl, ScalarRefImpl, ToDatumRef,
+    ToOwnedDatum, F32, F64,
+};
 use risingwave_common::{bail, try_match_expand};
 use risingwave_pb::plan_common::{AdditionalColumn, ColumnDesc, ColumnDescVersion};
 use thiserror::Error;
@@ -46,7 +49,7 @@ pub struct ProtobufAccessBuilder {
 
 impl AccessBuilder for ProtobufAccessBuilder {
     #[allow(clippy::unused_async)]
-    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_, '_>> {
+    async fn generate_accessor(&mut self, payload: Vec<u8>) -> ConnectorResult<AccessImpl<'_>> {
         let payload = if self.confluent_wire_type {
             resolve_pb_header(&payload)?
         } else {
@@ -344,14 +347,20 @@ fn recursive_parse_json(
     serde_json::Value::Object(ret)
 }
 
-pub fn from_protobuf_value(
+pub fn from_protobuf_value<'a>(
     field_desc: &FieldDescriptor,
-    value: &Value,
+    value: &'a Value,
     descriptor_pool: &Arc<DescriptorPool>,
-) -> AccessResult {
+) -> AccessResult<DatumCow<'a>> {
     let kind = field_desc.kind();
 
-    let v = match value {
+    macro_rules! borrowed {
+        ($v:expr) => {
+            return Ok(DatumCow::Borrowed(Some($v.into())))
+        };
+    }
+
+    let v: ScalarImpl = match value {
         Value::Bool(v) => ScalarImpl::Bool(*v),
         Value::I32(i) => ScalarImpl::Int32(*i),
         Value::U32(i) => ScalarImpl::Int64(*i as i64),
@@ -359,7 +368,7 @@ pub fn from_protobuf_value(
         Value::U64(i) => ScalarImpl::Decimal(Decimal::from(*i)),
         Value::F32(f) => ScalarImpl::Float32(F32::from(*f)),
         Value::F64(f) => ScalarImpl::Float64(F64::from(*f)),
-        Value::String(s) => ScalarImpl::Utf8(s.as_str().into()),
+        Value::String(s) => borrowed!(s.as_str()),
         Value::EnumNumber(idx) => {
             let enum_desc = kind.as_enum().ok_or_else(|| AccessError::TypeError {
                 expected: "enum".to_owned(),
@@ -375,9 +384,7 @@ pub fn from_protobuf_value(
             if dyn_msg.descriptor().full_name() == "google.protobuf.Any" {
                 // If the fields are not presented, default value is an empty string
                 if !dyn_msg.has_field_by_name("type_url") || !dyn_msg.has_field_by_name("value") {
-                    return Ok(Some(ScalarImpl::Jsonb(JsonbVal::from(
-                        serde_json::json! {""},
-                    ))));
+                    borrowed!(JsonbRef::empty_string());
                 }
 
                 // Sanity check
@@ -391,9 +398,8 @@ pub fn from_protobuf_value(
 
                 let payload_field_desc = dyn_msg.descriptor().get_field_by_name("value").unwrap();
 
-                let Some(ScalarImpl::Bytea(payload)) =
-                    from_protobuf_value(&payload_field_desc, &payload, descriptor_pool)?
-                else {
+                let payload = from_protobuf_value(&payload_field_desc, &payload, descriptor_pool)?;
+                let Some(ScalarRefImpl::Bytea(payload)) = payload.to_datum_ref() else {
                     bail_uncategorized!("expected bytes for dynamic message payload");
                 };
 
@@ -413,12 +419,13 @@ pub fn from_protobuf_value(
                 let full_name = msg_desc.clone().full_name().to_string();
 
                 // Decode the payload based on the `msg_desc`
-                let decoded_value = DynamicMessage::decode(msg_desc, payload.as_ref()).unwrap();
+                let decoded_value = DynamicMessage::decode(msg_desc, payload).unwrap();
                 let decoded_value = from_protobuf_value(
                     field_desc,
                     &Value::Message(decoded_value),
                     descriptor_pool,
                 )?
+                .to_owned_datum()
                 .unwrap();
 
                 // Extract the struct value
@@ -447,7 +454,9 @@ pub fn from_protobuf_value(
                     }
                     // use default value if dyn_msg doesn't has this field
                     let value = dyn_msg.get_field(&field_desc);
-                    rw_values.push(from_protobuf_value(&field_desc, &value, descriptor_pool)?);
+                    rw_values.push(
+                        from_protobuf_value(&field_desc, &value, descriptor_pool)?.to_owned_datum(),
+                    );
                 }
                 ScalarImpl::Struct(StructValue::new(rw_values))
             }
@@ -461,14 +470,14 @@ pub fn from_protobuf_value(
             }
             ScalarImpl::List(ListValue::new(builder.finish()))
         }
-        Value::Bytes(value) => ScalarImpl::Bytea(value.to_vec().into_boxed_slice()),
+        Value::Bytes(value) => borrowed!(&**value),
         _ => {
             return Err(AccessError::UnsupportedType {
                 ty: format!("{kind:?}"),
             });
         }
     };
-    Ok(Some(v))
+    Ok(Some(v).into())
 }
 
 /// Maps protobuf type to RW type.
@@ -579,11 +588,11 @@ pub(crate) fn resolve_pb_header(payload: &[u8]) -> ConnectorResult<&[u8]> {
 
 #[cfg(test)]
 mod test {
-    use std::collections::HashMap;
     use std::path::PathBuf;
 
     use prost::Message;
-    use risingwave_common::types::{DataType, ListValue, StructType};
+    use risingwave_common::types::StructType;
+    use risingwave_connector_codec::decoder::AccessExt;
     use risingwave_pb::catalog::StreamSourceInfo;
     use risingwave_pb::data::data_type::PbTypeName;
     use risingwave_pb::plan_common::{PbEncodeType, PbFormatType};
@@ -592,7 +601,6 @@ mod test {
     use super::*;
     use crate::parser::protobuf::recursive::all_types::{EnumType, ExampleOneof, NestedMessage};
     use crate::parser::protobuf::recursive::AllTypes;
-    use crate::parser::unified::Access;
     use crate::parser::SpecificParserConfig;
 
     fn schema_dir() -> String {
@@ -624,7 +632,7 @@ mod test {
             row_encode: PbEncodeType::Protobuf.into(),
             ..Default::default()
         };
-        let parser_config = SpecificParserConfig::new(&info, &HashMap::new())?;
+        let parser_config = SpecificParserConfig::new(&info, &Default::default())?;
         let conf = ProtobufParserConfig::new(parser_config.encoding_config).await?;
         let value = DynamicMessage::decode(conf.message_descriptor, PRE_GEN_PROTO_DATA).unwrap();
 
@@ -669,7 +677,7 @@ mod test {
             row_encode: PbEncodeType::Protobuf.into(),
             ..Default::default()
         };
-        let parser_config = SpecificParserConfig::new(&info, &HashMap::new())?;
+        let parser_config = SpecificParserConfig::new(&info, &Default::default())?;
         let conf = ProtobufParserConfig::new(parser_config.encoding_config).await?;
         let columns = conf.map_to_columns().unwrap();
 
@@ -718,7 +726,7 @@ mod test {
             row_encode: PbEncodeType::Protobuf.into(),
             ..Default::default()
         };
-        let parser_config = SpecificParserConfig::new(&info, &HashMap::new()).unwrap();
+        let parser_config = SpecificParserConfig::new(&info, &Default::default()).unwrap();
         let conf = ProtobufParserConfig::new(parser_config.encoding_config)
             .await
             .unwrap();
@@ -746,7 +754,7 @@ mod test {
             row_encode: PbEncodeType::Protobuf.into(),
             ..Default::default()
         };
-        let parser_config = SpecificParserConfig::new(&info, &HashMap::new()).unwrap();
+        let parser_config = SpecificParserConfig::new(&info, &Default::default()).unwrap();
 
         ProtobufParserConfig::new(parser_config.encoding_config)
             .await
@@ -896,7 +904,8 @@ mod test {
     }
 
     fn pb_eq(a: &ProtobufAccess, field_name: &str, value: ScalarImpl) {
-        let d = a.access(&[field_name], None).unwrap().unwrap();
+        let dummy_type = DataType::Varchar;
+        let d = a.access_owned(&[field_name], &dummy_type).unwrap().unwrap();
         assert_eq!(d, value, "field: {} value: {:?}", field_name, d);
     }
 
@@ -965,7 +974,9 @@ mod test {
         let field = value.fields().next().unwrap().0;
 
         if let Some(ret) =
-            from_protobuf_value(&field, &Value::Message(value), &conf.descriptor_pool).unwrap()
+            from_protobuf_value(&field, &Value::Message(value), &conf.descriptor_pool)
+                .unwrap()
+                .to_owned_datum()
         {
             println!("Decoded Value for ANY_GEN_PROTO_DATA: {:#?}", ret);
             println!("---------------------------");
@@ -1026,7 +1037,9 @@ mod test {
         let field = value.fields().next().unwrap().0;
 
         if let Some(ret) =
-            from_protobuf_value(&field, &Value::Message(value), &conf.descriptor_pool).unwrap()
+            from_protobuf_value(&field, &Value::Message(value), &conf.descriptor_pool)
+                .unwrap()
+                .to_owned_datum()
         {
             println!("Decoded Value for ANY_GEN_PROTO_DATA: {:#?}", ret);
             println!("---------------------------");
@@ -1098,7 +1111,9 @@ mod test {
         let field = value.fields().next().unwrap().0;
 
         if let Some(ret) =
-            from_protobuf_value(&field, &Value::Message(value), &conf.descriptor_pool).unwrap()
+            from_protobuf_value(&field, &Value::Message(value), &conf.descriptor_pool)
+                .unwrap()
+                .to_owned_datum()
         {
             println!("Decoded Value for ANY_RECURSIVE_GEN_PROTO_DATA: {:#?}", ret);
             println!("---------------------------");
diff --git a/src/connector/src/parser/scalar_adapter.rs b/src/connector/src/parser/scalar_adapter.rs
new file mode 100644
index 0000000000000..0f5d2d6d6d935
--- /dev/null
+++ b/src/connector/src/parser/scalar_adapter.rs
@@ -0,0 +1,379 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::str::FromStr;
+
+use anyhow::anyhow;
+use bytes::BytesMut;
+use pg_bigdecimal::PgNumeric;
+use risingwave_common::types::{DataType, Decimal, Int256, ListValue, ScalarImpl, ScalarRefImpl};
+use thiserror_ext::AsReport;
+use tokio_postgres::types::{to_sql_checked, FromSql, IsNull, Kind, ToSql, Type};
+
+use crate::error::ConnectorResult;
+
+#[derive(Clone, Debug)]
+pub struct EnumString(pub String);
+
+impl<'a> FromSql<'a> for EnumString {
+    fn from_sql(
+        _ty: &Type,
+        raw: &'a [u8],
+    ) -> Result<Self, Box<dyn std::error::Error + 'static + Sync + Send>> {
+        Ok(EnumString(String::from_utf8_lossy(raw).into_owned()))
+    }
+
+    fn accepts(ty: &Type) -> bool {
+        matches!(ty.kind(), Kind::Enum(_))
+    }
+}
+
+impl ToSql for EnumString {
+    to_sql_checked!();
+
+    fn to_sql(
+        &self,
+        ty: &Type,
+        out: &mut BytesMut,
+    ) -> Result<IsNull, Box<dyn std::error::Error + Sync + Send>>
+    where
+        Self: Sized,
+    {
+        match ty.kind() {
+            Kind::Enum(e) => {
+                if e.contains(&self.0) {
+                    out.extend_from_slice(self.0.as_bytes());
+                    Ok(IsNull::No)
+                } else {
+                    Err(format!(
+                        "EnumString value {} is not in the enum type {:?}",
+                        self.0, e
+                    )
+                    .into())
+                }
+            }
+            _ => Err("EnumString can only be used with ENUM types".into()),
+        }
+    }
+
+    fn accepts(ty: &Type) -> bool {
+        matches!(ty.kind(), Kind::Enum(_))
+    }
+}
+
+/// Adapter for `ScalarImpl` to Postgres data type,
+/// which can be used to encode/decode to/from Postgres value.
+#[derive(Debug)]
+pub(crate) enum ScalarAdapter {
+    Builtin(ScalarImpl),
+    Uuid(uuid::Uuid),
+    // Currently in order to handle the decimal beyond RustDecimal,
+    // we use the PgNumeric type to convert the decimal to a string/decimal/rw_int256.
+    Numeric(PgNumeric),
+    Enum(EnumString),
+    NumericList(Vec<Option<PgNumeric>>),
+    EnumList(Vec<Option<EnumString>>),
+    // UuidList is covered by List, while NumericList and EnumList are special cases.
+    // Note: The IntervalList is not supported.
+    List(Vec<Option<ScalarAdapter>>),
+}
+
+impl ToSql for ScalarAdapter {
+    to_sql_checked!();
+
+    fn to_sql(
+        &self,
+        ty: &Type,
+        out: &mut bytes::BytesMut,
+    ) -> Result<IsNull, Box<dyn std::error::Error + Sync + Send>> {
+        match self {
+            ScalarAdapter::Builtin(v) => v.to_sql(ty, out),
+            ScalarAdapter::Uuid(v) => v.to_sql(ty, out),
+            ScalarAdapter::Numeric(v) => v.to_sql(ty, out),
+            ScalarAdapter::Enum(v) => v.to_sql(ty, out),
+            ScalarAdapter::NumericList(v) => v.to_sql(ty, out),
+            ScalarAdapter::EnumList(v) => v.to_sql(ty, out),
+            ScalarAdapter::List(v) => v.to_sql(ty, out),
+        }
+    }
+
+    fn accepts(_ty: &Type) -> bool {
+        true
+    }
+}
+
+/// convert from Postgres uuid, numeric and enum to `ScalarAdapter`
+impl<'a> FromSql<'a> for ScalarAdapter {
+    fn from_sql(
+        ty: &Type,
+        raw: &'a [u8],
+    ) -> Result<Self, Box<dyn std::error::Error + Sync + Send>> {
+        match ty.kind() {
+            Kind::Simple => match *ty {
+                Type::UUID => Ok(ScalarAdapter::Uuid(uuid::Uuid::from_sql(ty, raw)?)),
+                // In order to cover the decimal beyond RustDecimal(only 28 digits are supported),
+                // we use the PgNumeric to handle decimal from postgres.
+                Type::NUMERIC => Ok(ScalarAdapter::Numeric(PgNumeric::from_sql(ty, raw)?)),
+                _ => Ok(ScalarAdapter::Builtin(ScalarImpl::from_sql(ty, raw)?)),
+            },
+            Kind::Enum(_) => Ok(ScalarAdapter::Enum(EnumString::from_sql(ty, raw)?)),
+            Kind::Array(Type::NUMERIC) => {
+                Ok(ScalarAdapter::NumericList(FromSql::from_sql(ty, raw)?))
+            }
+            Kind::Array(inner_type) if let Kind::Enum(_) = inner_type.kind() => {
+                Ok(ScalarAdapter::EnumList(FromSql::from_sql(ty, raw)?))
+            }
+            Kind::Array(_) => Ok(ScalarAdapter::List(FromSql::from_sql(ty, raw)?)),
+            _ => Err(anyhow!("failed to convert type {:?} to ScalarAdapter", ty).into()),
+        }
+    }
+
+    fn accepts(ty: &Type) -> bool {
+        match ty.kind() {
+            Kind::Simple => {
+                matches!(ty, &Type::UUID | &Type::NUMERIC) || <ScalarImpl as FromSql>::accepts(ty)
+            }
+            Kind::Enum(_) => true,
+            Kind::Array(inner_type) => <ScalarAdapter as FromSql>::accepts(inner_type),
+            _ => false,
+        }
+    }
+}
+
+impl ScalarAdapter {
+    pub fn name(&self) -> &'static str {
+        match self {
+            ScalarAdapter::Builtin(_) => "Builtin",
+            ScalarAdapter::Uuid(_) => "Uuid",
+            ScalarAdapter::Numeric(_) => "Numeric",
+            ScalarAdapter::Enum(_) => "Enum",
+            ScalarAdapter::EnumList(_) => "EnumList",
+            ScalarAdapter::NumericList(_) => "NumericList",
+            ScalarAdapter::List(_) => "List",
+        }
+    }
+
+    /// convert `ScalarRefImpl` to `ScalarAdapter` so that we can correctly encode to postgres value
+    pub(crate) fn from_scalar(
+        scalar: ScalarRefImpl<'_>,
+        ty: &Type,
+    ) -> ConnectorResult<ScalarAdapter> {
+        Ok(match (scalar, ty, ty.kind()) {
+            (ScalarRefImpl::Utf8(s), &Type::UUID, _) => ScalarAdapter::Uuid(s.parse()?),
+            (ScalarRefImpl::Utf8(s), &Type::NUMERIC, _) => {
+                ScalarAdapter::Numeric(string_to_pg_numeric(s))
+            }
+            (ScalarRefImpl::Int256(s), &Type::NUMERIC, _) => {
+                ScalarAdapter::Numeric(string_to_pg_numeric(&s.to_string()))
+            }
+            (ScalarRefImpl::Utf8(s), _, Kind::Enum(_)) => {
+                ScalarAdapter::Enum(EnumString(s.to_owned()))
+            }
+            (ScalarRefImpl::List(list), &Type::NUMERIC_ARRAY, _) => {
+                let mut vec = vec![];
+                for datum in list.iter() {
+                    vec.push(match datum {
+                        Some(ScalarRefImpl::Int256(s)) => Some(string_to_pg_numeric(&s.to_string())),
+                        Some(ScalarRefImpl::Decimal(s)) => Some(rw_numeric_to_pg_numeric(s)),
+                        Some(ScalarRefImpl::Utf8(s)) => Some(string_to_pg_numeric(s)),
+                        None => None,
+                        _ => {
+                            unreachable!("Only rw-numeric[], rw_int256[] and varchar[] are supported to convert to pg-numeric[]");
+                        }
+                    })
+                }
+                ScalarAdapter::NumericList(vec)
+            }
+            (ScalarRefImpl::List(list), _, Kind::Array(inner_type)) => match inner_type.kind() {
+                Kind::Enum(_) => {
+                    let mut vec = vec![];
+                    for datum in list.iter() {
+                        vec.push(match datum {
+                            Some(ScalarRefImpl::Utf8(s)) => Some(EnumString(s.to_owned())),
+                            _ => unreachable!(
+                                "Only non-null varchar[] is supported to convert to enum[]"
+                            ),
+                        })
+                    }
+                    ScalarAdapter::EnumList(vec)
+                }
+                _ => {
+                    let mut vec = vec![];
+                    for datum in list.iter() {
+                        vec.push(
+                            datum
+                                .map(|s| ScalarAdapter::from_scalar(s, inner_type))
+                                .transpose()?,
+                        );
+                    }
+                    ScalarAdapter::List(vec)
+                }
+            },
+            _ => ScalarAdapter::Builtin(scalar.into_scalar_impl()),
+        })
+    }
+
+    pub fn into_scalar(self, ty: &DataType) -> Option<ScalarImpl> {
+        match (self, &ty) {
+            (ScalarAdapter::Builtin(scalar), _) => Some(scalar),
+            (ScalarAdapter::Uuid(uuid), &DataType::Varchar) => {
+                Some(ScalarImpl::from(uuid.to_string()))
+            }
+            (ScalarAdapter::Numeric(numeric), &DataType::Varchar) => {
+                Some(ScalarImpl::from(pg_numeric_to_string(&numeric)))
+            }
+            (ScalarAdapter::Numeric(numeric), &DataType::Int256) => {
+                pg_numeric_to_rw_int256(&numeric)
+            }
+            (ScalarAdapter::Numeric(numeric), &DataType::Decimal) => {
+                pg_numeric_to_rw_numeric(&numeric)
+            }
+            (ScalarAdapter::Enum(EnumString(s)), &DataType::Varchar) => Some(ScalarImpl::from(s)),
+            (ScalarAdapter::NumericList(vec), &DataType::List(dtype)) => {
+                let mut builder = dtype.create_array_builder(0);
+                for val in vec {
+                    let scalar = match (val, &dtype) {
+                        // A numeric array contains special values like NaN, Inf, -Inf, which are not supported in Debezium,
+                        // when we encounter these special values, we fallback the array to NULL, returning None directly.
+                        (Some(numeric), box DataType::Varchar) => {
+                            if pg_numeric_is_special(&numeric) {
+                                return None;
+                            } else {
+                                ScalarAdapter::Numeric(numeric).into_scalar(dtype)
+                            }
+                        }
+                        (Some(numeric), box DataType::Int256 | box DataType::Decimal) => {
+                            if pg_numeric_is_special(&numeric) {
+                                return None;
+                            } else {
+                                // A PgNumeric can sometimes exceeds the range of Int256 and RwNumeric.
+                                // In our json parsing, we fallback the array to NULL in this case.
+                                // Here we keep the behavior consistent and return None directly.
+                                match ScalarAdapter::Numeric(numeric).into_scalar(dtype) {
+                                    Some(scalar) => Some(scalar),
+                                    None => {
+                                        return None;
+                                    }
+                                }
+                            }
+                        }
+                        (Some(_), _) => unreachable!(
+                            "Only rw-numeric[], rw_int256[] and varchar[] are supported to convert to pg-numeric[]"
+                        ),
+                        // This item is NULL, continue to handle next item.
+                        (None, _) => None,
+                    };
+                    builder.append(scalar);
+                }
+                Some(ScalarImpl::from(ListValue::new(builder.finish())))
+            }
+            (ScalarAdapter::EnumList(vec), &DataType::List(dtype)) => {
+                let mut builder = dtype.create_array_builder(0);
+                for val in vec {
+                    match val {
+                        Some(EnumString(s)) => {
+                            builder.append(Some(ScalarImpl::from(s)));
+                        }
+                        None => {
+                            return None;
+                        }
+                    }
+                }
+                Some(ScalarImpl::from(ListValue::new(builder.finish())))
+            }
+            (ScalarAdapter::List(vec), &DataType::List(dtype)) => {
+                // Due to https://github.com/risingwavelabs/risingwave/issues/16882, INTERVAL_ARRAY is not supported in Debezium, so we keep backfilling and CDC consistent.
+                if matches!(**dtype, DataType::Interval) {
+                    return None;
+                }
+                let mut builder = dtype.create_array_builder(0);
+                for val in vec {
+                    builder.append(val.and_then(|v| v.into_scalar(dtype)));
+                }
+                Some(ScalarImpl::from(ListValue::new(builder.finish())))
+            }
+            (scaler, ty) => {
+                tracing::error!(
+                    adapter = scaler.name(),
+                    rw_type = ty.pg_name(),
+                    "failed to convert from ScalarAdapter: invalid conversion"
+                );
+                None
+            }
+        }
+    }
+}
+
+fn pg_numeric_is_special(val: &PgNumeric) -> bool {
+    matches!(
+        val,
+        PgNumeric::NegativeInf | PgNumeric::PositiveInf | PgNumeric::NaN
+    )
+}
+
+fn pg_numeric_to_rw_int256(val: &PgNumeric) -> Option<ScalarImpl> {
+    match Int256::from_str(pg_numeric_to_string(val).as_str()) {
+        Ok(num) => Some(ScalarImpl::from(num)),
+        Err(err) => {
+            tracing::error!(error = %err.as_report(), "failed to convert PgNumeric to Int256");
+            None
+        }
+    }
+}
+
+fn pg_numeric_to_rw_numeric(val: &PgNumeric) -> Option<ScalarImpl> {
+    match val {
+        PgNumeric::NegativeInf => Some(ScalarImpl::from(Decimal::NegativeInf)),
+        PgNumeric::Normalized(big_decimal) => {
+            match Decimal::from_str(big_decimal.to_string().as_str()) {
+                Ok(num) => Some(ScalarImpl::from(num)),
+                Err(err) => {
+                    tracing::error!(error = %err.as_report(), "parse pg-numeric as rw-numeric failed (likely out-of-range");
+                    None
+                }
+            }
+        }
+        PgNumeric::PositiveInf => Some(ScalarImpl::from(Decimal::PositiveInf)),
+        PgNumeric::NaN => Some(ScalarImpl::from(Decimal::NaN)),
+    }
+}
+
+fn pg_numeric_to_string(val: &PgNumeric) -> String {
+    // TODO(kexiang): NEGATIVE_INFINITY -> -Infinity, POSITIVE_INFINITY -> Infinity, NAN -> NaN
+    // The current implementation is to ensure consistency with the behavior of cdc event parsor.
+    match val {
+        PgNumeric::NegativeInf => String::from("NEGATIVE_INFINITY"),
+        PgNumeric::Normalized(big_decimal) => big_decimal.to_string(),
+        PgNumeric::PositiveInf => String::from("POSITIVE_INFINITY"),
+        PgNumeric::NaN => String::from("NAN"),
+    }
+}
+
+fn string_to_pg_numeric(s: &str) -> PgNumeric {
+    match s {
+        "NEGATIVE_INFINITY" => PgNumeric::NegativeInf,
+        "POSITIVE_INFINITY" => PgNumeric::PositiveInf,
+        "NAN" => PgNumeric::NaN,
+        _ => PgNumeric::Normalized(s.parse().unwrap()),
+    }
+}
+
+fn rw_numeric_to_pg_numeric(val: Decimal) -> PgNumeric {
+    match val {
+        Decimal::NegativeInf => PgNumeric::NegativeInf,
+        Decimal::Normalized(inner) => PgNumeric::Normalized(inner.to_string().parse().unwrap()),
+        Decimal::PositiveInf => PgNumeric::PositiveInf,
+        Decimal::NaN => PgNumeric::NaN,
+    }
+}
diff --git a/src/connector/src/parser/unified/avro.rs b/src/connector/src/parser/unified/avro.rs
index 4e7d134f42554..68e95d6f78b9a 100644
--- a/src/connector/src/parser/unified/avro.rs
+++ b/src/connector/src/parser/unified/avro.rs
@@ -12,562 +12,4 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::str::FromStr;
-use std::sync::LazyLock;
-
-use apache_avro::schema::{DecimalSchema, RecordSchema};
-use apache_avro::types::Value;
-use apache_avro::{Decimal as AvroDecimal, Schema};
-use chrono::Datelike;
-use itertools::Itertools;
-use num_bigint::{BigInt, Sign};
-use risingwave_common::array::{ListValue, StructValue};
-use risingwave_common::bail;
-use risingwave_common::log::LogSuppresser;
-use risingwave_common::types::{
-    DataType, Date, Datum, Interval, JsonbVal, ScalarImpl, Time, Timestamp, Timestamptz,
-};
-use risingwave_common::util::iter_util::ZipEqFast;
-
-use super::{bail_uncategorized, uncategorized, Access, AccessError, AccessResult};
-use crate::error::ConnectorResult;
-#[derive(Clone)]
-/// Options for parsing an `AvroValue` into Datum, with an optional avro schema.
-pub struct AvroParseOptions<'a> {
-    pub schema: Option<&'a Schema>,
-    /// Strict Mode
-    /// If strict mode is disabled, an int64 can be parsed from an `AvroInt` (int32) value.
-    pub relax_numeric: bool,
-}
-
-impl<'a> Default for AvroParseOptions<'a> {
-    fn default() -> Self {
-        Self {
-            schema: None,
-            relax_numeric: true,
-        }
-    }
-}
-
-impl<'a> AvroParseOptions<'a> {
-    pub fn with_schema(mut self, schema: &'a Schema) -> Self {
-        self.schema = Some(schema);
-        self
-    }
-
-    fn extract_inner_schema(&self, key: Option<&'a str>) -> Option<&'a Schema> {
-        self.schema
-            .map(|schema| avro_extract_field_schema(schema, key))
-            .transpose()
-            .map_err(|_err| {
-                static LOG_SUPPERSSER: LazyLock<LogSuppresser> =
-                    LazyLock::new(LogSuppresser::default);
-                if let Ok(suppressed_count) = LOG_SUPPERSSER.check() {
-                    tracing::error!(suppressed_count, "extract sub-schema");
-                }
-            })
-            .ok()
-            .flatten()
-    }
-
-    /// Parse an avro value into expected type.
-    /// 3 kinds of type info are used to parsing things.
-    ///     - `type_expected`. The type that we expect the value is.
-    ///     - value type. The type info together with the value argument.
-    ///     - schema. The `AvroSchema` provided in option.
-    /// If both `type_expected` and schema are provided, it will check both strictly.
-    /// If only `type_expected` is provided, it will try to match the value type and the
-    /// `type_expected`, converting the value if possible. If only value is provided (without
-    /// schema and `type_expected`), the `DateType` will be inferred.
-    pub fn parse<'b>(&self, value: &'b Value, type_expected: Option<&'b DataType>) -> AccessResult
-    where
-        'b: 'a,
-    {
-        let create_error = || AccessError::TypeError {
-            expected: format!("{:?}", type_expected),
-            got: format!("{:?}", value),
-            value: String::new(),
-        };
-
-        let v: ScalarImpl = match (type_expected, value) {
-            (_, Value::Null) => return Ok(None),
-            (_, Value::Union(_, v)) => {
-                let schema = self.extract_inner_schema(None);
-                return Self {
-                    schema,
-                    relax_numeric: self.relax_numeric,
-                }
-                .parse(v, type_expected);
-            }
-            // ---- Boolean -----
-            (Some(DataType::Boolean) | None, Value::Boolean(b)) => (*b).into(),
-            // ---- Int16 -----
-            (Some(DataType::Int16), Value::Int(i)) if self.relax_numeric => (*i as i16).into(),
-            (Some(DataType::Int16), Value::Long(i)) if self.relax_numeric => (*i as i16).into(),
-
-            // ---- Int32 -----
-            (Some(DataType::Int32) | None, Value::Int(i)) => (*i).into(),
-            (Some(DataType::Int32), Value::Long(i)) if self.relax_numeric => (*i as i32).into(),
-            // ---- Int64 -----
-            (Some(DataType::Int64) | None, Value::Long(i)) => (*i).into(),
-            (Some(DataType::Int64), Value::Int(i)) if self.relax_numeric => (*i as i64).into(),
-            // ---- Float32 -----
-            (Some(DataType::Float32) | None, Value::Float(i)) => (*i).into(),
-            (Some(DataType::Float32), Value::Double(i)) => (*i as f32).into(),
-            // ---- Float64 -----
-            (Some(DataType::Float64) | None, Value::Double(i)) => (*i).into(),
-            (Some(DataType::Float64), Value::Float(i)) => (*i as f64).into(),
-            // ---- Decimal -----
-            (Some(DataType::Decimal) | None, Value::Decimal(avro_decimal)) => {
-                let (precision, scale) = match self.schema {
-                    Some(Schema::Decimal(DecimalSchema {
-                        precision, scale, ..
-                    })) => (*precision, *scale),
-                    _ => Err(create_error())?,
-                };
-                let decimal = avro_decimal_to_rust_decimal(avro_decimal.clone(), precision, scale)
-                    .map_err(|_| create_error())?;
-                ScalarImpl::Decimal(risingwave_common::types::Decimal::Normalized(decimal))
-            }
-            (Some(DataType::Decimal), Value::Record(fields)) => {
-                // VariableScaleDecimal has fixed fields, scale(int) and value(bytes)
-                let find_in_records = |field_name: &str| {
-                    fields
-                        .iter()
-                        .find(|field| field.0 == field_name)
-                        .map(|field| &field.1)
-                        .ok_or_else(|| {
-                            uncategorized!("`{field_name}` field not found in VariableScaleDecimal")
-                        })
-                };
-                let scale = match find_in_records("scale")? {
-                    Value::Int(scale) => *scale,
-                    avro_value => bail_uncategorized!(
-                        "scale field in VariableScaleDecimal is not int, got {:?}",
-                        avro_value
-                    ),
-                };
-
-                let value: BigInt = match find_in_records("value")? {
-                    Value::Bytes(bytes) => BigInt::from_signed_bytes_be(bytes),
-                    avro_value => bail_uncategorized!(
-                        "value field in VariableScaleDecimal is not bytes, got {:?}",
-                        avro_value
-                    ),
-                };
-
-                let negative = value.sign() == Sign::Minus;
-                let (lo, mid, hi) = extract_decimal(value.to_bytes_be().1)?;
-                let decimal =
-                    rust_decimal::Decimal::from_parts(lo, mid, hi, negative, scale as u32);
-                ScalarImpl::Decimal(risingwave_common::types::Decimal::Normalized(decimal))
-            }
-            // ---- Time -----
-            (Some(DataType::Time), Value::TimeMillis(ms)) => Time::with_milli(*ms as u32)
-                .map_err(|_| create_error())?
-                .into(),
-            (Some(DataType::Time), Value::TimeMicros(us)) => Time::with_micro(*us as u64)
-                .map_err(|_| create_error())?
-                .into(),
-            // ---- Date -----
-            (Some(DataType::Date) | None, Value::Date(days)) => {
-                Date::with_days(days + unix_epoch_days())
-                    .map_err(|_| create_error())?
-                    .into()
-            }
-            // ---- Varchar -----
-            (Some(DataType::Varchar) | None, Value::Enum(_, symbol)) => {
-                symbol.clone().into_boxed_str().into()
-            }
-            (Some(DataType::Varchar) | None, Value::String(s)) => s.clone().into_boxed_str().into(),
-            // ---- Timestamp -----
-            (Some(DataType::Timestamp) | None, Value::LocalTimestampMillis(ms)) => {
-                Timestamp::with_millis(*ms)
-                    .map_err(|_| create_error())?
-                    .into()
-            }
-            (Some(DataType::Timestamp) | None, Value::LocalTimestampMicros(us)) => {
-                Timestamp::with_micros(*us)
-                    .map_err(|_| create_error())?
-                    .into()
-            }
-
-            // ---- TimestampTz -----
-            (Some(DataType::Timestamptz) | None, Value::TimestampMillis(ms)) => {
-                Timestamptz::from_millis(*ms)
-                    .ok_or_else(|| {
-                        uncategorized!("timestamptz with milliseconds {ms} * 1000 is out of range")
-                    })?
-                    .into()
-            }
-            (Some(DataType::Timestamptz) | None, Value::TimestampMicros(us)) => {
-                Timestamptz::from_micros(*us).into()
-            }
-
-            // ---- Interval -----
-            (Some(DataType::Interval) | None, Value::Duration(duration)) => {
-                let months = u32::from(duration.months()) as i32;
-                let days = u32::from(duration.days()) as i32;
-                let usecs = (u32::from(duration.millis()) as i64) * 1000; // never overflows
-                ScalarImpl::Interval(Interval::from_month_day_usec(months, days, usecs))
-            }
-            // ---- Struct -----
-            (Some(DataType::Struct(struct_type_info)), Value::Record(descs)) => StructValue::new(
-                struct_type_info
-                    .names()
-                    .zip_eq_fast(struct_type_info.types())
-                    .map(|(field_name, field_type)| {
-                        let maybe_value = descs.iter().find(|(k, _v)| k == field_name);
-                        if let Some((_, value)) = maybe_value {
-                            let schema = self.extract_inner_schema(Some(field_name));
-                            Ok(Self {
-                                schema,
-                                relax_numeric: self.relax_numeric,
-                            }
-                            .parse(value, Some(field_type))?)
-                        } else {
-                            Ok(None)
-                        }
-                    })
-                    .collect::<Result<_, AccessError>>()?,
-            )
-            .into(),
-            (None, Value::Record(descs)) => {
-                let rw_values = descs
-                    .iter()
-                    .map(|(field_name, field_value)| {
-                        let schema = self.extract_inner_schema(Some(field_name));
-                        Self {
-                            schema,
-                            relax_numeric: self.relax_numeric,
-                        }
-                        .parse(field_value, None)
-                    })
-                    .collect::<Result<Vec<Datum>, AccessError>>()?;
-                ScalarImpl::Struct(StructValue::new(rw_values))
-            }
-            // ---- List -----
-            (Some(DataType::List(item_type)), Value::Array(array)) => ListValue::new({
-                let schema = self.extract_inner_schema(None);
-                let mut builder = item_type.create_array_builder(array.len());
-                for v in array {
-                    let value = Self {
-                        schema,
-                        relax_numeric: self.relax_numeric,
-                    }
-                    .parse(v, Some(item_type))?;
-                    builder.append(value);
-                }
-                builder.finish()
-            })
-            .into(),
-            // ---- Bytea -----
-            (Some(DataType::Bytea) | None, Value::Bytes(value)) => {
-                value.clone().into_boxed_slice().into()
-            }
-            // ---- Jsonb -----
-            (Some(DataType::Jsonb), Value::String(s)) => {
-                JsonbVal::from_str(s).map_err(|_| create_error())?.into()
-            }
-
-            (_expected, _got) => Err(create_error())?,
-        };
-        Ok(Some(v))
-    }
-}
-
-pub struct AvroAccess<'a, 'b> {
-    value: &'a Value,
-    options: AvroParseOptions<'b>,
-}
-
-impl<'a, 'b> AvroAccess<'a, 'b> {
-    pub fn new(value: &'a Value, options: AvroParseOptions<'b>) -> Self {
-        Self { value, options }
-    }
-}
-
-impl<'a, 'b> Access for AvroAccess<'a, 'b>
-where
-    'a: 'b,
-{
-    fn access(&self, path: &[&str], type_expected: Option<&DataType>) -> AccessResult {
-        let mut value = self.value;
-        let mut options: AvroParseOptions<'_> = self.options.clone();
-
-        let mut i = 0;
-        while i < path.len() {
-            let key = path[i];
-            let create_error = || AccessError::Undefined {
-                name: key.to_string(),
-                path: path.iter().take(i).join("."),
-            };
-            match value {
-                Value::Union(_, v) => {
-                    value = v;
-                    options.schema = options.extract_inner_schema(None);
-                    continue;
-                }
-                Value::Map(fields) if fields.contains_key(key) => {
-                    value = fields.get(key).unwrap();
-                    options.schema = None;
-                    i += 1;
-                    continue;
-                }
-                Value::Record(fields) => {
-                    if let Some((_, v)) = fields.iter().find(|(k, _)| k == key) {
-                        value = v;
-                        options.schema = options.extract_inner_schema(Some(key));
-                        i += 1;
-                        continue;
-                    }
-                }
-                _ => (),
-            }
-            Err(create_error())?;
-        }
-
-        options.parse(value, type_expected)
-    }
-}
-
-pub(crate) fn avro_decimal_to_rust_decimal(
-    avro_decimal: AvroDecimal,
-    _precision: usize,
-    scale: usize,
-) -> AccessResult<rust_decimal::Decimal> {
-    let negative = !avro_decimal.is_positive();
-    let bytes = avro_decimal.to_vec_unsigned();
-
-    let (lo, mid, hi) = extract_decimal(bytes)?;
-    Ok(rust_decimal::Decimal::from_parts(
-        lo,
-        mid,
-        hi,
-        negative,
-        scale as u32,
-    ))
-}
-
-pub(crate) fn extract_decimal(bytes: Vec<u8>) -> AccessResult<(u32, u32, u32)> {
-    match bytes.len() {
-        len @ 0..=4 => {
-            let mut pad = vec![0; 4 - len];
-            pad.extend_from_slice(&bytes);
-            let lo = u32::from_be_bytes(pad.try_into().unwrap());
-            Ok((lo, 0, 0))
-        }
-        len @ 5..=8 => {
-            let zero_len = 8 - len;
-            let mid_end = 4 - zero_len;
-
-            let mut pad = vec![0; zero_len];
-            pad.extend_from_slice(&bytes[..mid_end]);
-            let mid = u32::from_be_bytes(pad.try_into().unwrap());
-
-            let lo = u32::from_be_bytes(bytes[mid_end..].to_owned().try_into().unwrap());
-            Ok((lo, mid, 0))
-        }
-        len @ 9..=12 => {
-            let zero_len = 12 - len;
-            let hi_end = 4 - zero_len;
-            let mid_end = hi_end + 4;
-
-            let mut pad = vec![0; zero_len];
-            pad.extend_from_slice(&bytes[..hi_end]);
-            let hi = u32::from_be_bytes(pad.try_into().unwrap());
-
-            let mid = u32::from_be_bytes(bytes[hi_end..mid_end].to_owned().try_into().unwrap());
-
-            let lo = u32::from_be_bytes(bytes[mid_end..].to_owned().try_into().unwrap());
-            Ok((lo, mid, hi))
-        }
-        _ => bail_uncategorized!("invalid decimal bytes length {}", bytes.len()),
-    }
-}
-
-pub fn avro_schema_skip_union(schema: &Schema) -> ConnectorResult<&Schema> {
-    match schema {
-        Schema::Union(union_schema) => {
-            let inner_schema = union_schema
-                .variants()
-                .iter()
-                .find(|s| !matches!(s, &&Schema::Null))
-                .ok_or_else(|| {
-                    anyhow::format_err!("illegal avro record schema {:?}", union_schema)
-                })?;
-            Ok(inner_schema)
-        }
-        other => Ok(other),
-    }
-}
-// extract inner filed/item schema of record/array/union
-pub fn avro_extract_field_schema<'a>(
-    schema: &'a Schema,
-    name: Option<&'a str>,
-) -> ConnectorResult<&'a Schema> {
-    match schema {
-        Schema::Record(RecordSchema { fields, lookup, .. }) => {
-            let name =
-                name.ok_or_else(|| anyhow::format_err!("no name provided for a field in record"))?;
-            let index = lookup.get(name).ok_or_else(|| {
-                anyhow::format_err!("no field named '{}' in record: {:?}", name, schema)
-            })?;
-            let field = fields
-                .get(*index)
-                .ok_or_else(|| anyhow::format_err!("illegal avro record schema {:?}", schema))?;
-            Ok(&field.schema)
-        }
-        Schema::Array(schema) => Ok(schema),
-        Schema::Union(_) => avro_schema_skip_union(schema),
-        _ => bail!("avro schema is not a record or array"),
-    }
-}
-
-pub(crate) fn unix_epoch_days() -> i32 {
-    Date::from_ymd_uncheck(1970, 1, 1).0.num_days_from_ce()
-}
-
-#[cfg(test)]
-mod tests {
-    use apache_avro::Decimal as AvroDecimal;
-    use risingwave_common::types::{Decimal, Timestamptz};
-
-    use super::*;
-
-    #[test]
-    fn test_convert_decimal() {
-        // 280
-        let v = vec![1, 24];
-        let avro_decimal = AvroDecimal::from(v);
-        let rust_decimal = avro_decimal_to_rust_decimal(avro_decimal, 28, 0).unwrap();
-        assert_eq!(rust_decimal, rust_decimal::Decimal::from(280));
-
-        // 28.1
-        let v = vec![1, 25];
-        let avro_decimal = AvroDecimal::from(v);
-        let rust_decimal = avro_decimal_to_rust_decimal(avro_decimal, 28, 1).unwrap();
-        assert_eq!(rust_decimal, rust_decimal::Decimal::try_from(28.1).unwrap());
-
-        // 1.1234567891
-        let value = BigInt::from(11234567891_i64);
-        let negative = value.sign() == Sign::Minus;
-        let (lo, mid, hi) = extract_decimal(value.to_bytes_be().1).unwrap();
-        let decimal = rust_decimal::Decimal::from_parts(lo, mid, hi, negative, 10);
-        assert_eq!(
-            decimal,
-            rust_decimal::Decimal::try_from(1.1234567891).unwrap()
-        );
-
-        // 1.123456789123456789123456789
-        let v = vec![3, 161, 77, 58, 146, 180, 49, 220, 100, 4, 95, 21];
-        let avro_decimal = AvroDecimal::from(v);
-        let rust_decimal = avro_decimal_to_rust_decimal(avro_decimal, 28, 27).unwrap();
-        assert_eq!(
-            rust_decimal,
-            rust_decimal::Decimal::from_str("1.123456789123456789123456789").unwrap()
-        );
-    }
-
-    /// Convert Avro value to datum.For now, support the following [Avro type](https://avro.apache.org/docs/current/spec.html).
-    ///  - boolean
-    ///  - int : i32
-    ///  - long: i64
-    ///  - float: f32
-    ///  - double: f64
-    ///  - string: String
-    ///  - Date (the number of days from the unix epoch, 1970-1-1 UTC)
-    ///  - Timestamp (the number of milliseconds from the unix epoch,  1970-1-1 00:00:00.000 UTC)
-
-    pub(crate) fn from_avro_value(
-        value: Value,
-        value_schema: &Schema,
-        shape: &DataType,
-    ) -> crate::error::ConnectorResult<Datum> {
-        AvroParseOptions {
-            schema: Some(value_schema),
-            relax_numeric: true,
-        }
-        .parse(&value, Some(shape))
-        .map_err(Into::into)
-    }
-
-    #[test]
-    fn test_avro_timestamptz_micros() {
-        let v1 = Value::TimestampMicros(1620000000000000);
-        let v2 = Value::TimestampMillis(1620000000000);
-        let value_schema1 = Schema::TimestampMicros;
-        let value_schema2 = Schema::TimestampMillis;
-        let datum1 = from_avro_value(v1, &value_schema1, &DataType::Timestamptz).unwrap();
-        let datum2 = from_avro_value(v2, &value_schema2, &DataType::Timestamptz).unwrap();
-        assert_eq!(
-            datum1,
-            Some(ScalarImpl::Timestamptz(
-                Timestamptz::from_str("2021-05-03T00:00:00Z").unwrap()
-            ))
-        );
-        assert_eq!(
-            datum2,
-            Some(ScalarImpl::Timestamptz(
-                Timestamptz::from_str("2021-05-03T00:00:00Z").unwrap()
-            ))
-        );
-    }
-
-    #[test]
-    fn test_decimal_truncate() {
-        let schema = Schema::parse_str(
-            r#"
-            {
-                "type": "bytes",
-                "logicalType": "decimal",
-                "precision": 38,
-                "scale": 18
-            }
-            "#,
-        )
-        .unwrap();
-        let bytes = vec![0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f];
-        let value = Value::Decimal(AvroDecimal::from(bytes));
-        let options = AvroParseOptions::default().with_schema(&schema);
-        let resp = options.parse(&value, Some(&DataType::Decimal)).unwrap();
-        assert_eq!(
-            resp,
-            Some(ScalarImpl::Decimal(Decimal::Normalized(
-                rust_decimal::Decimal::from_str("0.017802464409370431").unwrap()
-            )))
-        );
-    }
-
-    #[test]
-    fn test_variable_scale_decimal() {
-        let schema = Schema::parse_str(
-            r#"
-            {
-                "type": "record",
-                "name": "VariableScaleDecimal",
-                "namespace": "io.debezium.data",
-                "fields": [
-                    {
-                        "name": "scale",
-                        "type": "int"
-                    },
-                    {
-                        "name": "value",
-                        "type": "bytes"
-                    }
-                ]
-            }
-            "#,
-        )
-        .unwrap();
-        let value = Value::Record(vec![
-            ("scale".to_string(), Value::Int(0)),
-            ("value".to_string(), Value::Bytes(vec![0x01, 0x02, 0x03])),
-        ]);
-
-        let options = AvroParseOptions::default().with_schema(&schema);
-        let resp = options.parse(&value, Some(&DataType::Decimal)).unwrap();
-        assert_eq!(resp, Some(ScalarImpl::Decimal(Decimal::from(66051))));
-    }
-}
+pub use risingwave_connector_codec::decoder::avro::*;
diff --git a/src/connector/src/parser/unified/bytes.rs b/src/connector/src/parser/unified/bytes.rs
index ff47424d60acf..2f8f27124a307 100644
--- a/src/connector/src/parser/unified/bytes.rs
+++ b/src/connector/src/parser/unified/bytes.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use risingwave_common::types::{DataType, ScalarImpl};
+use risingwave_common::types::{DataType, DatumCow, ScalarRefImpl};
 
 use super::{Access, AccessError, AccessResult};
 
@@ -29,14 +29,16 @@ impl<'a> BytesAccess<'a> {
     }
 }
 
-impl<'a> Access for BytesAccess<'a> {
+impl Access for BytesAccess<'_> {
     /// path is empty currently, `type_expected` should be `Bytea`
-    fn access(&self, path: &[&str], type_expected: Option<&DataType>) -> AccessResult {
-        if let DataType::Bytea = type_expected.unwrap() {
+    fn access<'a>(&'a self, path: &[&str], type_expected: &DataType) -> AccessResult<DatumCow<'a>> {
+        if let DataType::Bytea = type_expected {
             if self.column_name.is_none()
                 || (path.len() == 1 && self.column_name.as_ref().unwrap() == path[0])
             {
-                return Ok(Some(ScalarImpl::Bytea(Box::from(self.bytes.as_slice()))));
+                return Ok(DatumCow::Borrowed(Some(ScalarRefImpl::Bytea(
+                    self.bytes.as_slice(),
+                ))));
             }
             return Err(AccessError::Undefined {
                 name: path[0].to_string(),
diff --git a/src/connector/src/parser/unified/debezium.rs b/src/connector/src/parser/unified/debezium.rs
index 589fdfc8cfc45..d90463698577d 100644
--- a/src/connector/src/parser/unified/debezium.rs
+++ b/src/connector/src/parser/unified/debezium.rs
@@ -12,12 +12,49 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use risingwave_common::types::{DataType, Datum, ScalarImpl};
+use risingwave_common::types::{
+    DataType, Datum, DatumCow, Scalar, ScalarImpl, ScalarRefImpl, Timestamptz, ToDatumRef,
+};
+use risingwave_connector_codec::decoder::AccessExt;
+use risingwave_pb::plan_common::additional_column::ColumnType;
 
 use super::{Access, AccessError, AccessResult, ChangeEvent, ChangeEventOperation};
 use crate::parser::TransactionControl;
 use crate::source::{ConnectorProperties, SourceColumnDesc};
 
+// Example of Debezium JSON value:
+// {
+//     "payload":
+//     {
+//         "before": null,
+//         "after":
+//         {
+//             "O_ORDERKEY": 5,
+//             "O_CUSTKEY": 44485,
+//             "O_ORDERSTATUS": "F",
+//             "O_TOTALPRICE": "144659.20",
+//             "O_ORDERDATE": "1994-07-30"
+//         },
+//         "source":
+//         {
+//             "version": "1.9.7.Final",
+//             "connector": "mysql",
+//             "name": "RW_CDC_1002",
+//             "ts_ms": 1695277757000,
+//             "db": "mydb",
+//             "sequence": null,
+//             "table": "orders",
+//             "server_id": 0,
+//             "gtid": null,
+//             "file": "binlog.000008",
+//             "pos": 3693,
+//             "row": 0,
+//         },
+//         "op": "r",
+//         "ts_ms": 1695277757017,
+//         "transaction": null
+//     }
+// }
 pub struct DebeziumChangeEvent<A> {
     value_accessor: Option<A>,
     key_accessor: Option<A>,
@@ -26,6 +63,14 @@ pub struct DebeziumChangeEvent<A> {
 
 const BEFORE: &str = "before";
 const AFTER: &str = "after";
+
+const SOURCE: &str = "source";
+const SOURCE_TS_MS: &str = "ts_ms";
+const SOURCE_DB: &str = "db";
+const SOURCE_SCHEMA: &str = "schema";
+const SOURCE_TABLE: &str = "table";
+const SOURCE_COLLECTION: &str = "collection";
+
 const OP: &str = "op";
 pub const TRANSACTION_STATUS: &str = "status";
 pub const TRANSACTION_ID: &str = "id";
@@ -42,20 +87,26 @@ pub fn parse_transaction_meta(
     accessor: &impl Access,
     connector_props: &ConnectorProperties,
 ) -> AccessResult<TransactionControl> {
-    if let (Some(ScalarImpl::Utf8(status)), Some(ScalarImpl::Utf8(id))) = (
-        accessor.access(&[TRANSACTION_STATUS], Some(&DataType::Varchar))?,
-        accessor.access(&[TRANSACTION_ID], Some(&DataType::Varchar))?,
+    if let (Some(ScalarRefImpl::Utf8(status)), Some(ScalarRefImpl::Utf8(id))) = (
+        accessor
+            .access(&[TRANSACTION_STATUS], &DataType::Varchar)?
+            .to_datum_ref(),
+        accessor
+            .access(&[TRANSACTION_ID], &DataType::Varchar)?
+            .to_datum_ref(),
     ) {
         // The id field has different meanings for different databases:
         // PG: txID:LSN
         // MySQL: source_id:transaction_id (e.g. 3E11FA47-71CA-11E1-9E33-C80AA9429562:23)
-        match status.as_ref() {
+        match status {
             DEBEZIUM_TRANSACTION_STATUS_BEGIN => match *connector_props {
                 ConnectorProperties::PostgresCdc(_) => {
                     let (tx_id, _) = id.split_once(':').unwrap();
                     return Ok(TransactionControl::Begin { id: tx_id.into() });
                 }
-                ConnectorProperties::MysqlCdc(_) => return Ok(TransactionControl::Begin { id }),
+                ConnectorProperties::MysqlCdc(_) => {
+                    return Ok(TransactionControl::Begin { id: id.into() })
+                }
                 _ => {}
             },
             DEBEZIUM_TRANSACTION_STATUS_COMMIT => match *connector_props {
@@ -63,7 +114,9 @@ pub fn parse_transaction_meta(
                     let (tx_id, _) = id.split_once(':').unwrap();
                     return Ok(TransactionControl::Commit { id: tx_id.into() });
                 }
-                ConnectorProperties::MysqlCdc(_) => return Ok(TransactionControl::Commit { id }),
+                ConnectorProperties::MysqlCdc(_) => {
+                    return Ok(TransactionControl::Commit { id: id.into() })
+                }
                 _ => {}
             },
             _ => {}
@@ -80,14 +133,6 @@ impl<A> DebeziumChangeEvent<A>
 where
     A: Access,
 {
-    pub fn with_value(value_accessor: A) -> Self {
-        Self::new(None, Some(value_accessor))
-    }
-
-    pub fn with_key(key_accessor: A) -> Self {
-        Self::new(Some(key_accessor), None)
-    }
-
     /// Panic: one of the `key_accessor` or `value_accessor` must be provided.
     pub fn new(key_accessor: Option<A>, value_accessor: Option<A>) -> Self {
         assert!(key_accessor.is_some() || value_accessor.is_some());
@@ -126,7 +171,7 @@ impl<A> ChangeEvent for DebeziumChangeEvent<A>
 where
     A: Access,
 {
-    fn access_field(&self, desc: &SourceColumnDesc) -> super::AccessResult {
+    fn access_field(&self, desc: &SourceColumnDesc) -> super::AccessResult<DatumCow<'_>> {
         match self.op()? {
             ChangeEventOperation::Delete => {
                 // For delete events of MongoDB, the "before" and "after" field both are null in the value,
@@ -136,32 +181,80 @@ where
                         .key_accessor
                         .as_ref()
                         .expect("key_accessor must be provided for delete operation")
-                        .access(&[&desc.name], Some(&desc.data_type));
+                        .access(&[&desc.name], &desc.data_type);
                 }
 
                 if let Some(va) = self.value_accessor.as_ref() {
-                    va.access(&[BEFORE, &desc.name], Some(&desc.data_type))
+                    va.access(&[BEFORE, &desc.name], &desc.data_type)
                 } else {
                     self.key_accessor
                         .as_ref()
                         .unwrap()
-                        .access(&[&desc.name], Some(&desc.data_type))
+                        .access(&[&desc.name], &desc.data_type)
                 }
             }
 
             // value should not be None.
-            ChangeEventOperation::Upsert => self
-                .value_accessor
-                .as_ref()
-                .unwrap()
-                .access(&[AFTER, &desc.name], Some(&desc.data_type)),
+            ChangeEventOperation::Upsert => {
+                // For upsert operation, if desc is an additional column, access field in the `SOURCE` field.
+                desc.additional_column.column_type.as_ref().map_or_else(
+                    || {
+                        self.value_accessor
+                            .as_ref()
+                            .expect("value_accessor must be provided for upsert operation")
+                            .access(&[AFTER, &desc.name], &desc.data_type)
+                    },
+                    |additional_column_type| {
+                        match *additional_column_type {
+                            ColumnType::Timestamp(_) => {
+                                // access payload.source.ts_ms
+                                let ts_ms = self
+                                    .value_accessor
+                                    .as_ref()
+                                    .expect("value_accessor must be provided for upsert operation")
+                                    .access_owned(&[SOURCE, SOURCE_TS_MS], &DataType::Int64)?;
+                                Ok(DatumCow::Owned(ts_ms.map(|scalar| {
+                                    Timestamptz::from_millis(scalar.into_int64())
+                                        .expect("source.ts_ms must in millisecond")
+                                        .to_scalar_value()
+                                })))
+                            }
+                            ColumnType::DatabaseName(_) => self
+                                .value_accessor
+                                .as_ref()
+                                .expect("value_accessor must be provided for upsert operation")
+                                .access(&[SOURCE, SOURCE_DB], &desc.data_type),
+                            ColumnType::SchemaName(_) => self
+                                .value_accessor
+                                .as_ref()
+                                .expect("value_accessor must be provided for upsert operation")
+                                .access(&[SOURCE, SOURCE_SCHEMA], &desc.data_type),
+                            ColumnType::TableName(_) => self
+                                .value_accessor
+                                .as_ref()
+                                .expect("value_accessor must be provided for upsert operation")
+                                .access(&[SOURCE, SOURCE_TABLE], &desc.data_type),
+                            ColumnType::CollectionName(_) => self
+                                .value_accessor
+                                .as_ref()
+                                .expect("value_accessor must be provided for upsert operation")
+                                .access(&[SOURCE, SOURCE_COLLECTION], &desc.data_type),
+                            _ => Err(AccessError::UnsupportedAdditionalColumn {
+                                name: desc.name.clone(),
+                            }),
+                        }
+                    },
+                )
+            }
         }
     }
 
     fn op(&self) -> Result<ChangeEventOperation, AccessError> {
         if let Some(accessor) = &self.value_accessor {
-            if let Some(ScalarImpl::Utf8(op)) = accessor.access(&[OP], Some(&DataType::Varchar))? {
-                match op.as_ref() {
+            if let Some(ScalarRefImpl::Utf8(op)) =
+                accessor.access(&[OP], &DataType::Varchar)?.to_datum_ref()
+            {
+                match op {
                     DEBEZIUM_READ_OP | DEBEZIUM_CREATE_OP | DEBEZIUM_UPDATE_OP => {
                         return Ok(ChangeEventOperation::Upsert)
                     }
@@ -247,15 +340,12 @@ impl<A> Access for MongoJsonAccess<A>
 where
     A: Access,
 {
-    fn access(&self, path: &[&str], type_expected: Option<&DataType>) -> super::AccessResult {
+    fn access<'a>(&'a self, path: &[&str], type_expected: &DataType) -> AccessResult<DatumCow<'a>> {
         match path {
             ["after" | "before", "_id"] => {
-                let payload = self.access(&[path[0]], Some(&DataType::Jsonb))?;
+                let payload = self.access_owned(&[path[0]], &DataType::Jsonb)?;
                 if let Some(ScalarImpl::Jsonb(bson_doc)) = payload {
-                    Ok(extract_bson_id(
-                        type_expected.unwrap_or(&DataType::Jsonb),
-                        &bson_doc.take(),
-                    )?)
+                    Ok(extract_bson_id(type_expected, &bson_doc.take())?.into())
                 } else {
                     // fail to extract the "_id" field from the message payload
                     Err(AccessError::Undefined {
@@ -264,19 +354,16 @@ where
                     })?
                 }
             }
-            ["after" | "before", "payload"] => self.access(&[path[0]], Some(&DataType::Jsonb)),
+            ["after" | "before", "payload"] => self.access(&[path[0]], &DataType::Jsonb),
             // To handle a DELETE message, we need to extract the "_id" field from the message key, because it is not in the payload.
             // In addition, the "_id" field is named as "id" in the key. An example of message key:
             // {"schema":null,"payload":{"id":"{\"$oid\": \"65bc9fb6c485f419a7a877fe\"}"}}
             ["_id"] => {
                 let ret = self.accessor.access(path, type_expected);
                 if matches!(ret, Err(AccessError::Undefined { .. })) {
-                    let id_bson = self.accessor.access(&["id"], Some(&DataType::Jsonb))?;
+                    let id_bson = self.accessor.access_owned(&["id"], &DataType::Jsonb)?;
                     if let Some(ScalarImpl::Jsonb(bson_doc)) = id_bson {
-                        Ok(extract_bson_id(
-                            type_expected.unwrap_or(&DataType::Jsonb),
-                            &bson_doc.take(),
-                        )?)
+                        Ok(extract_bson_id(type_expected, &bson_doc.take())?.into())
                     } else {
                         // fail to extract the "_id" field from the message key
                         Err(AccessError::Undefined {
diff --git a/src/connector/src/parser/unified/json.rs b/src/connector/src/parser/unified/json.rs
index d9deb73d582d9..09704d9192a41 100644
--- a/src/connector/src/parser/unified/json.rs
+++ b/src/connector/src/parser/unified/json.rs
@@ -23,8 +23,10 @@ use risingwave_common::cast::{i64_to_timestamp, i64_to_timestamptz, str_to_bytea
 use risingwave_common::log::LogSuppresser;
 use risingwave_common::types::{
     DataType, Date, Decimal, Int256, Interval, JsonbVal, ScalarImpl, Time, Timestamp, Timestamptz,
+    ToOwnedDatum,
 };
 use risingwave_common::util::iter_util::ZipEqFast;
+use risingwave_connector_codec::decoder::utils::extract_decimal;
 use simd_json::prelude::{
     TypedValue, ValueAsContainer, ValueAsScalar, ValueObjectAccess, ValueTryAsScalar,
 };
@@ -33,7 +35,7 @@ use thiserror_ext::AsReport;
 
 use super::{Access, AccessError, AccessResult};
 use crate::parser::common::json_object_get_case_insensitive;
-use crate::parser::unified::avro::extract_decimal;
+use crate::parser::DatumCow;
 use crate::schema::{bail_invalid_option_error, InvalidOptionError};
 
 #[derive(Clone, Debug)]
@@ -199,11 +201,11 @@ impl JsonParseOptions {
         }
     }
 
-    pub fn parse(
+    pub fn parse<'a>(
         &self,
-        value: &BorrowedValue<'_>,
-        type_expected: Option<&DataType>,
-    ) -> AccessResult {
+        value: &'a BorrowedValue<'a>,
+        type_expected: &DataType,
+    ) -> AccessResult<DatumCow<'a>> {
         let create_error = || AccessError::TypeError {
             expected: format!("{:?}", type_expected),
             got: value.value_type().to_string(),
@@ -211,12 +213,12 @@ impl JsonParseOptions {
         };
 
         let v: ScalarImpl = match (type_expected, value.value_type()) {
-            (_, ValueType::Null) => return Ok(None),
+            (_, ValueType::Null) => return Ok(DatumCow::NULL),
             // ---- Boolean -----
-            (Some(DataType::Boolean) | None, ValueType::Bool) => value.as_bool().unwrap().into(),
+            (DataType::Boolean, ValueType::Bool) => value.as_bool().unwrap().into(),
 
             (
-                Some(DataType::Boolean),
+                DataType::Boolean,
                 ValueType::I64 | ValueType::I128 | ValueType::U64 | ValueType::U128,
             ) if matches!(self.boolean_handling, BooleanHandling::Relax { .. })
                 && matches!(value.as_i64(), Some(0i64) | Some(1i64)) =>
@@ -224,7 +226,7 @@ impl JsonParseOptions {
                 (value.as_i64() == Some(1i64)).into()
             }
 
-            (Some(DataType::Boolean), ValueType::String)
+            (DataType::Boolean, ValueType::String)
                 if matches!(
                     self.boolean_handling,
                     BooleanHandling::Relax {
@@ -256,11 +258,11 @@ impl JsonParseOptions {
             }
             // ---- Int16 -----
             (
-                Some(DataType::Int16),
+                DataType::Int16,
                 ValueType::I64 | ValueType::I128 | ValueType::U64 | ValueType::U128,
             ) => value.try_as_i16().map_err(|_| create_error())?.into(),
 
-            (Some(DataType::Int16), ValueType::String)
+            (DataType::Int16, ValueType::String)
                 if matches!(
                     self.numeric_handling,
                     NumericHandling::Relax {
@@ -277,11 +279,11 @@ impl JsonParseOptions {
             }
             // ---- Int32 -----
             (
-                Some(DataType::Int32),
+                DataType::Int32,
                 ValueType::I64 | ValueType::I128 | ValueType::U64 | ValueType::U128,
             ) => value.try_as_i32().map_err(|_| create_error())?.into(),
 
-            (Some(DataType::Int32), ValueType::String)
+            (DataType::Int32, ValueType::String)
                 if matches!(
                     self.numeric_handling,
                     NumericHandling::Relax {
@@ -297,15 +299,12 @@ impl JsonParseOptions {
                     .into()
             }
             // ---- Int64 -----
-            (None, ValueType::I64 | ValueType::U64) => {
-                value.try_as_i64().map_err(|_| create_error())?.into()
-            }
             (
-                Some(DataType::Int64),
+                DataType::Int64,
                 ValueType::I64 | ValueType::I128 | ValueType::U64 | ValueType::U128,
             ) => value.try_as_i64().map_err(|_| create_error())?.into(),
 
-            (Some(DataType::Int64), ValueType::String)
+            (DataType::Int64, ValueType::String)
                 if matches!(
                     self.numeric_handling,
                     NumericHandling::Relax {
@@ -322,12 +321,12 @@ impl JsonParseOptions {
             }
             // ---- Float32 -----
             (
-                Some(DataType::Float32),
+                DataType::Float32,
                 ValueType::I64 | ValueType::I128 | ValueType::U64 | ValueType::U128,
             ) if matches!(self.numeric_handling, NumericHandling::Relax { .. }) => {
                 (value.try_as_i64().map_err(|_| create_error())? as f32).into()
             }
-            (Some(DataType::Float32), ValueType::String)
+            (DataType::Float32, ValueType::String)
                 if matches!(
                     self.numeric_handling,
                     NumericHandling::Relax {
@@ -342,17 +341,17 @@ impl JsonParseOptions {
                     .map_err(|_| create_error())?
                     .into()
             }
-            (Some(DataType::Float32), ValueType::F64) => {
+            (DataType::Float32, ValueType::F64) => {
                 value.try_as_f32().map_err(|_| create_error())?.into()
             }
             // ---- Float64 -----
             (
-                Some(DataType::Float64),
+                DataType::Float64,
                 ValueType::I64 | ValueType::I128 | ValueType::U64 | ValueType::U128,
             ) if matches!(self.numeric_handling, NumericHandling::Relax { .. }) => {
                 (value.try_as_i64().map_err(|_| create_error())? as f64).into()
             }
-            (Some(DataType::Float64), ValueType::String)
+            (DataType::Float64, ValueType::String)
                 if matches!(
                     self.numeric_handling,
                     NumericHandling::Relax {
@@ -367,29 +366,37 @@ impl JsonParseOptions {
                     .map_err(|_| create_error())?
                     .into()
             }
-            (Some(DataType::Float64) | None, ValueType::F64) => {
+            (DataType::Float64, ValueType::F64) => {
                 value.try_as_f64().map_err(|_| create_error())?.into()
             }
             // ---- Decimal -----
-            (Some(DataType::Decimal) | None, ValueType::I128 | ValueType::U128) => {
+            (DataType::Decimal, ValueType::I128 | ValueType::U128) => {
                 Decimal::from_str(&value.try_as_i128().map_err(|_| create_error())?.to_string())
                     .map_err(|_| create_error())?
                     .into()
             }
-            (Some(DataType::Decimal), ValueType::I64 | ValueType::U64) => {
+            (DataType::Decimal, ValueType::I64 | ValueType::U64) => {
                 Decimal::from(value.try_as_i64().map_err(|_| create_error())?).into()
             }
 
-            (Some(DataType::Decimal), ValueType::F64) => {
+            (DataType::Decimal, ValueType::F64) => {
                 Decimal::try_from(value.try_as_f64().map_err(|_| create_error())?)
                     .map_err(|_| create_error())?
                     .into()
             }
-
-            (Some(DataType::Decimal), ValueType::String) => ScalarImpl::Decimal(
-                Decimal::from_str(value.as_str().unwrap()).map_err(|_err| create_error())?,
-            ),
-            (Some(DataType::Decimal), ValueType::Object) => {
+            (DataType::Decimal, ValueType::String) => {
+                let str = value.as_str().unwrap();
+                // the following values are special string generated by Debezium and should be handled separately
+                match str {
+                    "NAN" => ScalarImpl::Decimal(Decimal::NaN),
+                    "POSITIVE_INFINITY" => ScalarImpl::Decimal(Decimal::PositiveInf),
+                    "NEGATIVE_INFINITY" => ScalarImpl::Decimal(Decimal::NegativeInf),
+                    _ => {
+                        ScalarImpl::Decimal(Decimal::from_str(str).map_err(|_err| create_error())?)
+                    }
+                }
+            }
+            (DataType::Decimal, ValueType::Object) => {
                 // ref https://github.com/risingwavelabs/risingwave/issues/10628
                 // handle debezium json (variable scale): {"scale": int, "value": bytes}
                 let scale = value
@@ -412,21 +419,23 @@ impl JsonParseOptions {
             }
             // ---- Date -----
             (
-                Some(DataType::Date),
+                DataType::Date,
                 ValueType::I64 | ValueType::I128 | ValueType::U64 | ValueType::U128,
             ) => Date::with_days_since_unix_epoch(value.try_as_i32().map_err(|_| create_error())?)
                 .map_err(|_| create_error())?
                 .into(),
-            (Some(DataType::Date), ValueType::String) => value
+            (DataType::Date, ValueType::String) => value
                 .as_str()
                 .unwrap()
                 .parse::<Date>()
                 .map_err(|_| create_error())?
                 .into(),
             // ---- Varchar -----
-            (Some(DataType::Varchar) | None, ValueType::String) => value.as_str().unwrap().into(),
+            (DataType::Varchar, ValueType::String) => {
+                return Ok(DatumCow::Borrowed(Some(value.as_str().unwrap().into())))
+            }
             (
-                Some(DataType::Varchar),
+                DataType::Varchar,
                 ValueType::Bool
                 | ValueType::I64
                 | ValueType::I128
@@ -437,7 +446,7 @@ impl JsonParseOptions {
                 value.to_string().into()
             }
             (
-                Some(DataType::Varchar),
+                DataType::Varchar,
                 ValueType::Bool
                 | ValueType::I64
                 | ValueType::I128
@@ -450,14 +459,14 @@ impl JsonParseOptions {
                 value.to_string().into()
             }
             // ---- Time -----
-            (Some(DataType::Time), ValueType::String) => value
+            (DataType::Time, ValueType::String) => value
                 .as_str()
                 .unwrap()
                 .parse::<Time>()
                 .map_err(|_| create_error())?
                 .into(),
             (
-                Some(DataType::Time),
+                DataType::Time,
                 ValueType::I64 | ValueType::I128 | ValueType::U64 | ValueType::U128,
             ) => value
                 .as_i64()
@@ -469,20 +478,20 @@ impl JsonParseOptions {
                 .map_err(|_| create_error())?
                 .into(),
             // ---- Timestamp -----
-            (Some(DataType::Timestamp), ValueType::String) => value
+            (DataType::Timestamp, ValueType::String) => value
                 .as_str()
                 .unwrap()
                 .parse::<Timestamp>()
                 .map_err(|_| create_error())?
                 .into(),
             (
-                Some(DataType::Timestamp),
+                DataType::Timestamp,
                 ValueType::I64 | ValueType::I128 | ValueType::U64 | ValueType::U128,
             ) => i64_to_timestamp(value.as_i64().unwrap())
                 .map_err(|_| create_error())?
                 .into(),
             // ---- Timestamptz -----
-            (Some(DataType::Timestamptz), ValueType::String) => match self.timestamptz_handling {
+            (DataType::Timestamptz, ValueType::String) => match self.timestamptz_handling {
                 TimestamptzHandling::UtcWithoutSuffix => value
                     .as_str()
                     .unwrap()
@@ -497,11 +506,12 @@ impl JsonParseOptions {
                     .parse::<Timestamptz>()
                     .map_err(|_| create_error())?
                     .into(),
-            }
+            },
             (
-                Some(DataType::Timestamptz),
+                DataType::Timestamptz,
                 ValueType::I64 | ValueType::I128 | ValueType::U64 | ValueType::U128,
-            ) => value.as_i64()
+            ) => value
+                .as_i64()
                 .and_then(|num| match self.timestamptz_handling {
                     TimestamptzHandling::GuessNumberUnit => i64_to_timestamptz(num).ok(),
                     TimestamptzHandling::Micro => Some(Timestamptz::from_micros(num)),
@@ -512,16 +522,22 @@ impl JsonParseOptions {
                 .ok_or_else(create_error)?
                 .into(),
             // ---- Interval -----
-            (Some(DataType::Interval), ValueType::String) => {
-                value.as_str().unwrap().parse::<Interval>().map_err(|_| create_error())?.into()
-            }
+            (DataType::Interval, ValueType::String) => value
+                .as_str()
+                .unwrap()
+                .parse::<Interval>()
+                .map_err(|_| create_error())?
+                .into(),
             // ---- Struct -----
-            (Some(DataType::Struct(struct_type_info)), ValueType::Object) => StructValue::new(
-                struct_type_info
+            (DataType::Struct(struct_type_info), ValueType::Object) => {
+                // Collecting into a Result<Vec<_>> doesn't reserve the capacity in advance, so we `Vec::with_capacity` instead.
+                // https://github.com/rust-lang/rust/issues/48994
+                let mut fields = Vec::with_capacity(struct_type_info.types().len());
+                for (field_name, field_type) in struct_type_info
                     .names()
                     .zip_eq_fast(struct_type_info.types())
-                    .map(|(field_name, field_type)| {
-                        let field_value = json_object_get_case_insensitive(value, field_name)
+                {
+                    let field_value = json_object_get_case_insensitive(value, field_name)
                             .unwrap_or_else(|| {
                                 let error = AccessError::Undefined {
                                     name: field_name.to_owned(),
@@ -534,40 +550,34 @@ impl JsonParseOptions {
                                 }
                                 &BorrowedValue::Static(simd_json::StaticNode::Null)
                             });
-                        self.parse(field_value, Some(field_type))
-                    })
-                    .collect::<Result<_, _>>()?,
-            )
-            .into(),
-
-            (None, ValueType::Object) => StructValue::new(
-                value
-                    .as_object()
-                    .unwrap()
-                    .iter()
-                    .map(|(_field_name, field_value)| self.parse(field_value, None))
-                    .collect::<Result<_, _>>()?,
-            )
-            .into(),
+                    fields.push(
+                        self.parse(field_value, field_type)
+                            .map(|d| d.to_owned_datum())?,
+                    );
+                }
+                StructValue::new(fields).into()
+            }
 
             // String containing json object, e.g. "{\"a\": 1, \"b\": 2}"
             // Try to parse it as json object.
-            (Some(DataType::Struct(_)), ValueType::String)
+            (DataType::Struct(_), ValueType::String)
                 if matches!(self.struct_handling, StructHandling::AllowJsonString) =>
             {
                 // TODO: avoid copy by accepting `&mut BorrowedValue` in `parse` method.
                 let mut value = value.as_str().unwrap().as_bytes().to_vec();
                 let value =
                     simd_json::to_borrowed_value(&mut value[..]).map_err(|_| create_error())?;
-                return self.parse(&value, type_expected);
+                return self
+                    .parse(&value, type_expected)
+                    .map(|d| d.to_owned_datum().into());
             }
 
             // ---- List -----
-            (Some(DataType::List(item_type)), ValueType::Array) => ListValue::new({
+            (DataType::List(item_type), ValueType::Array) => ListValue::new({
                 let array = value.as_array().unwrap();
                 let mut builder = item_type.create_array_builder(array.len());
                 for v in array {
-                    let value = self.parse(v, Some(item_type))?;
+                    let value = self.parse(v, item_type)?;
                     builder.append(value);
                 }
                 builder.finish()
@@ -575,7 +585,7 @@ impl JsonParseOptions {
             .into(),
 
             // ---- Bytea -----
-            (Some(DataType::Bytea), ValueType::String) => match self.bytea_handling {
+            (DataType::Bytea, ValueType::String) => match self.bytea_handling {
                 ByteaHandling::Standard => str_to_bytea(value.as_str().unwrap())
                     .map_err(|_| create_error())?
                     .into(),
@@ -586,14 +596,14 @@ impl JsonParseOptions {
                     .into(),
             },
             // ---- Jsonb -----
-            (Some(DataType::Jsonb), ValueType::String)
+            (DataType::Jsonb, ValueType::String)
                 if matches!(self.json_value_handling, JsonValueHandling::AsString) =>
             {
                 JsonbVal::from_str(value.as_str().unwrap())
                     .map_err(|_| create_error())?
                     .into()
             }
-            (Some(DataType::Jsonb), _)
+            (DataType::Jsonb, _)
                 if matches!(self.json_value_handling, JsonValueHandling::AsValue) =>
             {
                 let value: serde_json::Value =
@@ -602,42 +612,37 @@ impl JsonParseOptions {
             }
             // ---- Int256 -----
             (
-                Some(DataType::Int256),
+                DataType::Int256,
                 ValueType::I64 | ValueType::I128 | ValueType::U64 | ValueType::U128,
             ) => Int256::from(value.try_as_i64().map_err(|_| create_error())?).into(),
 
-            (Some(DataType::Int256), ValueType::String) => {
-                Int256::from_str(value.as_str().unwrap())
-                    .map_err(|_| create_error())?
-                    .into()
-            }
+            (DataType::Int256, ValueType::String) => Int256::from_str(value.as_str().unwrap())
+                .map_err(|_| create_error())?
+                .into(),
 
             (_expected, _got) => Err(create_error())?,
         };
-        Ok(Some(v))
+        Ok(DatumCow::Owned(Some(v)))
     }
 }
 
-pub struct JsonAccess<'a, 'b> {
-    value: BorrowedValue<'b>,
+pub struct JsonAccess<'a> {
+    value: BorrowedValue<'a>,
     options: &'a JsonParseOptions,
 }
 
-impl<'a, 'b> JsonAccess<'a, 'b> {
-    pub fn new_with_options(value: BorrowedValue<'b>, options: &'a JsonParseOptions) -> Self {
+impl<'a> JsonAccess<'a> {
+    pub fn new_with_options(value: BorrowedValue<'a>, options: &'a JsonParseOptions) -> Self {
         Self { value, options }
     }
 
-    pub fn new(value: BorrowedValue<'b>) -> Self {
+    pub fn new(value: BorrowedValue<'a>) -> Self {
         Self::new_with_options(value, &JsonParseOptions::DEFAULT)
     }
 }
 
-impl<'a, 'b> Access for JsonAccess<'a, 'b>
-where
-    'a: 'b,
-{
-    fn access(&self, path: &[&str], type_expected: Option<&DataType>) -> AccessResult {
+impl Access for JsonAccess<'_> {
+    fn access<'a>(&'a self, path: &[&str], type_expected: &DataType) -> AccessResult<DatumCow<'a>> {
         let mut value = &self.value;
         for (idx, &key) in path.iter().enumerate() {
             if let Some(sub_value) = if self.options.ignoring_keycase {
diff --git a/src/connector/src/parser/unified/kv_event.rs b/src/connector/src/parser/unified/kv_event.rs
new file mode 100644
index 0000000000000..7e52d2f4c3c24
--- /dev/null
+++ b/src/connector/src/parser/unified/kv_event.rs
@@ -0,0 +1,86 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::types::{DataType, DatumCow};
+use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType;
+
+use super::{Access, AccessResult};
+use crate::parser::unified::AccessError;
+use crate::source::SourceColumnDesc;
+
+pub struct KvEvent<K, V> {
+    key_accessor: Option<K>,
+    value_accessor: Option<V>,
+}
+
+impl<K, V> Default for KvEvent<K, V> {
+    fn default() -> Self {
+        Self {
+            key_accessor: None,
+            value_accessor: None,
+        }
+    }
+}
+
+impl<K, V> KvEvent<K, V> {
+    pub fn with_key(&mut self, key: K)
+    where
+        K: Access,
+    {
+        self.key_accessor = Some(key);
+    }
+
+    pub fn with_value(&mut self, value: V)
+    where
+        V: Access,
+    {
+        self.value_accessor = Some(value);
+    }
+}
+
+impl<K, V> KvEvent<K, V>
+where
+    K: Access,
+    V: Access,
+{
+    fn access_key(&self, path: &[&str], type_expected: &DataType) -> AccessResult<DatumCow<'_>> {
+        if let Some(ka) = &self.key_accessor {
+            ka.access(path, type_expected)
+        } else {
+            Err(AccessError::Undefined {
+                name: "key".to_string(),
+                path: String::new(),
+            })
+        }
+    }
+
+    fn access_value(&self, path: &[&str], type_expected: &DataType) -> AccessResult<DatumCow<'_>> {
+        if let Some(va) = &self.value_accessor {
+            va.access(path, type_expected)
+        } else {
+            Err(AccessError::Undefined {
+                name: "value".to_string(),
+                path: String::new(),
+            })
+        }
+    }
+
+    pub fn access_field(&self, desc: &SourceColumnDesc) -> AccessResult<DatumCow<'_>> {
+        match desc.additional_column.column_type {
+            Some(AdditionalColumnType::Key(_)) => self.access_key(&[&desc.name], &desc.data_type),
+            None => self.access_value(&[&desc.name], &desc.data_type),
+            _ => unreachable!(),
+        }
+    }
+}
diff --git a/src/connector/src/parser/unified/maxwell.rs b/src/connector/src/parser/unified/maxwell.rs
index 1e26ced63c5d7..0bc6e7ab1d94a 100644
--- a/src/connector/src/parser/unified/maxwell.rs
+++ b/src/connector/src/parser/unified/maxwell.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use risingwave_common::types::{DataType, ScalarImpl};
+use risingwave_common::types::{DataType, DatumCow, ScalarRefImpl, ToDatumRef};
 
 use super::{Access, ChangeEvent};
 use crate::parser::unified::ChangeEventOperation;
@@ -36,8 +36,10 @@ where
 {
     fn op(&self) -> std::result::Result<super::ChangeEventOperation, super::AccessError> {
         const OP: &str = "type";
-        if let Some(ScalarImpl::Utf8(op)) = self.0.access(&[OP], Some(&DataType::Varchar))? {
-            match op.as_ref() {
+        if let Some(ScalarRefImpl::Utf8(op)) =
+            self.0.access(&[OP], &DataType::Varchar)?.to_datum_ref()
+        {
+            match op {
                 MAXWELL_INSERT_OP | MAXWELL_UPDATE_OP => return Ok(ChangeEventOperation::Upsert),
                 MAXWELL_DELETE_OP => return Ok(ChangeEventOperation::Delete),
                 _ => (),
@@ -49,8 +51,8 @@ where
         })
     }
 
-    fn access_field(&self, desc: &SourceColumnDesc) -> super::AccessResult {
+    fn access_field(&self, desc: &SourceColumnDesc) -> super::AccessResult<DatumCow<'_>> {
         const DATA: &str = "data";
-        self.0.access(&[DATA, &desc.name], Some(&desc.data_type))
+        self.0.access(&[DATA, &desc.name], &desc.data_type)
     }
 }
diff --git a/src/connector/src/parser/unified/mod.rs b/src/connector/src/parser/unified/mod.rs
index 31667cd42e47d..6c8316042026f 100644
--- a/src/connector/src/parser/unified/mod.rs
+++ b/src/connector/src/parser/unified/mod.rs
@@ -15,9 +15,10 @@
 //! Unified parsers for both normal events or CDC events of multiple message formats
 
 use auto_impl::auto_impl;
-use risingwave_common::types::{DataType, Datum};
-use thiserror::Error;
-use thiserror_ext::Macro;
+use risingwave_common::types::{DataType, DatumCow};
+pub use risingwave_connector_codec::decoder::{
+    bail_uncategorized, uncategorized, Access, AccessError, AccessResult,
+};
 
 use self::avro::AvroAccess;
 use self::bytes::BytesAccess;
@@ -30,28 +31,21 @@ pub mod avro;
 pub mod bytes;
 pub mod debezium;
 pub mod json;
+pub mod kv_event;
 pub mod maxwell;
 pub mod protobuf;
-pub mod upsert;
 pub mod util;
 
-pub type AccessResult<T = Datum> = std::result::Result<T, AccessError>;
-
-/// Access a certain field in an object according to the path
-pub trait Access {
-    fn access(&self, path: &[&str], type_expected: Option<&DataType>) -> AccessResult;
-}
-
-pub enum AccessImpl<'a, 'b> {
-    Avro(AvroAccess<'a, 'b>),
+pub enum AccessImpl<'a> {
+    Avro(AvroAccess<'a>),
     Bytes(BytesAccess<'a>),
     Protobuf(ProtobufAccess),
-    Json(JsonAccess<'a, 'b>),
-    MongoJson(MongoJsonAccess<JsonAccess<'a, 'b>>),
+    Json(JsonAccess<'a>),
+    MongoJson(MongoJsonAccess<JsonAccess<'a>>),
 }
 
-impl Access for AccessImpl<'_, '_> {
-    fn access(&self, path: &[&str], type_expected: Option<&DataType>) -> AccessResult {
+impl Access for AccessImpl<'_> {
+    fn access<'a>(&'a self, path: &[&str], type_expected: &DataType) -> AccessResult<DatumCow<'a>> {
         match self {
             Self::Avro(accessor) => accessor.access(path, type_expected),
             Self::Bytes(accessor) => accessor.access(path, type_expected),
@@ -72,39 +66,20 @@ pub enum ChangeEventOperation {
 #[auto_impl(&)]
 pub trait ChangeEvent {
     /// Access the operation type.
-    fn op(&self) -> std::result::Result<ChangeEventOperation, AccessError>;
-    /// Access the field after the operation.
-    fn access_field(&self, desc: &SourceColumnDesc) -> AccessResult;
+    fn op(&self) -> AccessResult<ChangeEventOperation>;
+    /// Access the field.
+    fn access_field(&self, desc: &SourceColumnDesc) -> AccessResult<DatumCow<'_>>;
 }
 
 impl<A> ChangeEvent for (ChangeEventOperation, A)
 where
     A: Access,
 {
-    fn op(&self) -> std::result::Result<ChangeEventOperation, AccessError> {
+    fn op(&self) -> AccessResult<ChangeEventOperation> {
         Ok(self.0)
     }
 
-    fn access_field(&self, desc: &SourceColumnDesc) -> AccessResult {
-        self.1.access(&[desc.name.as_str()], Some(&desc.data_type))
+    fn access_field(&self, desc: &SourceColumnDesc) -> AccessResult<DatumCow<'_>> {
+        self.1.access(&[desc.name.as_str()], &desc.data_type)
     }
 }
-
-#[derive(Error, Debug, Macro)]
-#[thiserror_ext(macro(mangle))]
-pub enum AccessError {
-    #[error("Undefined field `{name}` at `{path}`")]
-    Undefined { name: String, path: String },
-    #[error("Expected type `{expected}` but got `{got}` for `{value}`")]
-    TypeError {
-        expected: String,
-        got: String,
-        value: String,
-    },
-    #[error("Unsupported data type `{ty}`")]
-    UnsupportedType { ty: String },
-
-    /// Errors that are not categorized into variants above.
-    #[error("{message}")]
-    Uncategorized { message: String },
-}
diff --git a/src/connector/src/parser/unified/protobuf.rs b/src/connector/src/parser/unified/protobuf.rs
index cd9178c7dd08d..02febc22db247 100644
--- a/src/connector/src/parser/unified/protobuf.rs
+++ b/src/connector/src/parser/unified/protobuf.rs
@@ -12,16 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::borrow::Cow;
 use std::sync::{Arc, LazyLock};
 
 use prost_reflect::{DescriptorPool, DynamicMessage, ReflectMessage};
 use risingwave_common::log::LogSuppresser;
-use risingwave_common::types::DataType;
+use risingwave_common::types::{DataType, DatumCow, ToOwnedDatum};
 use thiserror_ext::AsReport;
 
 use super::{Access, AccessResult};
 use crate::parser::from_protobuf_value;
-use crate::parser::unified::{uncategorized, AccessError};
+use crate::parser::unified::uncategorized;
 
 pub struct ProtobufAccess {
     message: DynamicMessage,
@@ -38,7 +39,11 @@ impl ProtobufAccess {
 }
 
 impl Access for ProtobufAccess {
-    fn access(&self, path: &[&str], _type_expected: Option<&DataType>) -> AccessResult {
+    fn access<'a>(
+        &'a self,
+        path: &[&str],
+        _type_expected: &DataType,
+    ) -> AccessResult<DatumCow<'a>> {
         debug_assert_eq!(1, path.len());
         let field_desc = self
             .message
@@ -52,8 +57,14 @@ impl Access for ProtobufAccess {
                     tracing::error!(suppressed_count, "{}", e.as_report());
                 }
             })?;
-        let value = self.message.get_field(&field_desc);
 
-        from_protobuf_value(&field_desc, &value, &self.descriptor_pool)
+        match self.message.get_field(&field_desc) {
+            Cow::Borrowed(value) => from_protobuf_value(&field_desc, value, &self.descriptor_pool),
+
+            // `Owned` variant occurs only if there's no such field and the default value is returned.
+            Cow::Owned(value) => from_protobuf_value(&field_desc, &value, &self.descriptor_pool)
+                // enforce `Owned` variant to avoid returning a reference to a temporary value
+                .map(|d| d.to_owned_datum().into()),
+        }
     }
 }
diff --git a/src/connector/src/parser/unified/upsert.rs b/src/connector/src/parser/unified/upsert.rs
deleted file mode 100644
index 9129f0d16d864..0000000000000
--- a/src/connector/src/parser/unified/upsert.rs
+++ /dev/null
@@ -1,122 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use risingwave_common::types::DataType;
-use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType;
-
-use super::{Access, ChangeEvent, ChangeEventOperation};
-use crate::parser::unified::AccessError;
-use crate::source::SourceColumnDesc;
-
-/// `UpsertAccess` wraps a key-value message format into an upsert source.
-/// A key accessor and a value accessor are required.
-pub struct UpsertChangeEvent<K, V> {
-    key_accessor: Option<K>,
-    value_accessor: Option<V>,
-    key_column_name: Option<String>,
-}
-
-impl<K, V> Default for UpsertChangeEvent<K, V> {
-    fn default() -> Self {
-        Self {
-            key_accessor: None,
-            value_accessor: None,
-            key_column_name: None,
-        }
-    }
-}
-
-impl<K, V> UpsertChangeEvent<K, V> {
-    pub fn with_key(mut self, key: K) -> Self
-    where
-        K: Access,
-    {
-        self.key_accessor = Some(key);
-        self
-    }
-
-    pub fn with_value(mut self, value: V) -> Self
-    where
-        V: Access,
-    {
-        self.value_accessor = Some(value);
-        self
-    }
-
-    pub fn with_key_column_name(mut self, name: impl ToString) -> Self {
-        self.key_column_name = Some(name.to_string());
-        self
-    }
-}
-
-impl<K, V> Access for UpsertChangeEvent<K, V>
-where
-    K: Access,
-    V: Access,
-{
-    fn access(&self, path: &[&str], type_expected: Option<&DataType>) -> super::AccessResult {
-        let create_error = |name: String| AccessError::Undefined {
-            name,
-            path: String::new(),
-        };
-        match path.first() {
-            Some(&"key") => {
-                if let Some(ka) = &self.key_accessor {
-                    ka.access(&path[1..], type_expected)
-                } else {
-                    Err(create_error("key".to_string()))
-                }
-            }
-            Some(&"value") => {
-                if let Some(va) = &self.value_accessor {
-                    va.access(&path[1..], type_expected)
-                } else {
-                    Err(create_error("value".to_string()))
-                }
-            }
-            None => Ok(None),
-            Some(other) => Err(create_error(other.to_string())),
-        }
-    }
-}
-
-impl<K, V> ChangeEvent for UpsertChangeEvent<K, V>
-where
-    K: Access,
-    V: Access,
-{
-    fn op(&self) -> std::result::Result<ChangeEventOperation, AccessError> {
-        if let Ok(Some(_)) = self.access(&["value"], None) {
-            Ok(ChangeEventOperation::Upsert)
-        } else {
-            Ok(ChangeEventOperation::Delete)
-        }
-    }
-
-    fn access_field(&self, desc: &SourceColumnDesc) -> super::AccessResult {
-        match desc.additional_column.column_type {
-            Some(AdditionalColumnType::Key(_)) => {
-                if let Some(key_as_column_name) = &self.key_column_name
-                    && &desc.name == key_as_column_name
-                {
-                    self.access(&["key"], Some(&desc.data_type))
-                } else {
-                    self.access(&["key", &desc.name], Some(&desc.data_type))
-                }
-            }
-            None => self.access(&["value", &desc.name], Some(&desc.data_type)),
-            _ => unreachable!(),
-        }
-    }
-}
diff --git a/src/connector/src/parser/unified/util.rs b/src/connector/src/parser/unified/util.rs
index 948190edf685a..2ed7e1fbefd4e 100644
--- a/src/connector/src/parser/unified/util.rs
+++ b/src/connector/src/parser/unified/util.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::{Access, AccessResult, ChangeEvent};
+use super::{AccessResult, ChangeEvent};
 use crate::parser::unified::ChangeEventOperation;
 use crate::parser::SourceStreamChunkRowWriter;
 use crate::source::SourceColumnDesc;
@@ -24,8 +24,8 @@ pub fn apply_row_operation_on_stream_chunk_writer_with_op(
 ) -> AccessResult<()> {
     let f = |column: &SourceColumnDesc| row_op.access_field(column);
     match op {
-        ChangeEventOperation::Upsert => writer.insert(f),
-        ChangeEventOperation::Delete => writer.delete(f),
+        ChangeEventOperation::Upsert => writer.do_insert(f),
+        ChangeEventOperation::Delete => writer.do_delete(f),
     }
 }
 
@@ -36,10 +36,3 @@ pub fn apply_row_operation_on_stream_chunk_writer(
     let op = row_op.op()?;
     apply_row_operation_on_stream_chunk_writer_with_op(row_op, writer, op)
 }
-
-pub fn apply_row_accessor_on_stream_chunk_writer(
-    accessor: impl Access,
-    writer: &mut SourceStreamChunkRowWriter<'_>,
-) -> AccessResult<()> {
-    writer.insert(|column| accessor.access(&[&column.name], Some(&column.data_type)))
-}
diff --git a/src/connector/src/parser/upsert_parser.rs b/src/connector/src/parser/upsert_parser.rs
index 048fd0beca3ff..df5e3b66e3136 100644
--- a/src/connector/src/parser/upsert_parser.rs
+++ b/src/connector/src/parser/upsert_parser.rs
@@ -16,14 +16,13 @@ use risingwave_common::bail;
 use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType;
 
 use super::bytes_parser::BytesAccessBuilder;
-use super::unified::upsert::UpsertChangeEvent;
-use super::unified::util::apply_row_operation_on_stream_chunk_writer_with_op;
 use super::unified::{AccessImpl, ChangeEventOperation};
 use super::{
     AccessBuilderImpl, ByteStreamSourceParser, BytesProperties, EncodingProperties, EncodingType,
     SourceStreamChunkRowWriter, SpecificParserConfig,
 };
 use crate::error::ConnectorResult;
+use crate::parser::unified::kv_event::KvEvent;
 use crate::parser::ParserFormat;
 use crate::source::{SourceColumnDesc, SourceContext, SourceContextRef};
 
@@ -97,22 +96,26 @@ impl UpsertParser {
         payload: Option<Vec<u8>>,
         mut writer: SourceStreamChunkRowWriter<'_>,
     ) -> ConnectorResult<()> {
-        let mut row_op: UpsertChangeEvent<AccessImpl<'_, '_>, AccessImpl<'_, '_>> =
-            UpsertChangeEvent::default();
-        let mut change_event_op = ChangeEventOperation::Delete;
+        let mut row_op: KvEvent<AccessImpl<'_>, AccessImpl<'_>> = KvEvent::default();
         if let Some(data) = key {
-            row_op = row_op.with_key(self.key_builder.generate_accessor(data).await?);
+            row_op.with_key(self.key_builder.generate_accessor(data).await?);
         }
         // Empty payload of kafka is Some(vec![])
+        let change_event_op;
         if let Some(data) = payload
             && !data.is_empty()
         {
-            row_op = row_op.with_value(self.payload_builder.generate_accessor(data).await?);
+            row_op.with_value(self.payload_builder.generate_accessor(data).await?);
             change_event_op = ChangeEventOperation::Upsert;
+        } else {
+            change_event_op = ChangeEventOperation::Delete;
         }
-
-        apply_row_operation_on_stream_chunk_writer_with_op(row_op, &mut writer, change_event_op)
-            .map_err(Into::into)
+        let f = |column: &SourceColumnDesc| row_op.access_field(column);
+        match change_event_op {
+            ChangeEventOperation::Upsert => writer.do_insert(f)?,
+            ChangeEventOperation::Delete => writer.do_delete(f)?,
+        }
+        Ok(())
     }
 }
 
diff --git a/src/connector/src/parser/util.rs b/src/connector/src/parser/util.rs
index 27eb7a8144bdd..30cb1fbf7d62e 100644
--- a/src/connector/src/parser/util.rs
+++ b/src/connector/src/parser/util.rs
@@ -11,13 +11,13 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 
 use anyhow::Context;
 use bytes::Bytes;
 use reqwest::Url;
 use risingwave_common::bail;
-use risingwave_common::types::Datum;
+use risingwave_common::types::{Datum, DatumCow, DatumRef};
 use risingwave_pb::data::DataType as PbDataType;
 
 use crate::aws_utils::load_file_descriptor_from_s3;
@@ -38,9 +38,14 @@ macro_rules! log_error {
     };
 }
 pub(crate) use log_error;
+use risingwave_pb::plan_common::additional_column;
+use risingwave_pb::plan_common::additional_column::ColumnType;
+
+use crate::parser::{AccessError, AccessResult};
+use crate::source::cdc::DebeziumCdcMeta;
 
 /// get kafka topic name
-pub(super) fn get_kafka_topic(props: &HashMap<String, String>) -> ConnectorResult<&String> {
+pub(super) fn get_kafka_topic(props: &BTreeMap<String, String>) -> ConnectorResult<&String> {
     const KAFKA_TOPIC_KEY1: &str = "kafka.topic";
     const KAFKA_TOPIC_KEY2: &str = "topic";
 
@@ -127,13 +132,29 @@ pub(super) async fn bytes_from_url(
     }
 }
 
-pub fn extreact_timestamp_from_meta(meta: &SourceMeta) -> Option<Datum> {
+pub fn extreact_timestamp_from_meta(meta: &SourceMeta) -> Option<DatumRef<'_>> {
     match meta {
         SourceMeta::Kafka(kafka_meta) => kafka_meta.extract_timestamp(),
+        SourceMeta::DebeziumCdc(cdc_meta) => Some(cdc_meta.extract_timestamp()),
         _ => None,
     }
 }
 
+pub fn extract_cdc_meta_column<'a>(
+    cdc_meta: &'a DebeziumCdcMeta,
+    column_type: &additional_column::ColumnType,
+    column_name: &str,
+) -> AccessResult<DatumRef<'a>> {
+    match column_type {
+        ColumnType::Timestamp(_) => Ok(cdc_meta.extract_timestamp()),
+        ColumnType::DatabaseName(_) => Ok(cdc_meta.extract_database_name()),
+        ColumnType::TableName(_) => Ok(cdc_meta.extract_table_name()),
+        _ => Err(AccessError::UnsupportedAdditionalColumn {
+            name: column_name.to_string(),
+        }),
+    }
+}
+
 pub fn extract_headers_from_meta(meta: &SourceMeta) -> Option<Datum> {
     match meta {
         SourceMeta::Kafka(kafka_meta) => kafka_meta.extract_headers(), /* expect output of type `array[struct<varchar, bytea>]` */
@@ -141,11 +162,11 @@ pub fn extract_headers_from_meta(meta: &SourceMeta) -> Option<Datum> {
     }
 }
 
-pub fn extract_header_inner_from_meta(
-    meta: &SourceMeta,
+pub fn extract_header_inner_from_meta<'a>(
+    meta: &'a SourceMeta,
     inner_field: &str,
     data_type: Option<&PbDataType>,
-) -> Option<Datum> {
+) -> Option<DatumCow<'a>> {
     match meta {
         SourceMeta::Kafka(kafka_meta) => kafka_meta.extract_header_inner(inner_field, data_type), /* expect output of type `bytea` or `varchar` */
         _ => None,
diff --git a/src/connector/src/schema/avro.rs b/src/connector/src/schema/avro.rs
index 22c5fb4acadd1..557953dc5b342 100644
--- a/src/connector/src/schema/avro.rs
+++ b/src/connector/src/schema/avro.rs
@@ -12,42 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
-use std::sync::Arc;
-
 use apache_avro::Schema as AvroSchema;
 
-use super::loader::{LoadedSchema, SchemaLoader};
+use super::loader::LoadedSchema;
 use super::schema_registry::Subject;
 use super::SchemaFetchError;
 
-pub struct SchemaWithId {
-    pub schema: Arc<AvroSchema>,
-    pub id: i32,
-}
-
-/// Schema registry only
-pub async fn fetch_schema(
-    format_options: &BTreeMap<String, String>,
-    topic: &str,
-) -> Result<(SchemaWithId, SchemaWithId), SchemaFetchError> {
-    let loader = SchemaLoader::from_format_options(topic, format_options)?;
-
-    let (key_id, key_avro) = loader.load_key_schema().await?;
-    let (val_id, val_avro) = loader.load_val_schema().await?;
-
-    Ok((
-        SchemaWithId {
-            id: key_id,
-            schema: Arc::new(key_avro),
-        },
-        SchemaWithId {
-            id: val_id,
-            schema: Arc::new(val_avro),
-        },
-    ))
-}
-
 impl LoadedSchema for AvroSchema {
     fn compile(primary: Subject, _: Vec<Subject>) -> Result<Self, SchemaFetchError> {
         AvroSchema::parse_str(&primary.schema.content)
diff --git a/src/connector/src/schema/loader.rs b/src/connector/src/schema/loader.rs
index a50d8cced575b..0deb5c1e4d0ce 100644
--- a/src/connector/src/schema/loader.rs
+++ b/src/connector/src/schema/loader.rs
@@ -19,13 +19,10 @@ use risingwave_pb::catalog::PbSchemaRegistryNameStrategy;
 use super::schema_registry::{
     get_subject_by_strategy, handle_sr_list, name_strategy_from_str, Client, Subject,
 };
-use super::{invalid_option_error, InvalidOptionError, SchemaFetchError};
-
-const MESSAGE_NAME_KEY: &str = "message";
-const KEY_MESSAGE_NAME_KEY: &str = "key.message";
-const SCHEMA_LOCATION_KEY: &str = "schema.location";
-const SCHEMA_REGISTRY_KEY: &str = "schema.registry";
-const NAME_STRATEGY_KEY: &str = "schema.registry.name.strategy";
+use super::{
+    invalid_option_error, InvalidOptionError, SchemaFetchError, KEY_MESSAGE_NAME_KEY,
+    MESSAGE_NAME_KEY, NAME_STRATEGY_KEY, SCHEMA_REGISTRY_KEY,
+};
 
 pub struct SchemaLoader {
     pub client: Client,
diff --git a/src/connector/src/schema/schema_registry/util.rs b/src/connector/src/schema/schema_registry/util.rs
index 0d43f33baa31c..44b7a350e6823 100644
--- a/src/connector/src/schema/schema_registry/util.rs
+++ b/src/connector/src/schema/schema_registry/util.rs
@@ -150,6 +150,7 @@ pub struct Subject {
 #[derive(Debug, Deserialize)]
 pub struct SchemaReference {
     /// The name of the reference.
+    #[allow(dead_code)]
     pub name: String,
     /// The subject that the referenced schema belongs to
     pub subject: String,
diff --git a/src/connector/src/sink/big_query.rs b/src/connector/src/sink/big_query.rs
index e7f74614c9cf0..b2376246a52f7 100644
--- a/src/connector/src/sink/big_query.rs
+++ b/src/connector/src/sink/big_query.rs
@@ -13,27 +13,43 @@
 // limitations under the License.
 
 use core::mem;
-use std::collections::HashMap;
+use core::time::Duration;
+use std::collections::{BTreeMap, HashMap};
 use std::sync::Arc;
 
 use anyhow::anyhow;
 use async_trait::async_trait;
 use gcp_bigquery_client::model::query_request::QueryRequest;
-use gcp_bigquery_client::model::table_data_insert_all_request::TableDataInsertAllRequest;
-use gcp_bigquery_client::model::table_data_insert_all_request_rows::TableDataInsertAllRequestRows;
 use gcp_bigquery_client::Client;
+use google_cloud_bigquery::grpc::apiv1::bigquery_client::StreamingWriteClient;
+use google_cloud_bigquery::grpc::apiv1::conn_pool::{WriteConnectionManager, DOMAIN};
+use google_cloud_gax::conn::{ConnectionOptions, Environment};
+use google_cloud_gax::grpc::Request;
+use google_cloud_googleapis::cloud::bigquery::storage::v1::append_rows_request::{
+    ProtoData, Rows as AppendRowsRequestRows,
+};
+use google_cloud_googleapis::cloud::bigquery::storage::v1::{
+    AppendRowsRequest, ProtoRows, ProtoSchema,
+};
+use google_cloud_pubsub::client::google_cloud_auth;
+use google_cloud_pubsub::client::google_cloud_auth::credentials::CredentialsFile;
+use prost_reflect::{FieldDescriptor, MessageDescriptor};
+use prost_types::{
+    field_descriptor_proto, DescriptorProto, FieldDescriptorProto, FileDescriptorProto,
+    FileDescriptorSet,
+};
 use risingwave_common::array::{Op, StreamChunk};
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_common::types::DataType;
 use serde_derive::Deserialize;
-use serde_json::Value;
 use serde_with::{serde_as, DisplayFromStr};
 use url::Url;
+use uuid::Uuid;
 use with_options::WithOptions;
 use yup_oauth2::ServiceAccountKey;
 
-use super::encoder::{JsonEncoder, RowEncoder};
+use super::encoder::{ProtoEncoder, ProtoHeader, RowEncoder, SerTo};
 use super::writer::LogSinkerOf;
 use super::{SinkError, SINK_TYPE_APPEND_ONLY, SINK_TYPE_OPTION, SINK_TYPE_UPSERT};
 use crate::aws_utils::load_file_descriptor_from_s3;
@@ -44,6 +60,10 @@ use crate::sink::{
 };
 
 pub const BIGQUERY_SINK: &str = "bigquery";
+pub const CHANGE_TYPE: &str = "_CHANGE_TYPE";
+const DEFAULT_GRPC_CHANNEL_NUMS: usize = 4;
+const CONNECT_TIMEOUT: Option<Duration> = Some(Duration::from_secs(30));
+const CONNECTION_TIMEOUT: Option<Duration> = None;
 
 #[serde_as]
 #[derive(Deserialize, Debug, Clone, WithOptions)]
@@ -68,27 +88,44 @@ fn default_max_batch_rows() -> usize {
 }
 
 impl BigQueryCommon {
-    pub(crate) async fn build_client(&self, aws_auth_props: &AwsAuthProps) -> Result<Client> {
-        let service_account = if let Some(local_path) = &self.local_path {
-            let auth_json = std::fs::read_to_string(local_path)
-                .map_err(|err| SinkError::BigQuery(anyhow::anyhow!(err)))?;
-            serde_json::from_str::<ServiceAccountKey>(&auth_json)
-                .map_err(|err| SinkError::BigQuery(anyhow::anyhow!(err)))?
+    async fn build_client(&self, aws_auth_props: &AwsAuthProps) -> Result<Client> {
+        let auth_json = self.get_auth_json_from_path(aws_auth_props).await?;
+
+        let service_account = serde_json::from_str::<ServiceAccountKey>(&auth_json)
+            .map_err(|err| SinkError::BigQuery(anyhow::anyhow!(err)))?;
+        let client: Client = Client::from_service_account_key(service_account, false)
+            .await
+            .map_err(|err| SinkError::BigQuery(anyhow::anyhow!(err)))?;
+        Ok(client)
+    }
+
+    async fn build_writer_client(
+        &self,
+        aws_auth_props: &AwsAuthProps,
+    ) -> Result<StorageWriterClient> {
+        let auth_json = self.get_auth_json_from_path(aws_auth_props).await?;
+
+        let credentials_file = CredentialsFile::new_from_str(&auth_json)
+            .await
+            .map_err(|e| SinkError::BigQuery(e.into()))?;
+        let client = StorageWriterClient::new(credentials_file).await?;
+        Ok(client)
+    }
+
+    async fn get_auth_json_from_path(&self, aws_auth_props: &AwsAuthProps) -> Result<String> {
+        if let Some(local_path) = &self.local_path {
+            std::fs::read_to_string(local_path)
+                .map_err(|err| SinkError::BigQuery(anyhow::anyhow!(err)))
         } else if let Some(s3_path) = &self.s3_path {
             let url =
                 Url::parse(s3_path).map_err(|err| SinkError::BigQuery(anyhow::anyhow!(err)))?;
-            let auth_json = load_file_descriptor_from_s3(&url, aws_auth_props)
+            let auth_vec = load_file_descriptor_from_s3(&url, aws_auth_props)
                 .await
                 .map_err(|err| SinkError::BigQuery(anyhow::anyhow!(err)))?;
-            serde_json::from_slice::<ServiceAccountKey>(&auth_json)
-                .map_err(|err| SinkError::BigQuery(anyhow::anyhow!(err)))?
+            Ok(String::from_utf8(auth_vec).map_err(|e| SinkError::BigQuery(e.into()))?)
         } else {
-            return Err(SinkError::BigQuery(anyhow::anyhow!("`bigquery.local.path` and `bigquery.s3.path` set at least one, configure as needed.")));
-        };
-        let client: Client = Client::from_service_account_key(service_account, false)
-            .await
-            .map_err(|err| SinkError::BigQuery(anyhow::anyhow!(err)))?;
-        Ok(client)
+            Err(SinkError::BigQuery(anyhow::anyhow!("`bigquery.local.path` and `bigquery.s3.path` set at least one, configure as needed.")))
+        }
     }
 }
 
@@ -102,7 +139,7 @@ pub struct BigQueryConfig {
     pub r#type: String, // accept "append-only" or "upsert"
 }
 impl BigQueryConfig {
-    pub fn from_hashmap(properties: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(properties: BTreeMap<String, String>) -> Result<Self> {
         let config =
             serde_json::from_value::<BigQueryConfig>(serde_json::to_value(properties).unwrap())
                 .map_err(|e| SinkError::Config(anyhow!(e)))?;
@@ -187,9 +224,7 @@ impl BigQuerySink {
             DataType::Decimal => Ok("NUMERIC".to_owned()),
             DataType::Date => Ok("DATE".to_owned()),
             DataType::Varchar => Ok("STRING".to_owned()),
-            DataType::Time => Err(SinkError::BigQuery(anyhow::anyhow!(
-                "Bigquery cannot support Time"
-            ))),
+            DataType::Time => Ok("TIME".to_owned()),
             DataType::Timestamp => Ok("DATETIME".to_owned()),
             DataType::Timestamptz => Ok("TIMESTAMP".to_owned()),
             DataType::Interval => Ok("INTERVAL".to_owned()),
@@ -234,12 +269,10 @@ impl Sink for BigQuerySink {
     }
 
     async fn validate(&self) -> Result<()> {
-        if !self.is_append_only {
+        if !self.is_append_only && self.pk_indices.is_empty() {
             return Err(SinkError::Config(anyhow!(
-                "BigQuery sink don't support upsert"
-            )));
+                "Primary key not defined for upsert bigquery sink (please define in `primary_key` field)")));
         }
-
         let client = self
             .config
             .common
@@ -278,12 +311,20 @@ impl Sink for BigQuerySink {
 
 pub struct BigQuerySinkWriter {
     pub config: BigQueryConfig,
+    #[expect(dead_code)]
     schema: Schema,
+    #[expect(dead_code)]
     pk_indices: Vec<usize>,
-    client: Client,
+    client: StorageWriterClient,
     is_append_only: bool,
-    insert_request: TableDataInsertAllRequest,
-    row_encoder: JsonEncoder,
+    row_encoder: ProtoEncoder,
+    writer_pb_schema: ProtoSchema,
+    #[expect(dead_code)]
+    message_descriptor: MessageDescriptor,
+    write_stream: String,
+    proto_field: Option<FieldDescriptor>,
+    write_rows: Vec<AppendRowsRequestRows>,
+    write_rows_count: usize,
 }
 
 impl TryFrom<SinkParam> for BigQuerySink {
@@ -291,7 +332,7 @@ impl TryFrom<SinkParam> for BigQuerySink {
 
     fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
         let schema = param.schema();
-        let config = BigQueryConfig::from_hashmap(param.properties)?;
+        let config = BigQueryConfig::from_btreemap(param.properties)?;
         BigQuerySink::new(
             config,
             schema,
@@ -308,66 +349,131 @@ impl BigQuerySinkWriter {
         pk_indices: Vec<usize>,
         is_append_only: bool,
     ) -> Result<Self> {
-        let client = config.common.build_client(&config.aws_auth_props).await?;
+        let client = config
+            .common
+            .build_writer_client(&config.aws_auth_props)
+            .await?;
+        let mut descriptor_proto = build_protobuf_schema(
+            schema
+                .fields()
+                .iter()
+                .map(|f| (f.name.as_str(), &f.data_type)),
+            config.common.table.clone(),
+        )?;
+
+        if !is_append_only {
+            let field = FieldDescriptorProto {
+                name: Some(CHANGE_TYPE.to_string()),
+                number: Some((schema.len() + 1) as i32),
+                r#type: Some(field_descriptor_proto::Type::String.into()),
+                ..Default::default()
+            };
+            descriptor_proto.field.push(field);
+        }
+
+        let descriptor_pool = build_protobuf_descriptor_pool(&descriptor_proto);
+        let message_descriptor = descriptor_pool
+            .get_message_by_name(&config.common.table)
+            .ok_or_else(|| {
+                SinkError::BigQuery(anyhow::anyhow!(
+                    "Can't find message proto {}",
+                    &config.common.table
+                ))
+            })?;
+        let proto_field = if !is_append_only {
+            let proto_field = message_descriptor
+                .get_field_by_name(CHANGE_TYPE)
+                .ok_or_else(|| {
+                    SinkError::BigQuery(anyhow::anyhow!("Can't find {}", CHANGE_TYPE))
+                })?;
+            Some(proto_field)
+        } else {
+            None
+        };
+        let row_encoder = ProtoEncoder::new(
+            schema.clone(),
+            None,
+            message_descriptor.clone(),
+            ProtoHeader::None,
+        )?;
         Ok(Self {
+            write_stream: format!(
+                "projects/{}/datasets/{}/tables/{}/streams/_default",
+                config.common.project, config.common.dataset, config.common.table
+            ),
             config,
-            schema: schema.clone(),
+            schema,
             pk_indices,
             client,
             is_append_only,
-            insert_request: TableDataInsertAllRequest::new(),
-            row_encoder: JsonEncoder::new_with_bigquery(schema, None),
+            row_encoder,
+            message_descriptor,
+            proto_field,
+            writer_pb_schema: ProtoSchema {
+                proto_descriptor: Some(descriptor_proto),
+            },
+            write_rows: vec![],
+            write_rows_count: 0,
         })
     }
 
-    async fn append_only(&mut self, chunk: StreamChunk) -> Result<()> {
-        let mut insert_vec = Vec::with_capacity(chunk.capacity());
+    fn append_only(&mut self, chunk: StreamChunk) -> Result<Vec<Vec<u8>>> {
+        let mut serialized_rows: Vec<Vec<u8>> = Vec::with_capacity(chunk.capacity());
         for (op, row) in chunk.rows() {
             if op != Op::Insert {
-                return Err(SinkError::BigQuery(anyhow::anyhow!(
-                    "BigQuery sink don't support upsert"
-                )));
+                continue;
             }
-            insert_vec.push(TableDataInsertAllRequestRows {
-                insert_id: None,
-                json: Value::Object(self.row_encoder.encode(row)?),
-            })
+            serialized_rows.push(self.row_encoder.encode(row)?.ser_to()?)
         }
-        self.insert_request
-            .add_rows(insert_vec)
-            .map_err(|e| SinkError::BigQuery(e.into()))?;
-        if self
-            .insert_request
-            .len()
-            .ge(&self.config.common.max_batch_rows)
-        {
-            self.insert_data().await?;
-        }
-        Ok(())
+        Ok(serialized_rows)
     }
 
-    async fn insert_data(&mut self) -> Result<()> {
-        if !self.insert_request.is_empty() {
-            let insert_request =
-                mem::replace(&mut self.insert_request, TableDataInsertAllRequest::new());
-            let request = self
-                .client
-                .tabledata()
-                .insert_all(
-                    &self.config.common.project,
-                    &self.config.common.dataset,
-                    &self.config.common.table,
-                    insert_request,
-                )
-                .await
-                .map_err(|e| SinkError::BigQuery(e.into()))?;
-            if let Some(error) = request.insert_errors {
-                return Err(SinkError::BigQuery(anyhow::anyhow!(
-                    "Insert error: {:?}",
-                    error
-                )));
+    fn upsert(&mut self, chunk: StreamChunk) -> Result<Vec<Vec<u8>>> {
+        let mut serialized_rows: Vec<Vec<u8>> = Vec::with_capacity(chunk.capacity());
+        for (op, row) in chunk.rows() {
+            if op == Op::UpdateDelete {
+                continue;
             }
+            let mut pb_row = self.row_encoder.encode(row)?;
+            match op {
+                Op::Insert => pb_row
+                    .message
+                    .try_set_field(
+                        self.proto_field.as_ref().unwrap(),
+                        prost_reflect::Value::String("UPSERT".to_string()),
+                    )
+                    .map_err(|e| SinkError::BigQuery(e.into()))?,
+                Op::Delete => pb_row
+                    .message
+                    .try_set_field(
+                        self.proto_field.as_ref().unwrap(),
+                        prost_reflect::Value::String("DELETE".to_string()),
+                    )
+                    .map_err(|e| SinkError::BigQuery(e.into()))?,
+                Op::UpdateDelete => continue,
+                Op::UpdateInsert => pb_row
+                    .message
+                    .try_set_field(
+                        self.proto_field.as_ref().unwrap(),
+                        prost_reflect::Value::String("UPSERT".to_string()),
+                    )
+                    .map_err(|e| SinkError::BigQuery(e.into()))?,
+            };
+
+            serialized_rows.push(pb_row.ser_to()?)
         }
+        Ok(serialized_rows)
+    }
+
+    async fn write_rows(&mut self) -> Result<()> {
+        if self.write_rows.is_empty() {
+            return Ok(());
+        }
+        let rows = mem::take(&mut self.write_rows);
+        self.write_rows_count = 0;
+        self.client
+            .append_rows(rows, self.write_stream.clone())
+            .await?;
         Ok(())
     }
 }
@@ -375,13 +481,24 @@ impl BigQuerySinkWriter {
 #[async_trait]
 impl SinkWriter for BigQuerySinkWriter {
     async fn write_batch(&mut self, chunk: StreamChunk) -> Result<()> {
-        if self.is_append_only {
-            self.append_only(chunk).await
+        let serialized_rows = if self.is_append_only {
+            self.append_only(chunk)?
         } else {
-            Err(SinkError::BigQuery(anyhow::anyhow!(
-                "BigQuery sink don't support upsert"
-            )))
+            self.upsert(chunk)?
+        };
+        if !serialized_rows.is_empty() {
+            self.write_rows_count += serialized_rows.len();
+            let rows = AppendRowsRequestRows::ProtoRows(ProtoData {
+                writer_schema: Some(self.writer_pb_schema.clone()),
+                rows: Some(ProtoRows { serialized_rows }),
+            });
+            self.write_rows.push(rows);
+
+            if self.write_rows_count >= self.config.common.max_batch_rows {
+                self.write_rows().await?;
+            }
         }
+        Ok(())
     }
 
     async fn begin_epoch(&mut self, _epoch: u64) -> Result<()> {
@@ -392,8 +509,11 @@ impl SinkWriter for BigQuerySinkWriter {
         Ok(())
     }
 
-    async fn barrier(&mut self, _is_checkpoint: bool) -> Result<()> {
-        self.insert_data().await
+    async fn barrier(&mut self, is_checkpoint: bool) -> Result<()> {
+        if is_checkpoint {
+            self.write_rows().await?;
+        }
+        Ok(())
     }
 
     async fn update_vnode_bitmap(&mut self, _vnode_bitmap: Arc<Bitmap>) -> Result<()> {
@@ -401,11 +521,184 @@ impl SinkWriter for BigQuerySinkWriter {
     }
 }
 
+struct StorageWriterClient {
+    client: StreamingWriteClient,
+    #[expect(dead_code)]
+    environment: Environment,
+}
+impl StorageWriterClient {
+    pub async fn new(credentials: CredentialsFile) -> Result<Self> {
+        let ts_grpc = google_cloud_auth::token::DefaultTokenSourceProvider::new_with_credentials(
+            Self::bigquery_grpc_auth_config(),
+            Box::new(credentials),
+        )
+        .await
+        .map_err(|e| SinkError::BigQuery(e.into()))?;
+        let conn_options = ConnectionOptions {
+            connect_timeout: CONNECT_TIMEOUT,
+            timeout: CONNECTION_TIMEOUT,
+        };
+        let environment = Environment::GoogleCloud(Box::new(ts_grpc));
+        let conn = WriteConnectionManager::new(
+            DEFAULT_GRPC_CHANNEL_NUMS,
+            &environment,
+            DOMAIN,
+            &conn_options,
+        )
+        .await
+        .map_err(|e| SinkError::BigQuery(e.into()))?;
+        let client = conn.conn();
+        Ok(StorageWriterClient {
+            client,
+            environment,
+        })
+    }
+
+    pub async fn append_rows(
+        &mut self,
+        rows: Vec<AppendRowsRequestRows>,
+        write_stream: String,
+    ) -> Result<()> {
+        let mut resp_count = rows.len();
+        let append_req: Vec<AppendRowsRequest> = rows
+            .into_iter()
+            .map(|row| AppendRowsRequest {
+                write_stream: write_stream.clone(),
+                offset: None,
+                trace_id: Uuid::new_v4().hyphenated().to_string(),
+                missing_value_interpretations: HashMap::default(),
+                rows: Some(row),
+            })
+            .collect();
+        let mut resp = self
+            .client
+            .append_rows(Request::new(tokio_stream::iter(append_req)))
+            .await
+            .map_err(|e| SinkError::BigQuery(e.into()))?
+            .into_inner();
+        while let Some(append_rows_response) = resp
+            .message()
+            .await
+            .map_err(|e| SinkError::BigQuery(e.into()))?
+        {
+            resp_count -= 1;
+            if !append_rows_response.row_errors.is_empty() {
+                return Err(SinkError::BigQuery(anyhow::anyhow!(
+                    "Insert error {:?}",
+                    append_rows_response.row_errors
+                )));
+            }
+        }
+        assert_eq!(resp_count,0,"bigquery sink insert error: the number of response inserted is not equal to the number of request");
+        Ok(())
+    }
+
+    fn bigquery_grpc_auth_config() -> google_cloud_auth::project::Config<'static> {
+        google_cloud_auth::project::Config {
+            audience: Some(google_cloud_bigquery::grpc::apiv1::conn_pool::AUDIENCE),
+            scopes: Some(&google_cloud_bigquery::grpc::apiv1::conn_pool::SCOPES),
+            sub: None,
+        }
+    }
+}
+
+fn build_protobuf_descriptor_pool(desc: &DescriptorProto) -> prost_reflect::DescriptorPool {
+    let file_descriptor = FileDescriptorProto {
+        message_type: vec![desc.clone()],
+        name: Some("bigquery".to_string()),
+        ..Default::default()
+    };
+
+    prost_reflect::DescriptorPool::from_file_descriptor_set(FileDescriptorSet {
+        file: vec![file_descriptor],
+    })
+    .unwrap()
+}
+
+fn build_protobuf_schema<'a>(
+    fields: impl Iterator<Item = (&'a str, &'a DataType)>,
+    name: String,
+) -> Result<DescriptorProto> {
+    let mut proto = DescriptorProto {
+        name: Some(name),
+        ..Default::default()
+    };
+    let mut struct_vec = vec![];
+    let field_vec = fields
+        .enumerate()
+        .map(|(index, (name, data_type))| {
+            let (field, des_proto) =
+                build_protobuf_field(data_type, (index + 1) as i32, name.to_string())?;
+            if let Some(sv) = des_proto {
+                struct_vec.push(sv);
+            }
+            Ok(field)
+        })
+        .collect::<Result<Vec<_>>>()?;
+    proto.field = field_vec;
+    proto.nested_type = struct_vec;
+    Ok(proto)
+}
+
+fn build_protobuf_field(
+    data_type: &DataType,
+    index: i32,
+    name: String,
+) -> Result<(FieldDescriptorProto, Option<DescriptorProto>)> {
+    let mut field = FieldDescriptorProto {
+        name: Some(name.clone()),
+        number: Some(index),
+        ..Default::default()
+    };
+    match data_type {
+        DataType::Boolean => field.r#type = Some(field_descriptor_proto::Type::Bool.into()),
+        DataType::Int32 => field.r#type = Some(field_descriptor_proto::Type::Int32.into()),
+        DataType::Int16 | DataType::Int64 => {
+            field.r#type = Some(field_descriptor_proto::Type::Int64.into())
+        }
+        DataType::Float64 => field.r#type = Some(field_descriptor_proto::Type::Double.into()),
+        DataType::Decimal => field.r#type = Some(field_descriptor_proto::Type::String.into()),
+        DataType::Date => field.r#type = Some(field_descriptor_proto::Type::Int32.into()),
+        DataType::Varchar => field.r#type = Some(field_descriptor_proto::Type::String.into()),
+        DataType::Time => field.r#type = Some(field_descriptor_proto::Type::String.into()),
+        DataType::Timestamp => field.r#type = Some(field_descriptor_proto::Type::String.into()),
+        DataType::Timestamptz => field.r#type = Some(field_descriptor_proto::Type::String.into()),
+        DataType::Interval => field.r#type = Some(field_descriptor_proto::Type::String.into()),
+        DataType::Struct(s) => {
+            field.r#type = Some(field_descriptor_proto::Type::Message.into());
+            let name = format!("Struct{}", name);
+            let sub_proto = build_protobuf_schema(s.iter(), name.clone())?;
+            field.type_name = Some(name);
+            return Ok((field, Some(sub_proto)));
+        }
+        DataType::List(l) => {
+            let (mut field, proto) = build_protobuf_field(l.as_ref(), index, name.clone())?;
+            field.label = Some(field_descriptor_proto::Label::Repeated.into());
+            return Ok((field, proto));
+        }
+        DataType::Bytea => field.r#type = Some(field_descriptor_proto::Type::Bytes.into()),
+        DataType::Jsonb => field.r#type = Some(field_descriptor_proto::Type::String.into()),
+        DataType::Serial => field.r#type = Some(field_descriptor_proto::Type::Int64.into()),
+        DataType::Float32 | DataType::Int256 => {
+            return Err(SinkError::BigQuery(anyhow::anyhow!(
+                "Don't support Float32 and Int256"
+            )))
+        }
+    }
+    Ok((field, None))
+}
+
 #[cfg(test)]
 mod test {
+
+    use std::assert_matches::assert_matches;
+
+    use risingwave_common::catalog::{Field, Schema};
     use risingwave_common::types::{DataType, StructType};
 
-    use crate::sink::big_query::BigQuerySink;
+    use crate::sink::big_query::{
+        build_protobuf_descriptor_pool, build_protobuf_schema, BigQuerySink,
+    };
 
     #[tokio::test]
     async fn test_type_check() {
@@ -425,4 +718,63 @@ mod test {
             big_query_type_string
         );
     }
+
+    #[tokio::test]
+    async fn test_schema_check() {
+        let schema = Schema {
+            fields: vec![
+                Field::with_name(DataType::Int64, "v1"),
+                Field::with_name(DataType::Float64, "v2"),
+                Field::with_name(
+                    DataType::List(Box::new(DataType::Struct(StructType::new(vec![
+                        ("v1".to_owned(), DataType::List(Box::new(DataType::Int64))),
+                        (
+                            "v3".to_owned(),
+                            DataType::Struct(StructType::new(vec![
+                                ("v1".to_owned(), DataType::Int64),
+                                ("v2".to_owned(), DataType::Int64),
+                            ])),
+                        ),
+                    ])))),
+                    "v3",
+                ),
+            ],
+        };
+        let fields = schema
+            .fields()
+            .iter()
+            .map(|f| (f.name.as_str(), &f.data_type));
+        let desc = build_protobuf_schema(fields, "t1".to_string()).unwrap();
+        let pool = build_protobuf_descriptor_pool(&desc);
+        let t1_message = pool.get_message_by_name("t1").unwrap();
+        assert_matches!(
+            t1_message.get_field_by_name("v1").unwrap().kind(),
+            prost_reflect::Kind::Int64
+        );
+        assert_matches!(
+            t1_message.get_field_by_name("v2").unwrap().kind(),
+            prost_reflect::Kind::Double
+        );
+        assert_matches!(
+            t1_message.get_field_by_name("v3").unwrap().kind(),
+            prost_reflect::Kind::Message(_)
+        );
+
+        let v3_message = pool.get_message_by_name("t1.Structv3").unwrap();
+        assert_matches!(
+            v3_message.get_field_by_name("v1").unwrap().kind(),
+            prost_reflect::Kind::Int64
+        );
+        assert!(v3_message.get_field_by_name("v1").unwrap().is_list());
+
+        let v3_v3_message = pool.get_message_by_name("t1.Structv3.Structv3").unwrap();
+        assert_matches!(
+            v3_v3_message.get_field_by_name("v1").unwrap().kind(),
+            prost_reflect::Kind::Int64
+        );
+        assert_matches!(
+            v3_v3_message.get_field_by_name("v2").unwrap().kind(),
+            prost_reflect::Kind::Int64
+        );
+    }
 }
diff --git a/src/connector/src/sink/catalog/desc.rs b/src/connector/src/sink/catalog/desc.rs
index 0fb466e5a1742..87ccc7b96caf7 100644
--- a/src/connector/src/sink/catalog/desc.rs
+++ b/src/connector/src/sink/catalog/desc.rs
@@ -83,6 +83,7 @@ impl SinkDesc {
         owner: UserId,
         connection_id: Option<ConnectionId>,
         dependent_relations: Vec<TableId>,
+        secret_ref: BTreeMap<String, u32>,
     ) -> SinkCatalog {
         SinkCatalog {
             id: self.id,
@@ -108,6 +109,7 @@ impl SinkDesc {
             created_at_cluster_version: None,
             initialized_at_cluster_version: None,
             create_type: self.create_type,
+            secret_ref,
         }
     }
 
diff --git a/src/connector/src/sink/catalog/mod.rs b/src/connector/src/sink/catalog/mod.rs
index c0c0da9bc2047..f02eb2cdcf9e9 100644
--- a/src/connector/src/sink/catalog/mod.rs
+++ b/src/connector/src/sink/catalog/mod.rs
@@ -14,7 +14,7 @@
 
 pub mod desc;
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::BTreeMap;
 
 use anyhow::anyhow;
 use itertools::Itertools;
@@ -119,6 +119,8 @@ pub struct SinkFormatDesc {
     pub format: SinkFormat,
     pub encode: SinkEncode,
     pub options: BTreeMap<String, String>,
+
+    pub key_encode: Option<SinkEncode>,
 }
 
 /// TODO: consolidate with [`crate::source::SourceFormat`] and [`crate::parser::ProtocolProperties`].
@@ -136,6 +138,7 @@ pub enum SinkEncode {
     Protobuf,
     Avro,
     Template,
+    Text,
 }
 
 impl SinkFormatDesc {
@@ -166,6 +169,7 @@ impl SinkFormatDesc {
             format,
             encode,
             options: Default::default(),
+            key_encode: None,
         }))
     }
 
@@ -177,12 +181,16 @@ impl SinkFormatDesc {
             SinkFormat::Upsert => F::Upsert,
             SinkFormat::Debezium => F::Debezium,
         };
-        let encode = match self.encode {
+        let mapping_encode = |sink_encode: &SinkEncode| match sink_encode {
             SinkEncode::Json => E::Json,
             SinkEncode::Protobuf => E::Protobuf,
             SinkEncode::Avro => E::Avro,
             SinkEncode::Template => E::Template,
+            SinkEncode::Text => E::Text,
         };
+
+        let encode = mapping_encode(&self.encode);
+        let key_encode = self.key_encode.as_ref().map(|e| mapping_encode(e).into());
         let options = self
             .options
             .iter()
@@ -193,6 +201,7 @@ impl SinkFormatDesc {
             format: format.into(),
             encode: encode.into(),
             options,
+            key_encode,
         }
     }
 }
@@ -224,19 +233,37 @@ impl TryFrom<PbSinkFormatDesc> for SinkFormatDesc {
             E::Protobuf => SinkEncode::Protobuf,
             E::Template => SinkEncode::Template,
             E::Avro => SinkEncode::Avro,
-            e @ (E::Unspecified | E::Native | E::Csv | E::Bytes | E::None) => {
+            e @ (E::Unspecified | E::Native | E::Csv | E::Bytes | E::None | E::Text) => {
                 return Err(SinkError::Config(anyhow!(
                     "sink encode unsupported: {}",
                     e.as_str_name()
                 )))
             }
         };
+        let key_encode = match &value.key_encode() {
+            E::Text => Some(SinkEncode::Text),
+            E::Unspecified => None,
+            encode @ (E::Avro
+            | E::Bytes
+            | E::Csv
+            | E::Json
+            | E::Protobuf
+            | E::Template
+            | E::Native
+            | E::None) => {
+                return Err(SinkError::Config(anyhow!(
+                    "unsupported {} as sink key encode",
+                    encode.as_str_name()
+                )))
+            }
+        };
         let options = value.options.into_iter().collect();
 
         Ok(Self {
             format,
             encode,
             options,
+            key_encode,
         })
     }
 }
@@ -276,7 +303,7 @@ pub struct SinkCatalog {
     pub distribution_key: Vec<usize>,
 
     /// The properties of the sink.
-    pub properties: HashMap<String, String>,
+    pub properties: BTreeMap<String, String>,
 
     /// Owner of the sink.
     pub owner: UserId,
@@ -310,6 +337,9 @@ pub struct SinkCatalog {
     pub created_at_cluster_version: Option<String>,
     pub initialized_at_cluster_version: Option<String>,
     pub create_type: CreateType,
+
+    /// The secret reference for the sink, mapping from property name to secret id.
+    pub secret_ref: BTreeMap<String, u32>,
 }
 
 impl SinkCatalog {
@@ -351,6 +381,7 @@ impl SinkCatalog {
             created_at_cluster_version: self.created_at_cluster_version.clone(),
             initialized_at_cluster_version: self.initialized_at_cluster_version.clone(),
             create_type: self.create_type.to_proto() as i32,
+            secret_ref: self.secret_ref.clone(),
         }
     }
 
@@ -444,6 +475,7 @@ impl From<PbSink> for SinkCatalog {
             initialized_at_cluster_version: pb.initialized_at_cluster_version,
             created_at_cluster_version: pb.created_at_cluster_version,
             create_type: CreateType::from_proto(create_type),
+            secret_ref: pb.secret_ref,
         }
     }
 }
diff --git a/src/connector/src/sink/clickhouse.rs b/src/connector/src/sink/clickhouse.rs
index 5329118c73d61..c506f00e6d2ca 100644
--- a/src/connector/src/sink/clickhouse.rs
+++ b/src/connector/src/sink/clickhouse.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use core::fmt::Debug;
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
 
 use anyhow::anyhow;
 use clickhouse::insert::Insert;
@@ -48,7 +48,6 @@ const QUERY_ENGINE: &str =
 const QUERY_COLUMN: &str =
     "select distinct ?fields from system.columns where database = ? and table = ? order by ?";
 pub const CLICKHOUSE_SINK: &str = "clickhouse";
-const BUFFER_SIZE: usize = 1024;
 
 #[derive(Deserialize, Debug, Clone, WithOptions)]
 pub struct ClickHouseCommon {
@@ -78,7 +77,9 @@ enum ClickHouseEngine {
     ReplicatedReplacingMergeTree,
     ReplicatedSummingMergeTree,
     ReplicatedAggregatingMergeTree,
+    #[expect(dead_code)]
     ReplicatedCollapsingMergeTree(String),
+    #[expect(dead_code)]
     ReplicatedVersionedCollapsingMergeTree(String),
     ReplicatedGraphiteMergeTree,
 }
@@ -220,7 +221,7 @@ pub struct ClickHouseSink {
 }
 
 impl ClickHouseConfig {
-    pub fn from_hashmap(properties: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(properties: BTreeMap<String, String>) -> Result<Self> {
         let config =
             serde_json::from_value::<ClickHouseConfig>(serde_json::to_value(properties).unwrap())
                 .map_err(|e| SinkError::Config(anyhow!(e)))?;
@@ -241,7 +242,7 @@ impl TryFrom<SinkParam> for ClickHouseSink {
 
     fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
         let schema = param.schema();
-        let config = ClickHouseConfig::from_hashmap(param.properties)?;
+        let config = ClickHouseConfig::from_btreemap(param.properties)?;
         Ok(Self {
             config,
             schema,
@@ -378,11 +379,10 @@ impl Sink for ClickHouseSink {
 
     const SINK_NAME: &'static str = CLICKHOUSE_SINK;
 
-    fn is_sink_decouple(desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
+    fn is_sink_decouple(_desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
         match user_specified {
-            SinkDecouple::Default => Ok(desc.sink_type.is_append_only()),
+            SinkDecouple::Default | SinkDecouple::Enable => Ok(true),
             SinkDecouple::Disable => Ok(false),
-            SinkDecouple::Enable => Ok(true),
         }
     }
 
@@ -424,9 +424,12 @@ impl Sink for ClickHouseSink {
 }
 pub struct ClickHouseSinkWriter {
     pub config: ClickHouseConfig,
+    #[expect(dead_code)]
     schema: Schema,
+    #[expect(dead_code)]
     pk_indices: Vec<usize>,
     client: ClickHouseClient,
+    #[expect(dead_code)]
     is_append_only: bool,
     // Save some features of the clickhouse column type
     column_correct_vec: Vec<ClickHouseSchemaFeature>,
@@ -617,6 +620,7 @@ struct SystemColumn {
 
 #[derive(ClickHouseRow, Deserialize)]
 struct ClickhouseQueryEngine {
+    #[expect(dead_code)]
     name: String,
     engine: String,
     create_table_query: String,
diff --git a/src/connector/src/sink/iceberg/log_sink.rs b/src/connector/src/sink/decouple_checkpoint_log_sink.rs
similarity index 89%
rename from src/connector/src/sink/iceberg/log_sink.rs
rename to src/connector/src/sink/decouple_checkpoint_log_sink.rs
index fec6285774bbe..26576cf3e3666 100644
--- a/src/connector/src/sink/iceberg/log_sink.rs
+++ b/src/connector/src/sink/decouple_checkpoint_log_sink.rs
@@ -21,13 +21,16 @@ use crate::sink::log_store::{LogStoreReadItem, TruncateOffset};
 use crate::sink::writer::SinkWriter;
 use crate::sink::{LogSinker, Result, SinkLogReader, SinkMetrics};
 
-pub struct IcebergLogSinkerOf<W> {
+/// The `LogSinker` implementation used for commit-decoupled sinks (such as `Iceberg`, `DeltaLake` and `StarRocks`).
+/// The concurrent/frequent commit capability of these sinks is poor, so by leveraging the decoupled log reader,
+/// we delay the checkpoint barrier to make commits less frequent.
+pub struct DecoupleCheckpointLogSinkerOf<W> {
     writer: W,
     sink_metrics: SinkMetrics,
     commit_checkpoint_interval: NonZeroU64,
 }
 
-impl<W> IcebergLogSinkerOf<W> {
+impl<W> DecoupleCheckpointLogSinkerOf<W> {
     /// Create a log sinker with a commit checkpoint interval. The sinker should be used with a
     /// decouple log reader `KvLogStoreReader`.
     pub fn new(
@@ -35,7 +38,7 @@ impl<W> IcebergLogSinkerOf<W> {
         sink_metrics: SinkMetrics,
         commit_checkpoint_interval: NonZeroU64,
     ) -> Self {
-        IcebergLogSinkerOf {
+        DecoupleCheckpointLogSinkerOf {
             writer,
             sink_metrics,
             commit_checkpoint_interval,
@@ -44,8 +47,8 @@ impl<W> IcebergLogSinkerOf<W> {
 }
 
 #[async_trait]
-impl<W: SinkWriter<CommitMetadata = ()>> LogSinker for IcebergLogSinkerOf<W> {
-    async fn consume_log_and_sink(self, log_reader: &mut impl SinkLogReader) -> Result<()> {
+impl<W: SinkWriter<CommitMetadata = ()>> LogSinker for DecoupleCheckpointLogSinkerOf<W> {
+    async fn consume_log_and_sink(self, log_reader: &mut impl SinkLogReader) -> Result<!> {
         let mut sink_writer = self.writer;
         let sink_metrics = self.sink_metrics;
         #[derive(Debug)]
@@ -123,9 +126,7 @@ impl<W: SinkWriter<CommitMetadata = ()>> LogSinker for IcebergLogSinkerOf<W> {
                             sink_metrics
                                 .sink_commit_duration_metrics
                                 .observe(start_time.elapsed().as_millis() as f64);
-                            log_reader
-                                .truncate(TruncateOffset::Barrier { epoch })
-                                .await?;
+                            log_reader.truncate(TruncateOffset::Barrier { epoch })?;
                             current_checkpoint = 0;
                         } else {
                             sink_writer.barrier(false).await?;
diff --git a/src/connector/src/sink/deltalake.rs b/src/connector/src/sink/deltalake.rs
index 388a75c264a71..38427f935018d 100644
--- a/src/connector/src/sink/deltalake.rs
+++ b/src/connector/src/sink/deltalake.rs
@@ -12,7 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use core::num::NonZeroU64;
+use std::collections::{BTreeMap, HashMap};
 use std::sync::Arc;
 
 use anyhow::{anyhow, Context};
@@ -25,10 +26,12 @@ use deltalake::table::builder::s3_storage_options::{
 };
 use deltalake::writer::{DeltaWriter, RecordBatchWriter};
 use deltalake::DeltaTable;
-use risingwave_common::array::{to_deltalake_record_batch_with_schema, StreamChunk};
+use risingwave_common::array::arrow::DeltaLakeConvert;
+use risingwave_common::array::StreamChunk;
 use risingwave_common::bail;
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::Schema;
+use risingwave_common::session_config::sink_decouple::SinkDecouple;
 use risingwave_common::types::DataType;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_pb::connector_service::sink_metadata::Metadata::Serialized;
@@ -38,13 +41,15 @@ use serde_derive::{Deserialize, Serialize};
 use serde_with::serde_as;
 use with_options::WithOptions;
 
+use super::catalog::desc::SinkDesc;
 use super::coordinate::CoordinatedSinkWriter;
-use super::writer::{LogSinkerOf, SinkWriter};
+use super::decouple_checkpoint_log_sink::DecoupleCheckpointLogSinkerOf;
+use super::writer::SinkWriter;
 use super::{
     Result, Sink, SinkCommitCoordinator, SinkError, SinkParam, SinkWriterParam,
     SINK_TYPE_APPEND_ONLY, SINK_USER_FORCE_APPEND_ONLY_OPTION,
 };
-use crate::sink::writer::SinkWriterExt;
+use crate::deserialize_optional_u64_from_string;
 
 pub const DELTALAKE_SINK: &str = "deltalake";
 pub const DEFAULT_REGION: &str = "us-east-1";
@@ -64,6 +69,9 @@ pub struct DeltaLakeCommon {
     pub s3_endpoint: Option<String>,
     #[serde(rename = "gcs.service.account")]
     pub gcs_service_account: Option<String>,
+    /// Commit every n(>0) checkpoints, if n is not set, we will commit every checkpoint.
+    #[serde(default, deserialize_with = "deserialize_optional_u64_from_string")]
+    pub commit_checkpoint_interval: Option<u64>,
 }
 impl DeltaLakeCommon {
     pub async fn create_deltalake_client(&self) -> Result<DeltaTable> {
@@ -149,7 +157,7 @@ pub struct DeltaLakeConfig {
 }
 
 impl DeltaLakeConfig {
-    pub fn from_hashmap(properties: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(properties: BTreeMap<String, String>) -> Result<Self> {
         let config = serde_json::from_value::<DeltaLakeConfig>(
             serde_json::to_value(properties).map_err(|e| SinkError::DeltaLake(e.into()))?,
         )
@@ -268,10 +276,34 @@ fn check_field_type(rw_data_type: &DataType, dl_data_type: &DeltaLakeDataType) -
 
 impl Sink for DeltaLakeSink {
     type Coordinator = DeltaLakeSinkCommitter;
-    type LogSinker = LogSinkerOf<CoordinatedSinkWriter<DeltaLakeSinkWriter>>;
+    type LogSinker = DecoupleCheckpointLogSinkerOf<CoordinatedSinkWriter<DeltaLakeSinkWriter>>;
 
     const SINK_NAME: &'static str = DELTALAKE_SINK;
 
+    fn is_sink_decouple(desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
+        let config_decouple = if let Some(interval) =
+            desc.properties.get("commit_checkpoint_interval")
+            && interval.parse::<u64>().unwrap_or(0) > 1
+        {
+            true
+        } else {
+            false
+        };
+
+        match user_specified {
+            SinkDecouple::Default => Ok(config_decouple),
+            SinkDecouple::Disable => {
+                if config_decouple {
+                    return Err(SinkError::Config(anyhow!(
+                        "config conflict: DeltaLake config `commit_checkpoint_interval` larger than 1 means that sink decouple must be enabled, but session config sink_decouple is disabled"
+                    )));
+                }
+                Ok(false)
+            }
+            SinkDecouple::Enable => Ok(true),
+        }
+    }
+
     async fn new_log_sinker(&self, writer_param: SinkWriterParam) -> Result<Self::LogSinker> {
         let inner = DeltaLakeSinkWriter::new(
             self.config.clone(),
@@ -279,7 +311,7 @@ impl Sink for DeltaLakeSink {
             self.param.downstream_pk.clone(),
         )
         .await?;
-        Ok(CoordinatedSinkWriter::new(
+        let writer = CoordinatedSinkWriter::new(
             writer_param
                 .meta_client
                 .expect("should have meta client")
@@ -293,8 +325,18 @@ impl Sink for DeltaLakeSink {
             })?,
             inner,
         )
-        .await?
-        .into_log_sinker(writer_param.sink_metrics))
+        .await?;
+
+        let commit_checkpoint_interval =
+            NonZeroU64::new(self.config.common.commit_checkpoint_interval.unwrap_or(1)).expect(
+                "commit_checkpoint_interval should be greater than 0, and it should be checked in config validation",
+            );
+
+        Ok(DecoupleCheckpointLogSinkerOf::new(
+            writer,
+            writer_param.sink_metrics,
+            commit_checkpoint_interval,
+        ))
     }
 
     async fn validate(&self) -> Result<()> {
@@ -352,17 +394,20 @@ impl TryFrom<SinkParam> for DeltaLakeSink {
     type Error = SinkError;
 
     fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
-        let config = DeltaLakeConfig::from_hashmap(param.properties.clone())?;
+        let config = DeltaLakeConfig::from_btreemap(param.properties.clone())?;
         DeltaLakeSink::new(config, param)
     }
 }
 
 pub struct DeltaLakeSinkWriter {
     pub config: DeltaLakeConfig,
+    #[expect(dead_code)]
     schema: Schema,
+    #[expect(dead_code)]
     pk_indices: Vec<usize>,
     writer: RecordBatchWriter,
     dl_schema: Arc<deltalake::arrow::datatypes::Schema>,
+    #[expect(dead_code)]
     dl_table: DeltaTable,
 }
 
@@ -388,7 +433,8 @@ impl DeltaLakeSinkWriter {
     }
 
     async fn write(&mut self, chunk: StreamChunk) -> Result<()> {
-        let a = to_deltalake_record_batch_with_schema(self.dl_schema.clone(), &chunk)
+        let a = DeltaLakeConvert
+            .to_record_batch(self.dl_schema.clone(), &chunk)
             .context("convert record batch error")
             .map_err(SinkError::DeltaLake)?;
         self.writer.write(a).await?;
@@ -530,7 +576,7 @@ impl DeltaLakeWriteResult {
 mod test {
     use deltalake::kernel::DataType as SchemaDataType;
     use deltalake::operations::create::CreateBuilder;
-    use maplit::hashmap;
+    use maplit::btreemap;
     use risingwave_common::array::{Array, I32Array, Op, StreamChunk, Utf8Array};
     use risingwave_common::catalog::{Field, Schema};
 
@@ -551,7 +597,7 @@ mod test {
             .await
             .unwrap();
 
-        let properties = hashmap! {
+        let properties = btreemap! {
             "connector".to_string() => "deltalake".to_string(),
             "force_append_only".to_string() => "true".to_string(),
             "type".to_string() => "append-only".to_string(),
@@ -573,7 +619,7 @@ mod test {
             },
         ]);
 
-        let deltalake_config = DeltaLakeConfig::from_hashmap(properties).unwrap();
+        let deltalake_config = DeltaLakeConfig::from_btreemap(properties).unwrap();
         let deltalake_table = deltalake_config
             .common
             .create_deltalake_client()
diff --git a/src/connector/src/sink/doris.rs b/src/connector/src/sink/doris.rs
index 75283f200c254..643ad9e3b7b3b 100644
--- a/src/connector/src/sink/doris.rs
+++ b/src/connector/src/sink/doris.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::sync::Arc;
 
 use anyhow::{anyhow, Context};
@@ -77,7 +77,7 @@ pub struct DorisConfig {
     pub r#type: String, // accept "append-only" or "upsert"
 }
 impl DorisConfig {
-    pub fn from_hashmap(properties: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(properties: BTreeMap<String, String>) -> Result<Self> {
         let config =
             serde_json::from_value::<DorisConfig>(serde_json::to_value(properties).unwrap())
                 .map_err(|e| SinkError::Config(anyhow!(e)))?;
@@ -225,7 +225,9 @@ impl Sink for DorisSink {
 
 pub struct DorisSinkWriter {
     pub config: DorisConfig,
+    #[expect(dead_code)]
     schema: Schema,
+    #[expect(dead_code)]
     pk_indices: Vec<usize>,
     inserter_inner_builder: InserterInnerBuilder,
     is_append_only: bool,
@@ -238,7 +240,7 @@ impl TryFrom<SinkParam> for DorisSink {
 
     fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
         let schema = param.schema();
-        let config = DorisConfig::from_hashmap(param.properties)?;
+        let config = DorisConfig::from_btreemap(param.properties)?;
         DorisSink::new(
             config,
             schema,
diff --git a/src/connector/src/sink/doris_starrocks_connector.rs b/src/connector/src/sink/doris_starrocks_connector.rs
index 6c045c63beb47..3173b64389f22 100644
--- a/src/connector/src/sink/doris_starrocks_connector.rs
+++ b/src/connector/src/sink/doris_starrocks_connector.rs
@@ -17,12 +17,13 @@ use core::time::Duration;
 use std::collections::HashMap;
 use std::convert::Infallible;
 
-use anyhow::Context;
+use anyhow::{anyhow, Context};
 use base64::engine::general_purpose;
 use base64::Engine;
 use bytes::{BufMut, Bytes, BytesMut};
 use futures::StreamExt;
-use reqwest::{redirect, Body, Client, RequestBuilder, StatusCode};
+use reqwest::header::{HeaderName, HeaderValue};
+use reqwest::{redirect, Body, Client, Method, Request, RequestBuilder, Response, StatusCode};
 use tokio::sync::mpsc::UnboundedSender;
 use tokio::task::JoinHandle;
 use url::Url;
@@ -32,14 +33,12 @@ use super::{Result, SinkError};
 const BUFFER_SIZE: usize = 64 * 1024;
 const MIN_CHUNK_SIZE: usize = BUFFER_SIZE - 1024;
 pub(crate) const DORIS_SUCCESS_STATUS: [&str; 2] = ["Success", "Publish Timeout"];
+pub(crate) const STARROCKS_SUCCESS_STATUS: [&str; 1] = ["OK"];
 pub(crate) const DORIS_DELETE_SIGN: &str = "__DORIS_DELETE_SIGN__";
 pub(crate) const STARROCKS_DELETE_SIGN: &str = "__op";
 
-const SEND_CHUNK_TIMEOUT: Duration = Duration::from_secs(10);
 const WAIT_HANDDLE_TIMEOUT: Duration = Duration::from_secs(10);
 pub(crate) const POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(30);
-const DORIS: &str = "doris";
-const STARROCKS: &str = "starrocks";
 const LOCALHOST: &str = "localhost";
 const LOCALHOST_IP: &str = "127.0.0.1";
 pub struct HeaderBuilder {
@@ -151,14 +150,77 @@ impl HeaderBuilder {
         self
     }
 
+    /// Only used in Starrocks Transaction API
+    pub fn set_db(mut self, db: String) -> Self {
+        self.header.insert("db".to_string(), db);
+        self
+    }
+
+    /// Only used in Starrocks Transaction API
+    pub fn set_table(mut self, table: String) -> Self {
+        self.header.insert("table".to_string(), table);
+        self
+    }
+
     pub fn build(self) -> HashMap<String, String> {
         self.header
     }
 }
 
+/// Try getting BE url from a redirected response, returning `Ok(None)` indicates this request does
+/// not redirect.
+///
+/// The reason we handle the redirection manually is that if we let `reqwest` handle the redirection
+/// automatically, it will remove sensitive headers (such as Authorization) during the redirection,
+/// and there's no way to prevent this behavior.
+fn try_get_be_url(resp: &Response, fe_host: String) -> Result<Option<Url>> {
+    match resp.status() {
+        StatusCode::TEMPORARY_REDIRECT => {
+            let be_url = resp
+                .headers()
+                .get("location")
+                .ok_or_else(|| {
+                    SinkError::DorisStarrocksConnect(anyhow!("Can't get doris BE url in header",))
+                })?
+                .to_str()
+                .context("Can't get doris BE url in header")
+                .map_err(SinkError::DorisStarrocksConnect)?
+                .to_string();
+
+            let mut parsed_be_url = Url::parse(&be_url)
+                .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
+
+            if fe_host != LOCALHOST && fe_host != LOCALHOST_IP {
+                let be_host = parsed_be_url.host_str().ok_or_else(|| {
+                    SinkError::DorisStarrocksConnect(anyhow!("Can't get be host from url"))
+                })?;
+
+                if be_host == LOCALHOST || be_host == LOCALHOST_IP {
+                    // if be host is 127.0.0.1, we may can't connect to it directly,
+                    // so replace it with fe host
+                    parsed_be_url
+                        .set_host(Some(fe_host.as_str()))
+                        .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
+                }
+            }
+            Ok(Some(parsed_be_url))
+        }
+        StatusCode::OK => {
+            // Some of the `StarRocks` transactional APIs will respond directly from FE. For example,
+            // the request to `/api/transaction/commit` endpoint does not seem to redirect to BE.
+            // In this case, the request should be treated as finished.
+            Ok(None)
+        }
+        _ => Err(SinkError::DorisStarrocksConnect(anyhow!(
+            "Can't get doris BE url",
+        ))),
+    }
+}
+
 pub struct InserterInnerBuilder {
     url: String,
     header: HashMap<String, String>,
+    #[expect(dead_code)]
     sender: Option<Sender>,
     fe_host: String,
 }
@@ -170,11 +232,9 @@ impl InserterInnerBuilder {
         header: HashMap<String, String>,
     ) -> Result<Self> {
         let fe_host = Url::parse(&url)
-            .map_err(|err| SinkError::DorisStarrocksConnect(err.into()))?
+            .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?
             .host_str()
-            .ok_or_else(|| {
-                SinkError::DorisStarrocksConnect(anyhow::anyhow!("Can't get fe host from url"))
-            })?
+            .ok_or_else(|| SinkError::DorisStarrocksConnect(anyhow!("Can't get fe host from url")))?
             .to_string();
         let url = format!("{}/api/{}/{}/_stream_load", url, db, table);
 
@@ -205,73 +265,45 @@ impl InserterInnerBuilder {
         let resp = builder
             .send()
             .await
-            .map_err(|err| SinkError::DorisStarrocksConnect(err.into()))?;
-        // TODO: shall we let `reqwest` handle the redirect?
-        let mut be_url = if resp.status() == StatusCode::TEMPORARY_REDIRECT {
-            resp.headers()
-                .get("location")
-                .ok_or_else(|| {
-                    SinkError::DorisStarrocksConnect(anyhow::anyhow!(
-                        "Can't get doris BE url in header",
-                    ))
-                })?
-                .to_str()
-                .context("Can't get doris BE url in header")
-                .map_err(SinkError::DorisStarrocksConnect)?
-                .to_string()
-        } else {
-            return Err(SinkError::DorisStarrocksConnect(anyhow::anyhow!(
-                "Can't get doris BE url",
-            )));
-        };
+            .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
 
-        if self.fe_host != LOCALHOST && self.fe_host != LOCALHOST_IP {
-            let mut parsed_be_url =
-                Url::parse(&be_url).map_err(|err| SinkError::DorisStarrocksConnect(err.into()))?;
-            let be_host = parsed_be_url.host_str().ok_or_else(|| {
-                SinkError::DorisStarrocksConnect(anyhow::anyhow!("Can't get be host from url"))
-            })?;
-
-            if be_host == LOCALHOST || be_host == LOCALHOST_IP {
-                // if be host is 127.0.0.1, we may can't connect to it directly,
-                // so replace it with fe host
-                parsed_be_url
-                    .set_host(Some(self.fe_host.as_str()))
-                    .map_err(|err| SinkError::DorisStarrocksConnect(err.into()))?;
-                be_url = parsed_be_url.as_str().into();
-            }
-        }
+        let be_url = try_get_be_url(&resp, self.fe_host.clone())?
+            .ok_or_else(|| SinkError::DorisStarrocksConnect(anyhow!("Can't get doris BE url",)))?;
 
         let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
         let body = Body::wrap_stream(
             tokio_stream::wrappers::UnboundedReceiverStream::new(receiver).map(Ok::<_, Infallible>),
         );
-        let builder = self.build_request(be_url).body(body);
+        let builder = self.build_request(be_url.into()).body(body);
 
         let handle: JoinHandle<Result<Vec<u8>>> = tokio::spawn(async move {
             let response = builder
                 .send()
                 .await
-                .map_err(|err| SinkError::DorisStarrocksConnect(err.into()))?;
+                .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
             let status = response.status();
             let raw = response
                 .bytes()
                 .await
-                .map_err(|err| SinkError::DorisStarrocksConnect(err.into()))?
+                .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?
                 .into();
 
             if status == StatusCode::OK {
                 Ok(raw)
             } else {
-                Err(SinkError::DorisStarrocksConnect(anyhow::anyhow!(
+                let response_body = String::from_utf8(raw).map_err(|err| {
+                    SinkError::DorisStarrocksConnect(
+                        anyhow!(err).context("failed to parse response body"),
+                    )
+                })?;
+                Err(SinkError::DorisStarrocksConnect(anyhow!(
                     "Failed connection {:?},{:?}",
                     status,
-                    String::from_utf8(raw)
-                        .map_err(|err| SinkError::DorisStarrocksConnect(err.into()))?
+                    response_body
                 )))
             }
         });
-        Ok(InserterInner::new(sender, handle))
+        Ok(InserterInner::new(sender, handle, WAIT_HANDDLE_TIMEOUT))
     }
 }
 
@@ -281,13 +313,19 @@ pub struct InserterInner {
     sender: Option<Sender>,
     join_handle: Option<JoinHandle<Result<Vec<u8>>>>,
     buffer: BytesMut,
+    stream_load_http_timeout: Duration,
 }
 impl InserterInner {
-    pub fn new(sender: Sender, join_handle: JoinHandle<Result<Vec<u8>>>) -> Self {
+    pub fn new(
+        sender: Sender,
+        join_handle: JoinHandle<Result<Vec<u8>>>,
+        stream_load_http_timeout: Duration,
+    ) -> Self {
         Self {
             sender: Some(sender),
             join_handle: Some(join_handle),
             buffer: BytesMut::with_capacity(BUFFER_SIZE),
+            stream_load_http_timeout,
         }
     }
 
@@ -302,9 +340,7 @@ impl InserterInner {
             self.sender.take();
             self.wait_handle().await?;
 
-            Err(SinkError::DorisStarrocksConnect(anyhow::anyhow!(
-                "channel closed"
-            )))
+            Err(SinkError::DorisStarrocksConnect(anyhow!("channel closed")))
         } else {
             Ok(())
         }
@@ -319,13 +355,15 @@ impl InserterInner {
     }
 
     async fn wait_handle(&mut self) -> Result<Vec<u8>> {
-        let res =
-            match tokio::time::timeout(WAIT_HANDDLE_TIMEOUT, self.join_handle.as_mut().unwrap())
-                .await
-            {
-                Ok(res) => res.map_err(|err| SinkError::DorisStarrocksConnect(err.into()))??,
-                Err(err) => return Err(SinkError::DorisStarrocksConnect(err.into())),
-            };
+        let res = match tokio::time::timeout(
+            self.stream_load_http_timeout,
+            self.join_handle.as_mut().unwrap(),
+        )
+        .await
+        {
+            Ok(res) => res.map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))??,
+            Err(err) => return Err(SinkError::DorisStarrocksConnect(anyhow!(err))),
+        };
         Ok(res)
     }
 
@@ -337,3 +375,235 @@ impl InserterInner {
         self.wait_handle().await
     }
 }
+
+pub struct MetaRequestSender {
+    client: Client,
+    request: Request,
+    fe_host: String,
+}
+
+impl MetaRequestSender {
+    pub fn new(client: Client, request: Request, fe_host: String) -> Self {
+        Self {
+            client,
+            request,
+            fe_host,
+        }
+    }
+
+    /// Send the request and handle redirection if any.
+    /// The reason we handle the redirection manually is that if we let `reqwest` handle the redirection
+    /// automatically, it will remove sensitive headers (such as Authorization) during the redirection,
+    /// and there's no way to prevent this behavior.
+    ///
+    /// Another interesting point is that some of the `StarRocks` transactional APIs will respond directly from FE.
+    /// For example, the request to `/api/transaction/commit` endpoint does not seem to redirect to BE.
+    pub async fn send(self) -> Result<Bytes> {
+        let request = self.request;
+        let mut request_for_redirection = request
+            .try_clone()
+            .ok_or_else(|| SinkError::DorisStarrocksConnect(anyhow!("Can't clone request")))?;
+
+        let resp = self
+            .client
+            .execute(request)
+            .await
+            .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
+
+        let be_url = try_get_be_url(&resp, self.fe_host)?;
+
+        match be_url {
+            Some(be_url) => {
+                *request_for_redirection.url_mut() = be_url;
+
+                self.client
+                    .execute(request_for_redirection)
+                    .await
+                    .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?
+                    .bytes()
+                    .await
+                    .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))
+            }
+            None => resp
+                .bytes()
+                .await
+                .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err))),
+        }
+    }
+}
+
+pub struct StarrocksTxnRequestBuilder {
+    url_begin: String,
+    url_load: String,
+    url_prepare: String,
+    url_commit: String,
+    url_rollback: String,
+    header: HashMap<String, String>,
+    fe_host: String,
+    stream_load_http_timeout: Duration,
+    // The `reqwest` crate suggests us reuse the Client, and we don't need make it Arc, because it
+    // already uses an Arc internally.
+    client: Client,
+}
+
+impl StarrocksTxnRequestBuilder {
+    pub fn new(
+        url: String,
+        header: HashMap<String, String>,
+        stream_load_http_timeout_ms: u64,
+    ) -> Result<Self> {
+        let fe_host = Url::parse(&url)
+            .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?
+            .host_str()
+            .ok_or_else(|| SinkError::DorisStarrocksConnect(anyhow!("Can't get fe host from url")))?
+            .to_string();
+
+        let url_begin = format!("{}/api/transaction/begin", url);
+        let url_load = format!("{}/api/transaction/load", url);
+        let url_prepare = format!("{}/api/transaction/prepare", url);
+        let url_commit = format!("{}/api/transaction/commit", url);
+        let url_rollback = format!("{}/api/transaction/rollback", url);
+
+        let stream_load_http_timeout = Duration::from_millis(stream_load_http_timeout_ms);
+
+        let client = Client::builder()
+            .pool_idle_timeout(POOL_IDLE_TIMEOUT)
+            .redirect(redirect::Policy::none())
+            .build()
+            .unwrap();
+
+        Ok(Self {
+            url_begin,
+            url_load,
+            url_prepare,
+            url_commit,
+            url_rollback,
+            header,
+            fe_host,
+            stream_load_http_timeout,
+            client,
+        })
+    }
+
+    fn build_request(&self, uri: String, method: Method, label: String) -> Result<Request> {
+        let parsed_url =
+            Url::parse(&uri).map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
+        let mut request = Request::new(method, parsed_url);
+
+        if uri != self.url_load {
+            // Set timeout for non-load requests; load requests' timeout is controlled by `tokio::timeout`
+            *request.timeout_mut() = Some(self.stream_load_http_timeout);
+        }
+
+        let header = request.headers_mut();
+        for (k, v) in &self.header {
+            header.insert(
+                HeaderName::try_from(k)
+                    .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?,
+                HeaderValue::try_from(v)
+                    .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?,
+            );
+        }
+        header.insert(
+            "label",
+            HeaderValue::try_from(label)
+                .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?,
+        );
+
+        Ok(request)
+    }
+
+    pub fn build_begin_request_sender(&self, label: String) -> Result<MetaRequestSender> {
+        let request = self.build_request(self.url_begin.clone(), Method::POST, label)?;
+        Ok(MetaRequestSender::new(
+            self.client.clone(),
+            request,
+            self.fe_host.clone(),
+        ))
+    }
+
+    pub fn build_prepare_request_sender(&self, label: String) -> Result<MetaRequestSender> {
+        let request = self.build_request(self.url_prepare.clone(), Method::POST, label)?;
+        Ok(MetaRequestSender::new(
+            self.client.clone(),
+            request,
+            self.fe_host.clone(),
+        ))
+    }
+
+    pub fn build_commit_request_sender(&self, label: String) -> Result<MetaRequestSender> {
+        let request = self.build_request(self.url_commit.clone(), Method::POST, label)?;
+        Ok(MetaRequestSender::new(
+            self.client.clone(),
+            request,
+            self.fe_host.clone(),
+        ))
+    }
+
+    pub fn build_rollback_request_sender(&self, label: String) -> Result<MetaRequestSender> {
+        let request = self.build_request(self.url_rollback.clone(), Method::POST, label)?;
+        Ok(MetaRequestSender::new(
+            self.client.clone(),
+            request,
+            self.fe_host.clone(),
+        ))
+    }
+
+    pub async fn build_txn_inserter(&self, label: String) -> Result<InserterInner> {
+        let request = self.build_request(self.url_load.clone(), Method::PUT, label)?;
+        let mut request_for_redirection = request
+            .try_clone()
+            .ok_or_else(|| SinkError::DorisStarrocksConnect(anyhow!("Can't clone request")))?;
+
+        let resp = self
+            .client
+            .execute(request)
+            .await
+            .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
+
+        let be_url = try_get_be_url(&resp, self.fe_host.clone())?
+            .ok_or_else(|| SinkError::DorisStarrocksConnect(anyhow!("Can't get doris BE url",)))?;
+        *request_for_redirection.url_mut() = be_url;
+
+        let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
+        let body = Body::wrap_stream(
+            tokio_stream::wrappers::UnboundedReceiverStream::new(receiver).map(Ok::<_, Infallible>),
+        );
+        *request_for_redirection.body_mut() = Some(body);
+
+        let client = self.client.clone();
+        let handle: JoinHandle<Result<Vec<u8>>> = tokio::spawn(async move {
+            let response = client
+                .execute(request_for_redirection)
+                .await
+                .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
+
+            let status = response.status();
+            let raw = response
+                .bytes()
+                .await
+                .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?
+                .into();
+
+            if status == StatusCode::OK {
+                Ok(raw)
+            } else {
+                let response_body = String::from_utf8(raw).map_err(|err| {
+                    SinkError::DorisStarrocksConnect(
+                        anyhow!(err).context("failed to parse response body"),
+                    )
+                })?;
+                Err(SinkError::DorisStarrocksConnect(anyhow!(
+                    "Failed connection {:?},{:?}",
+                    status,
+                    response_body
+                )))
+            }
+        });
+        Ok(InserterInner::new(
+            sender,
+            handle,
+            self.stream_load_http_timeout,
+        ))
+    }
+}
diff --git a/src/connector/src/sink/dynamodb.rs b/src/connector/src/sink/dynamodb.rs
new file mode 100644
index 0000000000000..35b48c6e31faf
--- /dev/null
+++ b/src/connector/src/sink/dynamodb.rs
@@ -0,0 +1,400 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{BTreeMap, HashMap, HashSet};
+
+use anyhow::{anyhow, Context};
+use aws_sdk_dynamodb as dynamodb;
+use aws_sdk_dynamodb::client::Client;
+use aws_smithy_types::Blob;
+use dynamodb::types::{
+    AttributeValue, DeleteRequest, PutRequest, ReturnConsumedCapacity, ReturnItemCollectionMetrics,
+    TableStatus, WriteRequest,
+};
+use maplit::hashmap;
+use risingwave_common::array::{Op, RowRef, StreamChunk};
+use risingwave_common::catalog::{Field, Schema};
+use risingwave_common::row::Row as _;
+use risingwave_common::types::{DataType, ScalarRefImpl, ToText};
+use risingwave_common::util::iter_util::ZipEqDebug;
+use serde_derive::Deserialize;
+use serde_with::{serde_as, DisplayFromStr};
+use with_options::WithOptions;
+
+use super::log_store::DeliveryFutureManagerAddFuture;
+use super::writer::{
+    AsyncTruncateLogSinkerOf, AsyncTruncateSinkWriter, AsyncTruncateSinkWriterExt,
+};
+use super::{DummySinkCommitCoordinator, Result, Sink, SinkError, SinkParam, SinkWriterParam};
+use crate::connector_common::AwsAuthProps;
+use crate::error::ConnectorResult;
+
+pub const DYNAMO_DB_SINK: &str = "dynamodb";
+
+#[serde_as]
+#[derive(Deserialize, Debug, Clone, WithOptions)]
+pub struct DynamoDbConfig {
+    #[serde(rename = "table", alias = "dynamodb.table")]
+    pub table: String,
+
+    #[serde(rename = "dynamodb.max_batch_rows", default = "default_max_batch_rows")]
+    #[serde_as(as = "DisplayFromStr")]
+    pub max_batch_rows: usize,
+
+    #[serde(flatten)]
+    pub aws_auth_props: AwsAuthProps,
+}
+
+fn default_max_batch_rows() -> usize {
+    1024
+}
+
+impl DynamoDbConfig {
+    pub async fn build_client(&self) -> ConnectorResult<Client> {
+        let config = &self.aws_auth_props;
+        let aws_config = config.build_config().await?;
+
+        Ok(Client::new(&aws_config))
+    }
+
+    fn from_btreemap(values: BTreeMap<String, String>) -> Result<Self> {
+        serde_json::from_value::<DynamoDbConfig>(serde_json::to_value(values).unwrap())
+            .map_err(|e| SinkError::Config(anyhow!(e)))
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct DynamoDbSink {
+    pub config: DynamoDbConfig,
+    schema: Schema,
+    pk_indices: Vec<usize>,
+}
+
+impl Sink for DynamoDbSink {
+    type Coordinator = DummySinkCommitCoordinator;
+    type LogSinker = AsyncTruncateLogSinkerOf<DynamoDbSinkWriter>;
+
+    const SINK_NAME: &'static str = DYNAMO_DB_SINK;
+
+    async fn validate(&self) -> Result<()> {
+        let client = (self.config.build_client().await)
+            .context("validate DynamoDB sink error")
+            .map_err(SinkError::DynamoDb)?;
+
+        let table_name = &self.config.table;
+        let output = client
+            .describe_table()
+            .table_name(table_name)
+            .send()
+            .await
+            .map_err(|e| anyhow!(e))?;
+        let Some(table) = output.table else {
+            return Err(SinkError::DynamoDb(anyhow!(
+                "table {} not found",
+                table_name
+            )));
+        };
+        if !matches!(table.table_status(), Some(TableStatus::Active)) {
+            return Err(SinkError::DynamoDb(anyhow!(
+                "table {} is not active",
+                table_name
+            )));
+        }
+        let pk_set: HashSet<String> = self
+            .schema
+            .fields()
+            .iter()
+            .enumerate()
+            .filter(|(k, _)| self.pk_indices.contains(k))
+            .map(|(_, v)| v.name.clone())
+            .collect();
+        let key_schema = table.key_schema();
+
+        for key_element in key_schema.iter().map(|x| x.attribute_name()) {
+            if !pk_set.contains(key_element) {
+                return Err(SinkError::DynamoDb(anyhow!(
+                    "table {} key field {} not found in schema or not primary key",
+                    table_name,
+                    key_element
+                )));
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn new_log_sinker(&self, _writer_param: SinkWriterParam) -> Result<Self::LogSinker> {
+        Ok(
+            DynamoDbSinkWriter::new(self.config.clone(), self.schema.clone())
+                .await?
+                .into_log_sinker(usize::MAX),
+        )
+    }
+}
+
+impl TryFrom<SinkParam> for DynamoDbSink {
+    type Error = SinkError;
+
+    fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
+        let schema = param.schema();
+        let config = DynamoDbConfig::from_btreemap(param.properties)?;
+
+        Ok(Self {
+            config,
+            schema,
+            pk_indices: param.downstream_pk,
+        })
+    }
+}
+
+#[derive(Debug)]
+struct DynamoDbRequest {
+    inner: WriteRequest,
+    key_items: Vec<String>,
+}
+
+impl DynamoDbRequest {
+    fn extract_pk_values(&self) -> Option<Vec<AttributeValue>> {
+        let key = match (&self.inner.put_request(), &self.inner.delete_request()) {
+            (Some(put_req), None) => &put_req.item,
+            (None, Some(del_req)) => &del_req.key,
+            _ => return None,
+        };
+        let vs = key
+            .iter()
+            .filter(|(k, _)| self.key_items.contains(k))
+            .map(|(_, v)| v.clone())
+            .collect();
+        Some(vs)
+    }
+}
+
+struct DynamoDbPayloadWriter {
+    request_items: Vec<DynamoDbRequest>,
+    client: Client,
+    table: String,
+    dynamodb_keys: Vec<String>,
+}
+
+impl DynamoDbPayloadWriter {
+    fn write_one_insert(&mut self, item: HashMap<String, AttributeValue>) {
+        let put_req = PutRequest::builder().set_item(Some(item)).build().unwrap();
+        let req = WriteRequest::builder().put_request(put_req).build();
+        self.write_one_req(req);
+    }
+
+    fn write_one_delete(&mut self, key: HashMap<String, AttributeValue>) {
+        let key = key
+            .into_iter()
+            .filter(|(k, _)| self.dynamodb_keys.contains(k))
+            .collect();
+        let del_req = DeleteRequest::builder().set_key(Some(key)).build().unwrap();
+        let req = WriteRequest::builder().delete_request(del_req).build();
+        self.write_one_req(req);
+    }
+
+    fn write_one_req(&mut self, req: WriteRequest) {
+        let r_req = DynamoDbRequest {
+            inner: req,
+            key_items: self.dynamodb_keys.clone(),
+        };
+        if let Some(v) = r_req.extract_pk_values() {
+            self.request_items.retain(|item| {
+                !item
+                    .extract_pk_values()
+                    .unwrap_or_default()
+                    .iter()
+                    .all(|x| v.contains(x))
+            });
+        }
+        self.request_items.push(r_req);
+    }
+
+    async fn write_chunk(&mut self) -> Result<()> {
+        if !self.request_items.is_empty() {
+            let table = self.table.clone();
+            let req_items = std::mem::take(&mut self.request_items)
+                .into_iter()
+                .map(|r| r.inner)
+                .collect();
+            let reqs = hashmap! {
+                table => req_items,
+            };
+            self.client
+                .batch_write_item()
+                .set_request_items(Some(reqs))
+                .return_consumed_capacity(ReturnConsumedCapacity::None)
+                .return_item_collection_metrics(ReturnItemCollectionMetrics::None)
+                .send()
+                .await
+                .map_err(|e| {
+                    SinkError::DynamoDb(
+                        anyhow!(e).context("failed to delete item from DynamoDB sink"),
+                    )
+                })?;
+        }
+
+        Ok(())
+    }
+}
+
+pub struct DynamoDbSinkWriter {
+    max_batch_rows: usize,
+    payload_writer: DynamoDbPayloadWriter,
+    formatter: DynamoDbFormatter,
+}
+
+impl DynamoDbSinkWriter {
+    pub async fn new(config: DynamoDbConfig, schema: Schema) -> Result<Self> {
+        let client = config.build_client().await?;
+        let table_name = &config.table;
+        let output = client
+            .describe_table()
+            .table_name(table_name)
+            .send()
+            .await
+            .map_err(|e| anyhow!(e))?;
+        let Some(table) = output.table else {
+            return Err(SinkError::DynamoDb(anyhow!(
+                "table {} not found",
+                table_name
+            )));
+        };
+        let dynamodb_keys = table
+            .key_schema
+            .unwrap_or_default()
+            .into_iter()
+            .map(|k| k.attribute_name)
+            .collect();
+
+        let payload_writer = DynamoDbPayloadWriter {
+            request_items: Vec::new(),
+            client,
+            table: config.table,
+            dynamodb_keys,
+        };
+
+        Ok(Self {
+            max_batch_rows: config.max_batch_rows,
+            payload_writer,
+            formatter: DynamoDbFormatter { schema },
+        })
+    }
+
+    async fn write_chunk_inner(&mut self, chunk: StreamChunk) -> Result<()> {
+        for (op, row) in chunk.rows() {
+            let items = self.formatter.format_row(row)?;
+            match op {
+                Op::Insert | Op::UpdateInsert => {
+                    self.payload_writer.write_one_insert(items);
+                }
+                Op::Delete => {
+                    self.payload_writer.write_one_delete(items);
+                }
+                Op::UpdateDelete => {}
+            }
+        }
+        if self.payload_writer.request_items.len() >= self.max_batch_rows {
+            self.payload_writer.write_chunk().await?;
+        }
+        Ok(())
+    }
+
+    async fn flush(&mut self) -> Result<()> {
+        self.payload_writer.write_chunk().await
+    }
+}
+
+impl AsyncTruncateSinkWriter for DynamoDbSinkWriter {
+    async fn write_chunk<'a>(
+        &'a mut self,
+        chunk: StreamChunk,
+        _add_future: DeliveryFutureManagerAddFuture<'a, Self::DeliveryFuture>,
+    ) -> Result<()> {
+        self.write_chunk_inner(chunk).await
+    }
+
+    async fn barrier(&mut self, is_checkpoint: bool) -> Result<()> {
+        if is_checkpoint {
+            self.flush().await?;
+        }
+        Ok(())
+    }
+}
+
+struct DynamoDbFormatter {
+    schema: Schema,
+}
+
+impl DynamoDbFormatter {
+    fn format_row(&self, row: RowRef<'_>) -> Result<HashMap<String, AttributeValue>> {
+        row.iter()
+            .zip_eq_debug((self.schema.clone()).into_fields())
+            .map(|(scalar, field)| {
+                map_data_type(scalar, &field.data_type()).map(|attr| (field.name, attr))
+            })
+            .collect()
+    }
+}
+
+fn map_data_type(
+    scalar_ref: Option<ScalarRefImpl<'_>>,
+    data_type: &DataType,
+) -> Result<AttributeValue> {
+    let Some(scalar_ref) = scalar_ref else {
+        return Ok(AttributeValue::Null(true));
+    };
+    let attr = match data_type {
+        DataType::Int16
+        | DataType::Int32
+        | DataType::Int64
+        | DataType::Int256
+        | DataType::Float32
+        | DataType::Float64
+        | DataType::Decimal
+        | DataType::Serial => AttributeValue::N(scalar_ref.to_text_with_type(data_type)),
+        // TODO: jsonb as dynamic type (https://github.com/risingwavelabs/risingwave/issues/11699)
+        DataType::Varchar
+        | DataType::Interval
+        | DataType::Date
+        | DataType::Time
+        | DataType::Timestamp
+        | DataType::Timestamptz
+        | DataType::Jsonb => AttributeValue::S(scalar_ref.to_text_with_type(data_type)),
+        DataType::Boolean => AttributeValue::Bool(scalar_ref.into_bool()),
+        DataType::Bytea => AttributeValue::B(Blob::new(scalar_ref.into_bytea())),
+        DataType::List(datatype) => {
+            let list_attr = scalar_ref
+                .into_list()
+                .iter()
+                .map(|x| map_data_type(x, datatype))
+                .collect::<Result<Vec<_>>>()?;
+            AttributeValue::L(list_attr)
+        }
+        DataType::Struct(st) => {
+            let mut map = HashMap::with_capacity(st.len());
+            for (sub_datum_ref, sub_field) in
+                scalar_ref.into_struct().iter_fields_ref().zip_eq_debug(
+                    st.iter()
+                        .map(|(name, dt)| Field::with_name(dt.clone(), name)),
+                )
+            {
+                let attr = map_data_type(sub_datum_ref, &sub_field.data_type())?;
+                map.insert(sub_field.name.clone(), attr);
+            }
+            AttributeValue::M(map)
+        }
+    };
+    Ok(attr)
+}
diff --git a/src/connector/src/sink/elasticsearch.rs b/src/connector/src/sink/elasticsearch.rs
index 578c768f1ce30..236f90823c505 100644
--- a/src/connector/src/sink/elasticsearch.rs
+++ b/src/connector/src/sink/elasticsearch.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 
 use anyhow::anyhow;
 use risingwave_common::array::{
@@ -38,7 +38,7 @@ impl StreamChunkConverter {
         sink_name: &str,
         schema: Schema,
         pk_indices: &Vec<usize>,
-        properties: &HashMap<String, String>,
+        properties: &BTreeMap<String, String>,
     ) -> Result<Self> {
         if sink_name == ElasticSearchSink::SINK_NAME {
             let index_column = properties
diff --git a/src/connector/src/sink/encoder/avro.rs b/src/connector/src/sink/encoder/avro.rs
index 924beb281eda7..8122126727298 100644
--- a/src/connector/src/sink/encoder/avro.rs
+++ b/src/connector/src/sink/encoder/avro.rs
@@ -385,6 +385,10 @@ fn encode_field<D: MaybeData>(
             AvroSchema::Long => maybe.on_base(|s| Ok(Value::Long(s.into_int64())))?,
             _ => return no_match_err(),
         },
+        DataType::Serial => match inner {
+            AvroSchema::Long => maybe.on_base(|s| Ok(Value::Long(s.into_serial().into_inner())))?,
+            _ => return no_match_err(),
+        },
         DataType::Struct(st) => match inner {
             AvroSchema::Record { .. } => maybe.on_struct(st, inner)?,
             _ => return no_match_err(),
@@ -447,7 +451,7 @@ fn encode_field<D: MaybeData>(
         DataType::Decimal => return no_match_err(),
         DataType::Jsonb => return no_match_err(),
         // Group D: unsupported
-        DataType::Serial | DataType::Int256 => {
+        DataType::Int256 => {
             return no_match_err();
         }
     };
@@ -526,7 +530,14 @@ mod tests {
 
         test_ok(
             &DataType::Int64,
-            Some(ScalarImpl::Int64(std::i64::MAX)),
+            Some(ScalarImpl::Int64(i64::MAX)),
+            r#""long""#,
+            Value::Long(i64::MAX),
+        );
+
+        test_ok(
+            &DataType::Serial,
+            Some(ScalarImpl::Serial(i64::MAX.into())),
             r#""long""#,
             Value::Long(i64::MAX),
         );
diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs
index bbd424d5db036..e1ce9e61b6a1e 100644
--- a/src/connector/src/sink/encoder/json.rs
+++ b/src/connector/src/sink/encoder/json.rs
@@ -110,19 +110,6 @@ impl JsonEncoder {
         }
     }
 
-    pub fn new_with_bigquery(schema: Schema, col_indices: Option<Vec<usize>>) -> Self {
-        Self {
-            schema,
-            col_indices,
-            time_handling_mode: TimeHandlingMode::Milli,
-            date_handling_mode: DateHandlingMode::String,
-            timestamp_handling_mode: TimestampHandlingMode::String,
-            timestamptz_handling_mode: TimestamptzHandlingMode::UtcString,
-            custom_json_type: CustomJsonType::BigQuery,
-            kafka_connect: None,
-        }
-    }
-
     pub fn with_kafka_connect(self, kafka_connect: KafkaConnectParams) -> Self {
         Self {
             kafka_connect: Some(Arc::new(kafka_connect)),
@@ -200,14 +187,7 @@ fn datum_to_json_object(
 ) -> ArrayResult<Value> {
     let scalar_ref = match datum {
         None => {
-            if let CustomJsonType::BigQuery = custom_json_type
-                && matches!(field.data_type(), DataType::List(_))
-            {
-                // Bigquery need to convert null of array to empty array https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
-                return Ok(Value::Array(vec![]));
-            } else {
-                return Ok(Value::Null);
-            }
+            return Ok(Value::Null);
         }
         Some(datum) => datum,
     };
@@ -229,6 +209,10 @@ fn datum_to_json_object(
         (DataType::Int64, ScalarRefImpl::Int64(v)) => {
             json!(v)
         }
+        (DataType::Serial, ScalarRefImpl::Serial(v)) => {
+            // The serial type needs to be handled as a string to prevent primary key conflicts caused by the precision issues of JSON numbers.
+            json!(format!("{:#018x}", v.into_inner()))
+        }
         (DataType::Float32, ScalarRefImpl::Float32(v)) => {
             json!(f32::from(v))
         }
@@ -245,10 +229,7 @@ fn datum_to_json_object(
                 v.rescale(*s as u32);
                 json!(v.to_text())
             }
-            CustomJsonType::Es
-            | CustomJsonType::None
-            | CustomJsonType::BigQuery
-            | CustomJsonType::StarRocks => {
+            CustomJsonType::Es | CustomJsonType::None | CustomJsonType::StarRocks => {
                 json!(v.to_text())
             }
         },
@@ -300,7 +281,7 @@ fn datum_to_json_object(
         }
         (DataType::Jsonb, ScalarRefImpl::Jsonb(jsonb_ref)) => match custom_json_type {
             CustomJsonType::Es | CustomJsonType::StarRocks => JsonbVal::from(jsonb_ref).take(),
-            CustomJsonType::Doris(_) | CustomJsonType::None | CustomJsonType::BigQuery => {
+            CustomJsonType::Doris(_) | CustomJsonType::None => {
                 json!(jsonb_ref.to_string())
             }
         },
@@ -351,7 +332,7 @@ fn datum_to_json_object(
                         "starrocks can't support struct".to_string(),
                     ));
                 }
-                CustomJsonType::Es | CustomJsonType::None | CustomJsonType::BigQuery => {
+                CustomJsonType::Es | CustomJsonType::None => {
                     let mut map = Map::with_capacity(st.len());
                     for (sub_datum_ref, sub_field) in struct_ref.iter_fields_ref().zip_eq_debug(
                         st.iter()
@@ -425,7 +406,7 @@ pub(crate) fn schema_type_mapping(rw_type: &DataType) -> &'static str {
         DataType::List(_) => "array",
         DataType::Bytea => "bytes",
         DataType::Jsonb => "string",
-        DataType::Serial => "int32",
+        DataType::Serial => "string",
         DataType::Int256 => "string",
     }
 }
@@ -457,13 +438,13 @@ fn type_as_json_schema(rw_type: &DataType) -> Map<String, Value> {
 
 #[cfg(test)]
 mod tests {
-
     use risingwave_common::types::{
-        DataType, Date, Decimal, Interval, Scalar, ScalarImpl, StructRef, StructType, StructValue,
-        Time, Timestamp,
+        Date, Decimal, Interval, Scalar, ScalarImpl, StructRef, StructType, StructValue, Time,
+        Timestamp,
     };
 
     use super::*;
+
     #[test]
     fn test_to_json_basic_type() {
         let mock_field = Field {
@@ -508,7 +489,7 @@ mod tests {
                 data_type: DataType::Int64,
                 ..mock_field.clone()
             },
-            Some(ScalarImpl::Int64(std::i64::MAX).as_scalar_ref_impl()),
+            Some(ScalarImpl::Int64(i64::MAX).as_scalar_ref_impl()),
             DateHandlingMode::FromCe,
             TimestampHandlingMode::String,
             TimestamptzHandlingMode::UtcString,
@@ -518,7 +499,25 @@ mod tests {
         .unwrap();
         assert_eq!(
             serde_json::to_string(&int64_value).unwrap(),
-            std::i64::MAX.to_string()
+            i64::MAX.to_string()
+        );
+
+        let serial_value = datum_to_json_object(
+            &Field {
+                data_type: DataType::Serial,
+                ..mock_field.clone()
+            },
+            Some(ScalarImpl::Serial(i64::MAX.into()).as_scalar_ref_impl()),
+            DateHandlingMode::FromCe,
+            TimestampHandlingMode::String,
+            TimestamptzHandlingMode::UtcString,
+            TimeHandlingMode::Milli,
+            &CustomJsonType::None,
+        )
+        .unwrap();
+        assert_eq!(
+            serde_json::to_string(&serial_value).unwrap(),
+            format!("\"{:#018x}\"", i64::MAX)
         );
 
         // https://github.com/debezium/debezium/blob/main/debezium-core/src/main/java/io/debezium/time/ZonedTimestamp.java
@@ -843,7 +842,7 @@ mod tests {
         let schema = json_converter_with_schema(json!({}), "test".to_owned(), fields.iter())
             ["schema"]
             .to_string();
-        let ans = r#"{"fields":[{"field":"v1","optional":true,"type":"boolean"},{"field":"v2","optional":true,"type":"int16"},{"field":"v3","optional":true,"type":"int32"},{"field":"v4","optional":true,"type":"float"},{"field":"v5","optional":true,"type":"string"},{"field":"v6","optional":true,"type":"int32"},{"field":"v7","optional":true,"type":"string"},{"field":"v8","optional":true,"type":"int64"},{"field":"v9","optional":true,"type":"string"},{"field":"v10","fields":[{"field":"a","optional":true,"type":"int64"},{"field":"b","optional":true,"type":"string"},{"field":"c","fields":[{"field":"aa","optional":true,"type":"int64"},{"field":"bb","optional":true,"type":"double"}],"optional":true,"type":"struct"}],"optional":true,"type":"struct"},{"field":"v11","items":{"items":{"fields":[{"field":"aa","optional":true,"type":"int64"},{"field":"bb","optional":true,"type":"double"}],"optional":true,"type":"struct"},"optional":true,"type":"array"},"optional":true,"type":"array"},{"field":"12","optional":true,"type":"string"},{"field":"13","optional":true,"type":"int32"},{"field":"14","optional":true,"type":"string"}],"name":"test","optional":false,"type":"struct"}"#;
+        let ans = r#"{"fields":[{"field":"v1","optional":true,"type":"boolean"},{"field":"v2","optional":true,"type":"int16"},{"field":"v3","optional":true,"type":"int32"},{"field":"v4","optional":true,"type":"float"},{"field":"v5","optional":true,"type":"string"},{"field":"v6","optional":true,"type":"int32"},{"field":"v7","optional":true,"type":"string"},{"field":"v8","optional":true,"type":"int64"},{"field":"v9","optional":true,"type":"string"},{"field":"v10","fields":[{"field":"a","optional":true,"type":"int64"},{"field":"b","optional":true,"type":"string"},{"field":"c","fields":[{"field":"aa","optional":true,"type":"int64"},{"field":"bb","optional":true,"type":"double"}],"optional":true,"type":"struct"}],"optional":true,"type":"struct"},{"field":"v11","items":{"items":{"fields":[{"field":"aa","optional":true,"type":"int64"},{"field":"bb","optional":true,"type":"double"}],"optional":true,"type":"struct"},"optional":true,"type":"array"},"optional":true,"type":"array"},{"field":"12","optional":true,"type":"string"},{"field":"13","optional":true,"type":"string"},{"field":"14","optional":true,"type":"string"}],"name":"test","optional":false,"type":"struct"}"#;
         assert_eq!(schema, ans);
     }
 }
diff --git a/src/connector/src/sink/encoder/mod.rs b/src/connector/src/sink/encoder/mod.rs
index 0b6899dbad955..40d85625c5b1d 100644
--- a/src/connector/src/sink/encoder/mod.rs
+++ b/src/connector/src/sink/encoder/mod.rs
@@ -24,6 +24,7 @@ mod avro;
 mod json;
 mod proto;
 pub mod template;
+pub mod text;
 
 pub use avro::{AvroEncoder, AvroHeader};
 pub use json::JsonEncoder;
@@ -57,7 +58,7 @@ pub trait RowEncoder {
 /// * an json object
 /// * a protobuf message
 /// * an avro record
-/// into
+///   into
 /// * string (required by kinesis key)
 /// * bytes
 ///
@@ -144,8 +145,6 @@ pub enum CustomJsonType {
     Es,
     // starrocks' need jsonb is struct
     StarRocks,
-    // bigquery need null array -> []
-    BigQuery,
     None,
 }
 
diff --git a/src/connector/src/sink/encoder/proto.rs b/src/connector/src/sink/encoder/proto.rs
index a5f1090dbafaf..a0e4d41dc58de 100644
--- a/src/connector/src/sink/encoder/proto.rs
+++ b/src/connector/src/sink/encoder/proto.rs
@@ -77,7 +77,7 @@ impl ProtoEncoder {
 }
 
 pub struct ProtoEncoded {
-    message: DynamicMessage,
+    pub message: DynamicMessage,
     header: ProtoHeader,
 }
 
@@ -205,6 +205,7 @@ impl MaybeData for () {
 /// * Top level is always a message.
 /// * All message fields can be omitted in proto3.
 /// * All repeated elements must have a value.
+///
 /// So we handle [`ScalarRefImpl`] rather than [`DatumRef`] here.
 impl MaybeData for ScalarRefImpl<'_> {
     type Out = Value;
@@ -275,6 +276,7 @@ fn encode_fields<'a>(
 
 // Full name of Well-Known Types
 const WKT_TIMESTAMP: &str = "google.protobuf.Timestamp";
+#[expect(dead_code)]
 const WKT_BOOL_VALUE: &str = "google.protobuf.BoolValue";
 
 /// Handles both `validate` (without actual data) and `encode`.
@@ -307,7 +309,6 @@ fn encode_field<D: MaybeData>(
             proto_field.kind()
         )))
     };
-
     let value = match &data_type {
         // Group A: perfect match between RisingWave types and ProtoBuf types
         DataType::Boolean => match (expect_list, proto_field.kind()) {
@@ -364,18 +365,59 @@ fn encode_field<D: MaybeData>(
                     Ok(Value::Message(message.transcode_to_dynamic()))
                 })?
             }
+            (false, Kind::String) => {
+                maybe.on_base(|s| Ok(Value::String(s.into_timestamptz().to_string())))?
+            }
+            _ => return no_match_err(),
+        },
+        DataType::Jsonb => match (expect_list, proto_field.kind()) {
+            (false, Kind::String) => {
+                maybe.on_base(|s| Ok(Value::String(s.into_jsonb().to_string())))?
+            }
+            _ => return no_match_err(), /* Value, NullValue, Struct (map), ListValue
+                                         * Group C: experimental */
+        },
+        DataType::Int16 => match (expect_list, proto_field.kind()) {
+            (false, Kind::Int64) => maybe.on_base(|s| Ok(Value::I64(s.into_int16() as i64)))?,
             _ => return no_match_err(),
         },
-        DataType::Jsonb => return no_match_err(), // Value, NullValue, Struct (map), ListValue
-        // Group C: experimental
-        DataType::Int16 => return no_match_err(),
-        DataType::Date => return no_match_err(), // google.type.Date
-        DataType::Time => return no_match_err(), // google.type.TimeOfDay
-        DataType::Timestamp => return no_match_err(), // google.type.DateTime
-        DataType::Decimal => return no_match_err(), // google.type.Decimal
-        DataType::Interval => return no_match_err(),
-        // Group D: unsupported
-        DataType::Serial | DataType::Int256 => {
+        DataType::Date => match (expect_list, proto_field.kind()) {
+            (false, Kind::Int32) => {
+                maybe.on_base(|s| Ok(Value::I32(s.into_date().get_nums_days_unix_epoch())))?
+            }
+            _ => return no_match_err(), // google.type.Date
+        },
+        DataType::Time => match (expect_list, proto_field.kind()) {
+            (false, Kind::String) => {
+                maybe.on_base(|s| Ok(Value::String(s.into_time().to_string())))?
+            }
+            _ => return no_match_err(), // google.type.TimeOfDay
+        },
+        DataType::Timestamp => match (expect_list, proto_field.kind()) {
+            (false, Kind::String) => {
+                maybe.on_base(|s| Ok(Value::String(s.into_timestamp().to_string())))?
+            }
+            _ => return no_match_err(), // google.type.DateTime
+        },
+        DataType::Decimal => match (expect_list, proto_field.kind()) {
+            (false, Kind::String) => {
+                maybe.on_base(|s| Ok(Value::String(s.into_decimal().to_string())))?
+            }
+            _ => return no_match_err(), // google.type.Decimal
+        },
+        DataType::Interval => match (expect_list, proto_field.kind()) {
+            (false, Kind::String) => {
+                maybe.on_base(|s| Ok(Value::String(s.into_interval().as_iso_8601())))?
+            }
+            _ => return no_match_err(), // Group D: unsupported
+        },
+        DataType::Serial => match (expect_list, proto_field.kind()) {
+            (false, Kind::Int64) => {
+                maybe.on_base(|s| Ok(Value::I64(s.into_serial().as_row_id())))?
+            }
+            _ => return no_match_err(), // Group D: unsupported
+        },
+        DataType::Int256 => {
             return no_match_err();
         }
     };
@@ -398,7 +440,6 @@ mod tests {
         let pool_bytes = std::fs::read(pool_path).unwrap();
         let pool = prost_reflect::DescriptorPool::decode(pool_bytes.as_ref()).unwrap();
         let descriptor = pool.get_message_by_name("recursive.AllTypes").unwrap();
-
         let schema = Schema::new(vec![
             Field::with_name(DataType::Boolean, "bool_field"),
             Field::with_name(DataType::Varchar, "string_field"),
diff --git a/src/connector/src/sink/encoder/text.rs b/src/connector/src/sink/encoder/text.rs
new file mode 100644
index 0000000000000..734ac8bd6a425
--- /dev/null
+++ b/src/connector/src/sink/encoder/text.rs
@@ -0,0 +1,108 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::catalog::Schema;
+use risingwave_common::types::{DataType, ToText};
+
+use super::RowEncoder;
+
+pub struct TextEncoder {
+    pub schema: Schema,
+    // the column must contain only one element
+    pub col_index: usize,
+}
+
+impl TextEncoder {
+    pub fn new(schema: Schema, col_index: usize) -> Self {
+        Self { schema, col_index }
+    }
+}
+
+impl RowEncoder for TextEncoder {
+    type Output = String;
+
+    fn schema(&self) -> &risingwave_common::catalog::Schema {
+        &self.schema
+    }
+
+    fn col_indices(&self) -> Option<&[usize]> {
+        Some(std::slice::from_ref(&self.col_index))
+    }
+
+    fn encode_cols(
+        &self,
+        row: impl risingwave_common::row::Row,
+        col_indices: impl Iterator<Item = usize>,
+    ) -> crate::sink::Result<Self::Output> {
+        // It is guaranteed by the caller that col_indices contains only one element
+        let mut result = String::new();
+        for col_index in col_indices {
+            let datum = row.datum_at(col_index);
+            let data_type = &self.schema.fields[col_index].data_type;
+            if data_type == &DataType::Boolean {
+                result = if let Some(scalar_impl) = datum {
+                    scalar_impl.into_bool().to_string()
+                } else {
+                    "NULL".to_string()
+                }
+            } else {
+                result = datum.to_text_with_type(data_type);
+            }
+        }
+
+        Ok(result)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use risingwave_common::catalog::Field;
+    use risingwave_common::row::OwnedRow;
+    use risingwave_common::types::ScalarImpl;
+
+    use super::*;
+
+    #[test]
+    fn test_text_encoder_ser_bool() {
+        let schema = Schema::new(vec![Field::with_name(DataType::Boolean, "col1")]);
+        let encoder = TextEncoder::new(schema, 0);
+
+        let row = OwnedRow::new(vec![Some(ScalarImpl::Bool(true))]);
+        assert_eq!(
+            encoder
+                .encode_cols(&row, std::iter::once(0))
+                .unwrap()
+                .as_str(),
+            "true"
+        );
+
+        let row = OwnedRow::new(vec![Some(ScalarImpl::Bool(false))]);
+        assert_eq!(
+            encoder
+                .encode_cols(&row, std::iter::once(0))
+                .unwrap()
+                .as_str(),
+            "false"
+        );
+
+        let row = OwnedRow::new(vec![None]);
+        assert_eq!(
+            encoder
+                .encode_cols(&row, std::iter::once(0))
+                .unwrap()
+                .as_str(),
+            "NULL"
+        );
+    }
+}
diff --git a/src/connector/src/sink/formatter/mod.rs b/src/connector/src/sink/formatter/mod.rs
index d923d337a3ffb..4628a925da98d 100644
--- a/src/connector/src/sink/formatter/mod.rs
+++ b/src/connector/src/sink/formatter/mod.rs
@@ -24,10 +24,12 @@ mod upsert;
 pub use append_only::AppendOnlyFormatter;
 pub use debezium_json::{DebeziumAdapterOpts, DebeziumJsonFormatter};
 use risingwave_common::catalog::Schema;
+use risingwave_common::types::DataType;
 pub use upsert::UpsertFormatter;
 
 use super::catalog::{SinkEncode, SinkFormat, SinkFormatDesc};
 use super::encoder::template::TemplateEncoder;
+use super::encoder::text::TextEncoder;
 use super::encoder::{
     DateHandlingMode, KafkaConnectParams, TimeHandlingMode, TimestamptzHandlingMode,
 };
@@ -43,9 +45,9 @@ pub trait SinkFormatter {
     type V;
 
     /// * Key may be None so that messages are partitioned using round-robin.
-    /// For example append-only without `primary_key` (aka `downstream_pk`) set.
+    ///   For example append-only without `primary_key` (aka `downstream_pk`) set.
     /// * Value may be None so that messages with same key are removed during log compaction.
-    /// For example debezium tombstone event.
+    ///   For example debezium tombstone event.
     fn format_chunk(
         &self,
         chunk: &StreamChunk,
@@ -68,13 +70,226 @@ macro_rules! tri {
 }
 
 pub enum SinkFormatterImpl {
+    // append-only
     AppendOnlyJson(AppendOnlyFormatter<JsonEncoder, JsonEncoder>),
+    AppendOnlyTextJson(AppendOnlyFormatter<TextEncoder, JsonEncoder>),
+    AppendOnlyAvro(AppendOnlyFormatter<AvroEncoder, AvroEncoder>),
+    AppendOnlyTextAvro(AppendOnlyFormatter<TextEncoder, AvroEncoder>),
     AppendOnlyProto(AppendOnlyFormatter<JsonEncoder, ProtoEncoder>),
+    AppendOnlyTextProto(AppendOnlyFormatter<TextEncoder, ProtoEncoder>),
+    AppendOnlyTemplate(AppendOnlyFormatter<TemplateEncoder, TemplateEncoder>),
+    AppendOnlyTextTemplate(AppendOnlyFormatter<TextEncoder, TemplateEncoder>),
+    // upsert
     UpsertJson(UpsertFormatter<JsonEncoder, JsonEncoder>),
+    UpsertTextJson(UpsertFormatter<TextEncoder, JsonEncoder>),
     UpsertAvro(UpsertFormatter<AvroEncoder, AvroEncoder>),
-    DebeziumJson(DebeziumJsonFormatter),
-    AppendOnlyTemplate(AppendOnlyFormatter<TemplateEncoder, TemplateEncoder>),
+    UpsertTextAvro(UpsertFormatter<TextEncoder, AvroEncoder>),
     UpsertTemplate(UpsertFormatter<TemplateEncoder, TemplateEncoder>),
+    UpsertTextTemplate(UpsertFormatter<TextEncoder, TemplateEncoder>),
+    // debezium
+    DebeziumJson(DebeziumJsonFormatter),
+}
+
+#[derive(Debug, Clone)]
+pub struct EncoderParams<'a> {
+    format_desc: &'a SinkFormatDesc,
+    schema: Schema,
+    db_name: String,
+    sink_from_name: String,
+    topic: &'a str,
+}
+
+/// Each encoder shall be able to be built from parameters.
+///
+/// This is not part of `RowEncoder` trait, because that one is about how an encoder completes its
+/// own job as a self-contained unit, with a custom `new` asking for only necessary info; while this
+/// one is about how different encoders can be selected from a common SQL interface.
+pub trait EncoderBuild: Sized {
+    /// Pass `pk_indices: None` for value/payload and `Some` for key. Certain encoder builds
+    /// differently when used as key vs value.
+    async fn build(params: EncoderParams<'_>, pk_indices: Option<Vec<usize>>) -> Result<Self>;
+}
+
+impl EncoderBuild for JsonEncoder {
+    async fn build(b: EncoderParams<'_>, pk_indices: Option<Vec<usize>>) -> Result<Self> {
+        let timestamptz_mode = TimestamptzHandlingMode::from_options(&b.format_desc.options)?;
+        let encoder = JsonEncoder::new(
+            b.schema,
+            pk_indices,
+            DateHandlingMode::FromCe,
+            TimestampHandlingMode::Milli,
+            timestamptz_mode,
+            TimeHandlingMode::Milli,
+        );
+        let encoder = if let Some(s) = b.format_desc.options.get("schemas.enable") {
+            match s.to_lowercase().parse::<bool>() {
+                Ok(true) => {
+                    let kafka_connect = KafkaConnectParams {
+                        schema_name: format!("{}.{}", b.db_name, b.sink_from_name),
+                    };
+                    encoder.with_kafka_connect(kafka_connect)
+                }
+                Ok(false) => encoder,
+                _ => {
+                    return Err(SinkError::Config(anyhow!(
+                        "schemas.enable is expected to be `true` or `false`, got {s}",
+                    )));
+                }
+            }
+        } else {
+            encoder
+        };
+        Ok(encoder)
+    }
+}
+
+impl EncoderBuild for ProtoEncoder {
+    async fn build(b: EncoderParams<'_>, pk_indices: Option<Vec<usize>>) -> Result<Self> {
+        // TODO: better to be a compile-time assert
+        assert!(pk_indices.is_none());
+        // By passing `None` as `aws_auth_props`, reading from `s3://` not supported yet.
+        let (descriptor, sid) =
+            crate::schema::protobuf::fetch_descriptor(&b.format_desc.options, b.topic, None)
+                .await
+                .map_err(|e| SinkError::Config(anyhow!(e)))?;
+        let header = match sid {
+            None => ProtoHeader::None,
+            Some(sid) => ProtoHeader::ConfluentSchemaRegistry(sid),
+        };
+        ProtoEncoder::new(b.schema, None, descriptor, header)
+    }
+}
+
+impl EncoderBuild for TextEncoder {
+    async fn build(params: EncoderParams<'_>, pk_indices: Option<Vec<usize>>) -> Result<Self> {
+        let Some(pk_indices) = pk_indices else {
+            return Err(SinkError::Config(anyhow!(
+                "TextEncoder requires primary key columns to be specified"
+            )));
+        };
+        if pk_indices.len() != 1 {
+            return Err(SinkError::Config(anyhow!(
+                    "The key encode is TEXT, but the primary key has {} columns. The key encode TEXT requires the primary key to be a single column",
+                    pk_indices.len()
+                )));
+        }
+
+        let schema_ref = params.schema.fields().get(pk_indices[0]).ok_or_else(|| {
+            SinkError::Config(anyhow!(
+                "The primary key column index {} is out of bounds in schema {:?}",
+                pk_indices[0],
+                params.schema
+            ))
+        })?;
+        match &schema_ref.data_type() {
+            DataType::Varchar
+            | DataType::Boolean
+            | DataType::Int16
+            | DataType::Int32
+            | DataType::Int64
+            | DataType::Int256
+            | DataType::Serial => {}
+            _ => {
+                // why we don't allow float as text for key encode: https://github.com/risingwavelabs/risingwave/pull/16377#discussion_r1591864960
+                return Err(SinkError::Config(
+                    anyhow!(
+                            "The key encode is TEXT, but the primary key column {} has type {}. The key encode TEXT requires the primary key column to be of type varchar, bool, small int, int, big int, serial or rw_int256.",
+                            schema_ref.name,
+                            schema_ref.data_type
+                        ),
+                ));
+            }
+        }
+
+        Ok(Self::new(params.schema, pk_indices[0]))
+    }
+}
+
+impl EncoderBuild for AvroEncoder {
+    async fn build(b: EncoderParams<'_>, pk_indices: Option<Vec<usize>>) -> Result<Self> {
+        let loader =
+            crate::schema::SchemaLoader::from_format_options(b.topic, &b.format_desc.options)
+                .map_err(|e| SinkError::Config(anyhow!(e)))?;
+
+        let (schema_id, avro) = match pk_indices {
+            Some(_) => loader
+                .load_key_schema()
+                .await
+                .map_err(|e| SinkError::Config(anyhow!(e)))?,
+            None => loader
+                .load_val_schema()
+                .await
+                .map_err(|e| SinkError::Config(anyhow!(e)))?,
+        };
+        AvroEncoder::new(
+            b.schema,
+            pk_indices,
+            std::sync::Arc::new(avro),
+            AvroHeader::ConfluentSchemaRegistry(schema_id),
+        )
+    }
+}
+
+impl EncoderBuild for TemplateEncoder {
+    async fn build(b: EncoderParams<'_>, pk_indices: Option<Vec<usize>>) -> Result<Self> {
+        let option_name = match pk_indices {
+            Some(_) => KEY_FORMAT,
+            None => VALUE_FORMAT,
+        };
+        let template = b.format_desc.options.get(option_name).ok_or_else(|| {
+            SinkError::Config(anyhow!(
+                "Cannot find '{option_name}',please set it or use JSON"
+            ))
+        })?;
+        Ok(TemplateEncoder::new(b.schema, pk_indices, template.clone()))
+    }
+}
+
+struct FormatterParams<'a> {
+    builder: EncoderParams<'a>,
+    pk_indices: Vec<usize>,
+}
+
+/// Each formatter shall be able to be built from parameters.
+///
+/// This is not part of `SinkFormatter` trait, because that is about how a formatter completes its
+/// own job as a self-contained unit, with a custom `new` asking for only necessary info; while this
+/// one is about how different formatters can be selected from a common SQL interface.
+trait FormatterBuild: Sized {
+    async fn build(b: FormatterParams<'_>) -> Result<Self>;
+}
+
+impl<KE: EncoderBuild, VE: EncoderBuild> FormatterBuild for AppendOnlyFormatter<KE, VE> {
+    async fn build(b: FormatterParams<'_>) -> Result<Self> {
+        let key_encoder = match b.pk_indices.is_empty() {
+            true => None,
+            false => Some(KE::build(b.builder.clone(), Some(b.pk_indices)).await?),
+        };
+        let val_encoder = VE::build(b.builder, None).await?;
+        Ok(AppendOnlyFormatter::new(key_encoder, val_encoder))
+    }
+}
+
+impl<KE: EncoderBuild, VE: EncoderBuild> FormatterBuild for UpsertFormatter<KE, VE> {
+    async fn build(b: FormatterParams<'_>) -> Result<Self> {
+        let key_encoder = KE::build(b.builder.clone(), Some(b.pk_indices)).await?;
+        let val_encoder = VE::build(b.builder, None).await?;
+        Ok(UpsertFormatter::new(key_encoder, val_encoder))
+    }
+}
+
+impl FormatterBuild for DebeziumJsonFormatter {
+    async fn build(b: FormatterParams<'_>) -> Result<Self> {
+        assert_eq!(b.builder.format_desc.encode, SinkEncode::Json);
+
+        Ok(DebeziumJsonFormatter::new(
+            b.builder.schema,
+            b.pk_indices,
+            b.builder.db_name,
+            b.builder.sink_from_name,
+            DebeziumAdapterOpts::default(),
+        ))
+    }
 }
 
 impl SinkFormatterImpl {
@@ -86,187 +301,83 @@ impl SinkFormatterImpl {
         sink_from_name: String,
         topic: &str,
     ) -> Result<Self> {
-        let err_unsupported = || {
-            Err(SinkError::Config(anyhow!(
-                "sink format/encode unsupported: {:?} {:?}",
-                format_desc.format,
-                format_desc.encode,
-            )))
+        use {SinkEncode as E, SinkFormat as F, SinkFormatterImpl as Impl};
+        let p = FormatterParams {
+            builder: EncoderParams {
+                format_desc,
+                schema,
+                db_name,
+                sink_from_name,
+                topic,
+            },
+            pk_indices,
         };
-        let timestamptz_mode = TimestamptzHandlingMode::from_options(&format_desc.options)?;
-
-        match format_desc.format {
-            SinkFormat::AppendOnly => {
-                let key_encoder = (!pk_indices.is_empty()).then(|| {
-                    JsonEncoder::new(
-                        schema.clone(),
-                        Some(pk_indices.clone()),
-                        DateHandlingMode::FromCe,
-                        TimestampHandlingMode::Milli,
-                        timestamptz_mode,
-                        TimeHandlingMode::Milli,
-                    )
-                });
-
-                match format_desc.encode {
-                    SinkEncode::Json => {
-                        let val_encoder = JsonEncoder::new(
-                            schema,
-                            None,
-                            DateHandlingMode::FromCe,
-                            TimestampHandlingMode::Milli,
-                            timestamptz_mode,
-                            TimeHandlingMode::Milli,
-                        );
-                        let formatter = AppendOnlyFormatter::new(key_encoder, val_encoder);
-                        Ok(SinkFormatterImpl::AppendOnlyJson(formatter))
-                    }
-                    SinkEncode::Protobuf => {
-                        // By passing `None` as `aws_auth_props`, reading from `s3://` not supported yet.
-                        let (descriptor, sid) = crate::schema::protobuf::fetch_descriptor(
-                            &format_desc.options,
-                            topic,
-                            None,
-                        )
-                        .await
-                        .map_err(|e| SinkError::Config(anyhow!(e)))?;
-                        let header = match sid {
-                            None => ProtoHeader::None,
-                            Some(sid) => ProtoHeader::ConfluentSchemaRegistry(sid),
-                        };
-                        let val_encoder = ProtoEncoder::new(schema, None, descriptor, header)?;
-                        let formatter = AppendOnlyFormatter::new(key_encoder, val_encoder);
-                        Ok(SinkFormatterImpl::AppendOnlyProto(formatter))
-                    }
-                    SinkEncode::Avro => err_unsupported(),
-                    SinkEncode::Template => {
-                        let key_format = format_desc.options.get(KEY_FORMAT).ok_or_else(|| {
-                            SinkError::Config(anyhow!(
-                                "Cannot find 'key_format',please set it or use JSON"
-                            ))
-                        })?;
-                        let value_format =
-                            format_desc.options.get(VALUE_FORMAT).ok_or_else(|| {
-                                SinkError::Config(anyhow!(
-                                    "Cannot find 'redis_value_format',please set it or use JSON"
-                                ))
-                            })?;
-                        let key_encoder = TemplateEncoder::new(
-                            schema.clone(),
-                            Some(pk_indices),
-                            key_format.clone(),
-                        );
-                        let val_encoder = TemplateEncoder::new(schema, None, value_format.clone());
-                        Ok(SinkFormatterImpl::AppendOnlyTemplate(
-                            AppendOnlyFormatter::new(Some(key_encoder), val_encoder),
-                        ))
-                    }
+
+        // When defining `SinkFormatterImpl` we already linked each variant (eg `AppendOnlyJson`) to
+        // an instantiation (eg `AppendOnlyFormatter<JsonEncoder, JsonEncoder>`) that implements the
+        // trait `FormatterBuild`.
+        //
+        // Here we just need to match a `(format, encode)` to a variant, and rustc shall be able to
+        // find the corresponding instantiation.
+
+        // However,
+        //   `Impl::AppendOnlyJson(FormatterBuild::build(p).await?)`
+        // fails to be inferred without the following dummy wrapper.
+        async fn build<T: FormatterBuild>(p: FormatterParams<'_>) -> Result<T> {
+            T::build(p).await
+        }
+
+        Ok(
+            match (
+                &format_desc.format,
+                &format_desc.encode,
+                &format_desc.key_encode,
+            ) {
+                (F::AppendOnly, E::Json, Some(E::Text)) => {
+                    Impl::AppendOnlyTextJson(build(p).await?)
                 }
-            }
-            SinkFormat::Debezium => {
-                if format_desc.encode != SinkEncode::Json {
-                    return err_unsupported();
+                (F::AppendOnly, E::Json, None) => Impl::AppendOnlyJson(build(p).await?),
+                (F::AppendOnly, E::Avro, Some(E::Text)) => {
+                    Impl::AppendOnlyTextAvro(build(p).await?)
                 }
-
-                Ok(SinkFormatterImpl::DebeziumJson(DebeziumJsonFormatter::new(
-                    schema,
-                    pk_indices,
-                    db_name,
-                    sink_from_name,
-                    DebeziumAdapterOpts::default(),
-                )))
-            }
-            SinkFormat::Upsert => {
-                match format_desc.encode {
-                    SinkEncode::Json => {
-                        let mut key_encoder = JsonEncoder::new(
-                            schema.clone(),
-                            Some(pk_indices),
-                            DateHandlingMode::FromCe,
-                            TimestampHandlingMode::Milli,
-                            timestamptz_mode,
-                            TimeHandlingMode::Milli,
-                        );
-                        let mut val_encoder = JsonEncoder::new(
-                            schema,
-                            None,
-                            DateHandlingMode::FromCe,
-                            TimestampHandlingMode::Milli,
-                            timestamptz_mode,
-                            TimeHandlingMode::Milli,
-                        );
-
-                        if let Some(s) = format_desc.options.get("schemas.enable") {
-                            match s.to_lowercase().parse::<bool>() {
-                                Ok(true) => {
-                                    let kafka_connect = KafkaConnectParams {
-                                        schema_name: format!("{}.{}", db_name, sink_from_name),
-                                    };
-                                    key_encoder =
-                                        key_encoder.with_kafka_connect(kafka_connect.clone());
-                                    val_encoder = val_encoder.with_kafka_connect(kafka_connect);
-                                }
-                                Ok(false) => {}
-                                _ => {
-                                    return Err(SinkError::Config(anyhow!(
-                                        "schemas.enable is expected to be `true` or `false`, got {}",
-                                        s
-                                    )));
-                                }
-                            }
-                        };
-
-                        // Initialize the upsert_stream
-                        let formatter = UpsertFormatter::new(key_encoder, val_encoder);
-                        Ok(SinkFormatterImpl::UpsertJson(formatter))
-                    }
-                    SinkEncode::Template => {
-                        let key_format = format_desc.options.get(KEY_FORMAT).ok_or_else(|| {
-                            SinkError::Config(anyhow!(
-                                "Cannot find 'key_format',please set it or use JSON"
-                            ))
-                        })?;
-                        let value_format =
-                            format_desc.options.get(VALUE_FORMAT).ok_or_else(|| {
-                                SinkError::Config(anyhow!(
-                                    "Cannot find 'redis_value_format',please set it or use JSON"
-                                ))
-                            })?;
-                        let key_encoder = TemplateEncoder::new(
-                            schema.clone(),
-                            Some(pk_indices),
-                            key_format.clone(),
-                        );
-                        let val_encoder = TemplateEncoder::new(schema, None, value_format.clone());
-                        Ok(SinkFormatterImpl::UpsertTemplate(UpsertFormatter::new(
-                            key_encoder,
-                            val_encoder,
-                        )))
-                    }
-                    SinkEncode::Avro => {
-                        let (key_schema, val_schema) =
-                            crate::schema::avro::fetch_schema(&format_desc.options, topic)
-                                .await
-                                .map_err(|e| SinkError::Config(anyhow!(e)))?;
-                        let key_encoder = AvroEncoder::new(
-                            schema.clone(),
-                            Some(pk_indices),
-                            key_schema.schema,
-                            AvroHeader::ConfluentSchemaRegistry(key_schema.id),
-                        )?;
-                        let val_encoder = AvroEncoder::new(
-                            schema.clone(),
-                            None,
-                            val_schema.schema,
-                            AvroHeader::ConfluentSchemaRegistry(val_schema.id),
-                        )?;
-                        let formatter = UpsertFormatter::new(key_encoder, val_encoder);
-                        Ok(SinkFormatterImpl::UpsertAvro(formatter))
-                    }
-                    SinkEncode::Protobuf => err_unsupported(),
+                (F::AppendOnly, E::Avro, None) => Impl::AppendOnlyAvro(build(p).await?),
+                (F::AppendOnly, E::Protobuf, Some(E::Text)) => {
+                    Impl::AppendOnlyTextProto(build(p).await?)
                 }
-            }
-        }
+                (F::AppendOnly, E::Protobuf, None) => Impl::AppendOnlyProto(build(p).await?),
+                (F::AppendOnly, E::Template, Some(E::Text)) => {
+                    Impl::AppendOnlyTextTemplate(build(p).await?)
+                }
+                (F::AppendOnly, E::Template, None) => Impl::AppendOnlyTemplate(build(p).await?),
+                (F::Upsert, E::Json, Some(E::Text)) => Impl::UpsertTextJson(build(p).await?),
+                (F::Upsert, E::Json, None) => Impl::UpsertJson(build(p).await?),
+                (F::Upsert, E::Avro, Some(E::Text)) => Impl::UpsertTextAvro(build(p).await?),
+                (F::Upsert, E::Avro, None) => Impl::UpsertAvro(build(p).await?),
+                (F::Upsert, E::Template, Some(E::Text)) => {
+                    Impl::UpsertTextTemplate(build(p).await?)
+                }
+                (F::Upsert, E::Template, None) => Impl::UpsertTemplate(build(p).await?),
+                (F::Debezium, E::Json, None) => Impl::DebeziumJson(build(p).await?),
+                (F::AppendOnly | F::Upsert, E::Text, _) => {
+                    return Err(SinkError::Config(anyhow!(
+                        "ENCODE TEXT is only valid as key encode."
+                    )));
+                }
+                (F::AppendOnly, E::Avro, _)
+                | (F::Upsert, E::Protobuf, _)
+                | (F::Debezium, E::Json, Some(_))
+                | (F::Debezium, E::Avro | E::Protobuf | E::Template | E::Text, _)
+                | (F::AppendOnly | F::Upsert, _, Some(E::Template) | Some(E::Json) | Some(E::Avro) | Some(E::Protobuf)) // reject other encode as key encode
+                => {
+                    return Err(SinkError::Config(anyhow!(
+                        "sink format/encode/key_encode unsupported: {:?} {:?} {:?}",
+                        format_desc.format,
+                        format_desc.encode,
+                        format_desc.key_encode
+                    )));
+                }
+            },
+        )
     }
 }
 
@@ -275,11 +386,20 @@ macro_rules! dispatch_sink_formatter_impl {
     ($impl:expr, $name:ident, $body:expr) => {
         match $impl {
             SinkFormatterImpl::AppendOnlyJson($name) => $body,
+            SinkFormatterImpl::AppendOnlyTextJson($name) => $body,
+            SinkFormatterImpl::AppendOnlyAvro($name) => $body,
+            SinkFormatterImpl::AppendOnlyTextAvro($name) => $body,
             SinkFormatterImpl::AppendOnlyProto($name) => $body,
+            SinkFormatterImpl::AppendOnlyTextProto($name) => $body,
+
             SinkFormatterImpl::UpsertJson($name) => $body,
+            SinkFormatterImpl::UpsertTextJson($name) => $body,
             SinkFormatterImpl::UpsertAvro($name) => $body,
+            SinkFormatterImpl::UpsertTextAvro($name) => $body,
             SinkFormatterImpl::DebeziumJson($name) => $body,
+            SinkFormatterImpl::AppendOnlyTextTemplate($name) => $body,
             SinkFormatterImpl::AppendOnlyTemplate($name) => $body,
+            SinkFormatterImpl::UpsertTextTemplate($name) => $body,
             SinkFormatterImpl::UpsertTemplate($name) => $body,
         }
     };
@@ -290,11 +410,20 @@ macro_rules! dispatch_sink_formatter_str_key_impl {
     ($impl:expr, $name:ident, $body:expr) => {
         match $impl {
             SinkFormatterImpl::AppendOnlyJson($name) => $body,
+            SinkFormatterImpl::AppendOnlyTextJson($name) => $body,
+            SinkFormatterImpl::AppendOnlyAvro(_) => unreachable!(),
+            SinkFormatterImpl::AppendOnlyTextAvro($name) => $body,
             SinkFormatterImpl::AppendOnlyProto($name) => $body,
+            SinkFormatterImpl::AppendOnlyTextProto($name) => $body,
+
             SinkFormatterImpl::UpsertJson($name) => $body,
+            SinkFormatterImpl::UpsertTextJson($name) => $body,
             SinkFormatterImpl::UpsertAvro(_) => unreachable!(),
+            SinkFormatterImpl::UpsertTextAvro($name) => $body,
             SinkFormatterImpl::DebeziumJson($name) => $body,
+            SinkFormatterImpl::AppendOnlyTextTemplate($name) => $body,
             SinkFormatterImpl::AppendOnlyTemplate($name) => $body,
+            SinkFormatterImpl::UpsertTextTemplate($name) => $body,
             SinkFormatterImpl::UpsertTemplate($name) => $body,
         }
     };
diff --git a/src/connector/src/sink/google_pubsub.rs b/src/connector/src/sink/google_pubsub.rs
new file mode 100644
index 0000000000000..ad039ad020704
--- /dev/null
+++ b/src/connector/src/sink/google_pubsub.rs
@@ -0,0 +1,288 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::BTreeMap;
+
+use anyhow::{anyhow, Context};
+use google_cloud_gax::conn::Environment;
+use google_cloud_googleapis::pubsub::v1::PubsubMessage;
+use google_cloud_pubsub::apiv1;
+use google_cloud_pubsub::client::google_cloud_auth::credentials::CredentialsFile;
+use google_cloud_pubsub::client::google_cloud_auth::project;
+use google_cloud_pubsub::client::google_cloud_auth::token::DefaultTokenSourceProvider;
+use google_cloud_pubsub::client::{Client, ClientConfig};
+use google_cloud_pubsub::publisher::Publisher;
+use risingwave_common::array::StreamChunk;
+use risingwave_common::catalog::Schema;
+use risingwave_common::session_config::sink_decouple::SinkDecouple;
+use serde_derive::Deserialize;
+use serde_with::serde_as;
+use tonic::Status;
+use with_options::WithOptions;
+
+use super::catalog::desc::SinkDesc;
+use super::catalog::SinkFormatDesc;
+use super::formatter::SinkFormatterImpl;
+use super::log_store::DeliveryFutureManagerAddFuture;
+use super::writer::{
+    AsyncTruncateLogSinkerOf, AsyncTruncateSinkWriter, AsyncTruncateSinkWriterExt, FormattedSink,
+};
+use super::{DummySinkCommitCoordinator, Result, Sink, SinkError, SinkParam, SinkWriterParam};
+use crate::dispatch_sink_formatter_str_key_impl;
+
+pub const PUBSUB_SINK: &str = "google_pubsub";
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, WithOptions)]
+pub struct GooglePubSubConfig {
+    /// The Google Pub/Sub Project ID
+    #[serde(rename = "pubsub.project_id")]
+    pub project_id: String,
+
+    /// Specifies the Pub/Sub topic to publish messages
+    #[serde(rename = "pubsub.topic")]
+    pub topic: String,
+
+    /// The Google Pub/Sub endpoint URL
+    #[serde(rename = "pubsub.endpoint")]
+    pub endpoint: String,
+
+    /// use the connector with a pubsub emulator
+    /// <https://cloud.google.com/pubsub/docs/emulator>
+    #[serde(rename = "pubsub.emulator_host")]
+    pub emulator_host: Option<String>,
+
+    /// A JSON string containing the service account credentials for authorization,
+    /// see the [service-account](https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account) credentials guide.
+    /// The provided account credential must have the
+    /// `pubsub.publisher` [role](https://cloud.google.com/pubsub/docs/access-control#roles)
+    #[serde(rename = "pubsub.credentials")]
+    pub credentials: Option<String>,
+}
+
+impl GooglePubSubConfig {
+    fn from_btreemap(values: BTreeMap<String, String>) -> Result<Self> {
+        serde_json::from_value::<GooglePubSubConfig>(serde_json::to_value(values).unwrap())
+            .map_err(|e| SinkError::Config(anyhow!(e)))
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct GooglePubSubSink {
+    pub config: GooglePubSubConfig,
+    is_append_only: bool,
+
+    schema: Schema,
+    pk_indices: Vec<usize>,
+    format_desc: SinkFormatDesc,
+    db_name: String,
+    sink_from_name: String,
+}
+
+impl Sink for GooglePubSubSink {
+    type Coordinator = DummySinkCommitCoordinator;
+    type LogSinker = AsyncTruncateLogSinkerOf<GooglePubSubSinkWriter>;
+
+    const SINK_NAME: &'static str = PUBSUB_SINK;
+
+    fn is_sink_decouple(_desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
+        match user_specified {
+            SinkDecouple::Default | SinkDecouple::Enable => Ok(true),
+            SinkDecouple::Disable => Ok(false),
+        }
+    }
+
+    async fn validate(&self) -> Result<()> {
+        if !self.is_append_only {
+            return Err(SinkError::GooglePubSub(anyhow!(
+                "Google Pub/Sub sink only support append-only mode"
+            )));
+        }
+
+        let conf = &self.config;
+        if matches!((&conf.emulator_host, &conf.credentials), (None, None)) {
+            return Err(SinkError::GooglePubSub(anyhow!(
+                "Configure at least one of `pubsub.emulator_host` and `pubsub.credentials` in the Google Pub/Sub sink"
+            )));
+        }
+
+        Ok(())
+    }
+
+    async fn new_log_sinker(&self, _writer_param: SinkWriterParam) -> Result<Self::LogSinker> {
+        Ok(GooglePubSubSinkWriter::new(
+            self.config.clone(),
+            self.schema.clone(),
+            self.pk_indices.clone(),
+            &self.format_desc,
+            self.db_name.clone(),
+            self.sink_from_name.clone(),
+        )
+        .await?
+        .into_log_sinker(usize::MAX))
+    }
+}
+
+impl TryFrom<SinkParam> for GooglePubSubSink {
+    type Error = SinkError;
+
+    fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
+        let schema = param.schema();
+        let config = GooglePubSubConfig::from_btreemap(param.properties)?;
+
+        let format_desc = param
+            .format_desc
+            .ok_or_else(|| SinkError::Config(anyhow!("missing FORMAT ... ENCODE ...")))?;
+        Ok(Self {
+            config,
+            is_append_only: param.sink_type.is_append_only(),
+
+            schema,
+            pk_indices: param.downstream_pk,
+            format_desc,
+            db_name: param.db_name,
+            sink_from_name: param.sink_from_name,
+        })
+    }
+}
+
+struct GooglePubSubPayloadWriter {
+    publisher: Publisher,
+}
+
+impl GooglePubSubSinkWriter {
+    pub async fn new(
+        config: GooglePubSubConfig,
+        schema: Schema,
+        pk_indices: Vec<usize>,
+        format_desc: &SinkFormatDesc,
+        db_name: String,
+        sink_from_name: String,
+    ) -> Result<Self> {
+        let environment = if let Some(ref cred) = config.credentials {
+            let auth_config = project::Config {
+                audience: Some(apiv1::conn_pool::AUDIENCE),
+                scopes: Some(&apiv1::conn_pool::SCOPES),
+                sub: None,
+            };
+            let cred_file = CredentialsFile::new_from_str(cred).await.map_err(|e| {
+                SinkError::GooglePubSub(
+                    anyhow!(e).context("Failed to create Google Cloud Pub/Sub credentials file"),
+                )
+            })?;
+            let provider =
+                DefaultTokenSourceProvider::new_with_credentials(auth_config, Box::new(cred_file))
+                    .await
+                    .map_err(|e| {
+                        SinkError::GooglePubSub(
+                            anyhow!(e).context(
+                                "Failed to create Google Cloud Pub/Sub token source provider",
+                            ),
+                        )
+                    })?;
+            Environment::GoogleCloud(Box::new(provider))
+        } else if let Some(emu_host) = config.emulator_host {
+            Environment::Emulator(emu_host)
+        } else {
+            return Err(SinkError::GooglePubSub(anyhow!(
+                "Missing emulator_host or credentials in Google Pub/Sub sink"
+            )));
+        };
+
+        let client_config = ClientConfig {
+            endpoint: config.endpoint,
+            project_id: Some(config.project_id),
+            environment,
+            ..Default::default()
+        };
+        let client = Client::new(client_config)
+            .await
+            .map_err(|e| SinkError::GooglePubSub(anyhow!(e)))?;
+
+        let topic = async {
+            let topic = client.topic(&config.topic);
+            if !topic.exists(None).await? {
+                topic.create(None, None).await?;
+            }
+            Ok(topic)
+        }
+        .await
+        .map_err(|e: Status| SinkError::GooglePubSub(anyhow!(e)))?;
+
+        let formatter = SinkFormatterImpl::new(
+            format_desc,
+            schema,
+            pk_indices,
+            db_name,
+            sink_from_name,
+            topic.fully_qualified_name(),
+        )
+        .await?;
+
+        let publisher = topic.new_publisher(None);
+        let payload_writer = GooglePubSubPayloadWriter { publisher };
+
+        Ok(Self {
+            payload_writer,
+            formatter,
+        })
+    }
+}
+
+pub struct GooglePubSubSinkWriter {
+    payload_writer: GooglePubSubPayloadWriter,
+    formatter: SinkFormatterImpl,
+}
+
+impl AsyncTruncateSinkWriter for GooglePubSubSinkWriter {
+    async fn write_chunk<'a>(
+        &'a mut self,
+        chunk: StreamChunk,
+        _add_future: DeliveryFutureManagerAddFuture<'a, Self::DeliveryFuture>,
+    ) -> Result<()> {
+        dispatch_sink_formatter_str_key_impl!(
+            &self.formatter,
+            formatter,
+            self.payload_writer.write_chunk(chunk, formatter).await
+        )
+    }
+}
+
+impl FormattedSink for GooglePubSubPayloadWriter {
+    type K = String;
+    type V = Vec<u8>;
+
+    async fn write_one(&mut self, k: Option<Self::K>, v: Option<Self::V>) -> Result<()> {
+        let ordering_key = k.unwrap_or_default();
+        match v {
+            Some(data) => {
+                let msg = PubsubMessage {
+                    data,
+                    ordering_key,
+                    ..Default::default()
+                };
+                let awaiter = self.publisher.publish(msg).await;
+                awaiter
+                    .get()
+                    .await
+                    .context("Google Pub/Sub sink error")
+                    .map_err(SinkError::GooglePubSub)
+                    .map(|_| ())
+            }
+            None => Err(SinkError::GooglePubSub(anyhow!(
+                "Google Pub/Sub sink error: missing value to publish"
+            ))),
+        }
+    }
+}
diff --git a/src/connector/src/sink/iceberg/mod.rs b/src/connector/src/sink/iceberg/mod.rs
index 22972d5629076..7d40570fc8fc2 100644
--- a/src/connector/src/sink/iceberg/mod.rs
+++ b/src/connector/src/sink/iceberg/mod.rs
@@ -13,11 +13,10 @@
 // limitations under the License.
 
 mod jni_catalog;
-mod log_sink;
 mod mock_catalog;
 mod prometheus;
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::fmt::Debug;
 use std::num::NonZeroU64;
 use std::ops::Deref;
@@ -41,9 +40,8 @@ use icelake::transaction::Transaction;
 use icelake::types::{data_file_from_json, data_file_to_json, Any, DataFile};
 use icelake::{Table, TableIdentifier};
 use itertools::Itertools;
-use risingwave_common::array::{
-    iceberg_to_arrow_type, to_iceberg_record_batch_with_schema, Op, StreamChunk,
-};
+use risingwave_common::array::arrow::{IcebergArrowConvert, ToArrow};
+use risingwave_common::array::{Op, StreamChunk};
 use risingwave_common::bail;
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::Schema;
@@ -55,11 +53,11 @@ use thiserror_ext::AsReport;
 use url::Url;
 use with_options::WithOptions;
 
-use self::log_sink::IcebergLogSinkerOf;
 use self::mock_catalog::MockCatalog;
 use self::prometheus::monitored_base_file_writer::MonitoredBaseFileWriterBuilder;
 use self::prometheus::monitored_position_delete_writer::MonitoredPositionDeleteWriterBuilder;
 use super::catalog::desc::SinkDesc;
+use super::decouple_checkpoint_log_sink::DecoupleCheckpointLogSinkerOf;
 use super::{
     Sink, SinkError, SinkWriterParam, SINK_TYPE_APPEND_ONLY, SINK_TYPE_OPTION, SINK_TYPE_UPSERT,
 };
@@ -136,7 +134,7 @@ pub struct IcebergConfig {
 }
 
 impl IcebergConfig {
-    pub fn from_hashmap(values: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(values: BTreeMap<String, String>) -> Result<Self> {
         let mut config =
             serde_json::from_value::<IcebergConfig>(serde_json::to_value(&values).unwrap())
                 .map_err(|e| SinkError::Config(anyhow!(e)))?;
@@ -445,7 +443,7 @@ impl TryFrom<SinkParam> for IcebergSink {
     type Error = SinkError;
 
     fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
-        let config = IcebergConfig::from_hashmap(param.properties.clone())?;
+        let config = IcebergConfig::from_btreemap(param.properties.clone())?;
         IcebergSink::new(config, param)
     }
 }
@@ -475,7 +473,7 @@ impl IcebergSink {
             .try_into()
             .map_err(|err: icelake::Error| SinkError::Iceberg(anyhow!(err)))?;
 
-        try_matches_arrow_schema(&sink_schema, &iceberg_schema, false)
+        try_matches_arrow_schema(&sink_schema, &iceberg_schema)
             .map_err(|err| SinkError::Iceberg(anyhow!(err)))?;
 
         Ok(table)
@@ -517,7 +515,7 @@ impl IcebergSink {
 
 impl Sink for IcebergSink {
     type Coordinator = IcebergSinkCommitter;
-    type LogSinker = IcebergLogSinkerOf<CoordinatedSinkWriter<IcebergWriter>>;
+    type LogSinker = DecoupleCheckpointLogSinkerOf<CoordinatedSinkWriter<IcebergWriter>>;
 
     const SINK_NAME: &'static str = ICEBERG_SINK;
 
@@ -578,7 +576,7 @@ impl Sink for IcebergSink {
                 "commit_checkpoint_interval should be greater than 0, and it should be checked in config validation",
             );
 
-        Ok(IcebergLogSinkerOf::new(
+        Ok(DecoupleCheckpointLogSinkerOf::new(
             writer,
             writer_param.sink_metrics,
             commit_checkpoint_interval,
@@ -797,14 +795,15 @@ impl SinkWriter for IcebergWriter {
                 let filters =
                     chunk.visibility() & ops.iter().map(|op| *op == Op::Insert).collect::<Bitmap>();
                 chunk.set_visibility(filters);
-                let chunk =
-                    to_iceberg_record_batch_with_schema(self.schema.clone(), &chunk.compact())
-                        .map_err(|err| SinkError::Iceberg(anyhow!(err)))?;
+                let chunk = IcebergArrowConvert
+                    .to_record_batch(self.schema.clone(), &chunk.compact())
+                    .map_err(|err| SinkError::Iceberg(anyhow!(err)))?;
 
                 writer.write(chunk).await?;
             }
             IcebergWriterEnum::Upsert(writer) => {
-                let chunk = to_iceberg_record_batch_with_schema(self.schema.clone(), &chunk)
+                let chunk = IcebergArrowConvert
+                    .to_record_batch(self.schema.clone(), &chunk)
                     .map_err(|err| SinkError::Iceberg(anyhow!(err)))?;
 
                 writer
@@ -1002,11 +1001,9 @@ impl SinkCommitCoordinator for IcebergSinkCommitter {
 }
 
 /// Try to match our schema with iceberg schema.
-/// `for_source` = true means the schema is used for source, otherwise it's used for sink.
 pub fn try_matches_arrow_schema(
     rw_schema: &Schema,
     arrow_schema: &ArrowSchema,
-    for_source: bool,
 ) -> anyhow::Result<()> {
     if rw_schema.fields.len() != arrow_schema.fields().len() {
         bail!(
@@ -1029,17 +1026,11 @@ pub fn try_matches_arrow_schema(
             .ok_or_else(|| anyhow!("Field {} not found in our schema", arrow_field.name()))?;
 
         // Iceberg source should be able to read iceberg decimal type.
-        // Since the arrow type default conversion is used by udf, in udf, decimal is converted to
-        // large binary type which is not compatible with iceberg decimal type,
-        // so we need to convert it to decimal type manually.
-        let converted_arrow_data_type = if for_source
-            && matches!(our_field_type, risingwave_common::types::DataType::Decimal)
-        {
-            // RisingWave decimal type cannot specify precision and scale, so we use the default value.
-            ArrowDataType::Decimal128(38, 0)
-        } else {
-            iceberg_to_arrow_type(our_field_type).map_err(|e| anyhow!(e))?
-        };
+        let converted_arrow_data_type = IcebergArrowConvert
+            .to_arrow_field("", our_field_type)
+            .map_err(|e| anyhow!(e))?
+            .data_type()
+            .clone();
 
         let compatible = match (&converted_arrow_data_type, arrow_field.data_type()) {
             (ArrowDataType::Decimal128(_, _), ArrowDataType::Decimal128(_, _)) => true,
@@ -1057,7 +1048,7 @@ pub fn try_matches_arrow_schema(
 
 #[cfg(test)]
 mod test {
-    use std::collections::HashMap;
+    use std::collections::BTreeMap;
 
     use risingwave_common::catalog::Field;
 
@@ -1080,7 +1071,7 @@ mod test {
             ArrowField::new("c", ArrowDataType::Int32, false),
         ]);
 
-        try_matches_arrow_schema(&risingwave_schema, &arrow_schema, false).unwrap();
+        try_matches_arrow_schema(&risingwave_schema, &arrow_schema).unwrap();
 
         let risingwave_schema = Schema::new(vec![
             Field::with_name(DataType::Int32, "d"),
@@ -1094,7 +1085,7 @@ mod test {
             ArrowField::new("d", ArrowDataType::Int32, false),
             ArrowField::new("c", ArrowDataType::Int32, false),
         ]);
-        try_matches_arrow_schema(&risingwave_schema, &arrow_schema, false).unwrap();
+        try_matches_arrow_schema(&risingwave_schema, &arrow_schema).unwrap();
     }
 
     #[test]
@@ -1120,7 +1111,7 @@ mod test {
         .map(|(k, v)| (k.to_string(), v.to_string()))
         .collect();
 
-        let iceberg_config = IcebergConfig::from_hashmap(values).unwrap();
+        let iceberg_config = IcebergConfig::from_btreemap(values).unwrap();
 
         let expected_iceberg_config = IcebergConfig {
             connector: "iceberg".to_string(),
@@ -1152,8 +1143,8 @@ mod test {
         );
     }
 
-    async fn test_create_catalog(configs: HashMap<String, String>) {
-        let iceberg_config = IcebergConfig::from_hashmap(configs).unwrap();
+    async fn test_create_catalog(configs: BTreeMap<String, String>) {
+        let iceberg_config = IcebergConfig::from_btreemap(configs).unwrap();
 
         let table = iceberg_config.load_table().await.unwrap();
 
diff --git a/src/connector/src/sink/iceberg/prometheus/monitored_partition_writer.rs b/src/connector/src/sink/iceberg/prometheus/monitored_partition_writer.rs
index c0bb5e097323c..c2134d1974551 100644
--- a/src/connector/src/sink/iceberg/prometheus/monitored_partition_writer.rs
+++ b/src/connector/src/sink/iceberg/prometheus/monitored_partition_writer.rs
@@ -27,6 +27,7 @@ pub struct MonitoredFanoutPartitionedWriterBuilder<B: IcebergWriterBuilder> {
 }
 
 impl<B: IcebergWriterBuilder> MonitoredFanoutPartitionedWriterBuilder<B> {
+    #[expect(dead_code)]
     pub fn new(
         inner: FanoutPartitionedWriterBuilder<B>,
         partition_num: LabelGuardedIntGauge<2>,
diff --git a/src/connector/src/sink/iceberg/prometheus/monitored_write_writer.rs b/src/connector/src/sink/iceberg/prometheus/monitored_write_writer.rs
index 0f2877490fb23..be9f35aae2a51 100644
--- a/src/connector/src/sink/iceberg/prometheus/monitored_write_writer.rs
+++ b/src/connector/src/sink/iceberg/prometheus/monitored_write_writer.rs
@@ -28,6 +28,7 @@ pub struct MonitoredWriteWriterBuilder<B: IcebergWriterBuilder> {
 
 impl<B: IcebergWriterBuilder> MonitoredWriteWriterBuilder<B> {
     /// Create writer context.
+    #[expect(dead_code)]
     pub fn new(
         inner: B,
         write_qps: LabelGuardedIntCounter<2>,
diff --git a/src/connector/src/sink/kafka.rs b/src/connector/src/sink/kafka.rs
index 8d56441e53488..617f427ae71f1 100644
--- a/src/connector/src/sink/kafka.rs
+++ b/src/connector/src/sink/kafka.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::fmt::Debug;
 use std::sync::Arc;
 use std::time::Duration;
@@ -35,7 +35,9 @@ use with_options::WithOptions;
 
 use super::catalog::{SinkFormat, SinkFormatDesc};
 use super::{Sink, SinkError, SinkParam};
-use crate::connector_common::{KafkaCommon, KafkaPrivateLinkCommon, RdKafkaPropertiesCommon};
+use crate::connector_common::{
+    AwsAuthProps, KafkaCommon, KafkaPrivateLinkCommon, RdKafkaPropertiesCommon,
+};
 use crate::sink::catalog::desc::SinkDesc;
 use crate::sink::formatter::SinkFormatterImpl;
 use crate::sink::log_store::DeliveryFutureManagerAddFuture;
@@ -43,7 +45,9 @@ use crate::sink::writer::{
     AsyncTruncateLogSinkerOf, AsyncTruncateSinkWriter, AsyncTruncateSinkWriterExt, FormattedSink,
 };
 use crate::sink::{DummySinkCommitCoordinator, Result, SinkWriterParam};
-use crate::source::kafka::{KafkaProperties, KafkaSplitEnumerator, PrivateLinkProducerContext};
+use crate::source::kafka::{
+    KafkaContextCommon, KafkaProperties, KafkaSplitEnumerator, RwProducerContext,
+};
 use crate::source::{SourceEnumeratorContext, SplitEnumerator};
 use crate::{
     deserialize_duration_from_string, deserialize_u32_from_string, dispatch_sink_formatter_impl,
@@ -162,6 +166,10 @@ pub struct RdKafkaPropertiesProducer {
     )]
     #[serde_as(as = "DisplayFromStr")]
     max_in_flight_requests_per_connection: usize,
+
+    #[serde(rename = "properties.request.required.acks")]
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    request_required_acks: Option<i32>,
 }
 
 impl RdKafkaPropertiesProducer {
@@ -196,6 +204,9 @@ impl RdKafkaPropertiesProducer {
         if let Some(v) = &self.compression_codec {
             c.set("compression.codec", v.to_string());
         }
+        if let Some(v) = self.request_required_acks {
+            c.set("request.required.acks", v.to_string());
+        }
         c.set("message.timeout.ms", self.message_timeout_ms.to_string());
         c.set(
             "max.in.flight.requests.per.connection",
@@ -237,10 +248,13 @@ pub struct KafkaConfig {
 
     #[serde(flatten)]
     pub privatelink_common: KafkaPrivateLinkCommon,
+
+    #[serde(flatten)]
+    pub aws_auth_props: AwsAuthProps,
 }
 
 impl KafkaConfig {
-    pub fn from_hashmap(values: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(values: BTreeMap<String, String>) -> Result<Self> {
         let config = serde_json::from_value::<KafkaConfig>(serde_json::to_value(values).unwrap())
             .map_err(|e| SinkError::Config(anyhow!(e)))?;
 
@@ -250,8 +264,6 @@ impl KafkaConfig {
     pub(crate) fn set_client(&self, c: &mut rdkafka::ClientConfig) {
         self.rdkafka_properties_common.set_client(c);
         self.rdkafka_properties_producer.set_client(c);
-
-        tracing::info!("kafka client starts with: {:?}", c);
     }
 }
 
@@ -267,6 +279,7 @@ impl From<KafkaConfig> for KafkaProperties {
             rdkafka_properties_common: val.rdkafka_properties_common,
             rdkafka_properties_consumer: Default::default(),
             privatelink_common: val.privatelink_common,
+            aws_auth_props: val.aws_auth_props,
             unknown_fields: Default::default(),
         }
     }
@@ -287,7 +300,7 @@ impl TryFrom<SinkParam> for KafkaSink {
 
     fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
         let schema = param.schema();
-        let config = KafkaConfig::from_hashmap(param.properties)?;
+        let config = KafkaConfig::from_btreemap(param.properties)?;
         Ok(Self {
             config,
             schema,
@@ -307,11 +320,10 @@ impl Sink for KafkaSink {
 
     const SINK_NAME: &'static str = KAFKA_SINK;
 
-    fn is_sink_decouple(desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
+    fn is_sink_decouple(_desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
         match user_specified {
-            SinkDecouple::Default => Ok(desc.sink_type.is_append_only()),
+            SinkDecouple::Default | SinkDecouple::Enable => Ok(true),
             SinkDecouple::Disable => Ok(false),
-            SinkDecouple::Enable => Ok(true),
         }
     }
 
@@ -386,7 +398,7 @@ const KAFKA_WRITER_MAX_QUEUE_SIZE_RATIO: f32 = 1.2;
 const KAFKA_WRITER_MAX_QUEUE_SIZE: usize = 100000;
 
 struct KafkaPayloadWriter<'a> {
-    inner: &'a FutureProducer<PrivateLinkProducerContext>,
+    inner: &'a FutureProducer<RwProducerContext>,
     add_future: DeliveryFutureManagerAddFuture<'a, KafkaSinkDeliveryFuture>,
     config: &'a KafkaConfig,
 }
@@ -395,13 +407,13 @@ pub type KafkaSinkDeliveryFuture = impl TryFuture<Ok = (), Error = SinkError> +
 
 pub struct KafkaSinkWriter {
     formatter: SinkFormatterImpl,
-    inner: FutureProducer<PrivateLinkProducerContext>,
+    inner: FutureProducer<RwProducerContext>,
     config: KafkaConfig,
 }
 
 impl KafkaSinkWriter {
     async fn new(config: KafkaConfig, formatter: SinkFormatterImpl) -> Result<Self> {
-        let inner: FutureProducer<PrivateLinkProducerContext> = {
+        let inner: FutureProducer<RwProducerContext> = {
             let mut c = ClientConfig::new();
 
             // KafkaConfig configuration
@@ -412,13 +424,16 @@ impl KafkaSinkWriter {
             c.set("bootstrap.servers", &config.common.brokers);
 
             // Create the producer context, will be used to create the producer
-            let producer_ctx = PrivateLinkProducerContext::new(
-                config.privatelink_common.broker_rewrite_map.clone(),
-                // fixme: enable kafka native metrics for sink
+            let broker_rewrite_map = config.privatelink_common.broker_rewrite_map.clone();
+            let ctx_common = KafkaContextCommon::new(
+                broker_rewrite_map,
                 None,
                 None,
-            )?;
-
+                config.aws_auth_props.clone(),
+                config.common.is_aws_msk_iam(),
+            )
+            .await?;
+            let producer_ctx = RwProducerContext::new(ctx_common);
             // Generate the producer
             c.create_with_context(producer_ctx).await?
         };
@@ -569,7 +584,7 @@ impl<'a> FormattedSink for KafkaPayloadWriter<'a> {
 
 #[cfg(test)]
 mod test {
-    use maplit::hashmap;
+    use maplit::btreemap;
     use risingwave_common::catalog::Field;
     use risingwave_common::types::DataType;
 
@@ -582,7 +597,7 @@ mod test {
 
     #[test]
     fn parse_rdkafka_props() {
-        let props: HashMap<String, String> = hashmap! {
+        let props: BTreeMap<String, String> = btreemap! {
             // basic
             // "connector".to_string() => "kafka".to_string(),
             "properties.bootstrap.server".to_string() => "localhost:9092".to_string(),
@@ -603,8 +618,9 @@ mod test {
             "properties.compression.codec".to_string() => "zstd".to_string(),
             "properties.message.timeout.ms".to_string() => "114514".to_string(),
             "properties.max.in.flight.requests.per.connection".to_string() => "114514".to_string(),
+            "properties.request.required.acks".to_string() => "-1".to_string(),
         };
-        let c = KafkaConfig::from_hashmap(props).unwrap();
+        let c = KafkaConfig::from_btreemap(props).unwrap();
         assert_eq!(
             c.rdkafka_properties_producer.queue_buffering_max_ms,
             Some(114.514f64)
@@ -619,8 +635,12 @@ mod test {
                 .max_in_flight_requests_per_connection,
             114514
         );
+        assert_eq!(
+            c.rdkafka_properties_producer.request_required_acks,
+            Some(-1)
+        );
 
-        let props: HashMap<String, String> = hashmap! {
+        let props: BTreeMap<String, String> = btreemap! {
             // basic
             "connector".to_string() => "kafka".to_string(),
             "properties.bootstrap.server".to_string() => "localhost:9092".to_string(),
@@ -629,9 +649,9 @@ mod test {
 
             "properties.enable.idempotence".to_string() => "True".to_string(), // can only be 'true' or 'false'
         };
-        assert!(KafkaConfig::from_hashmap(props).is_err());
+        assert!(KafkaConfig::from_btreemap(props).is_err());
 
-        let props: HashMap<String, String> = hashmap! {
+        let props: BTreeMap<String, String> = btreemap! {
             // basic
             "connector".to_string() => "kafka".to_string(),
             "properties.bootstrap.server".to_string() => "localhost:9092".to_string(),
@@ -639,9 +659,9 @@ mod test {
             "type".to_string() => "append-only".to_string(),
             "properties.queue.buffering.max.kbytes".to_string() => "-114514".to_string(), // usize cannot be negative
         };
-        assert!(KafkaConfig::from_hashmap(props).is_err());
+        assert!(KafkaConfig::from_btreemap(props).is_err());
 
-        let props: HashMap<String, String> = hashmap! {
+        let props: BTreeMap<String, String> = btreemap! {
             // basic
             "connector".to_string() => "kafka".to_string(),
             "properties.bootstrap.server".to_string() => "localhost:9092".to_string(),
@@ -649,12 +669,12 @@ mod test {
             "type".to_string() => "append-only".to_string(),
             "properties.compression.codec".to_string() => "notvalid".to_string(), // has to be a valid CompressionCodec
         };
-        assert!(KafkaConfig::from_hashmap(props).is_err());
+        assert!(KafkaConfig::from_btreemap(props).is_err());
     }
 
     #[test]
     fn parse_kafka_config() {
-        let properties: HashMap<String, String> = hashmap! {
+        let properties: BTreeMap<String, String> = btreemap! {
             // "connector".to_string() => "kafka".to_string(),
             "properties.bootstrap.server".to_string() => "localhost:9092".to_string(),
             "topic".to_string() => "test".to_string(),
@@ -669,48 +689,48 @@ mod test {
             // PrivateLink
             "broker.rewrite.endpoints".to_string() => "{\"broker1\": \"10.0.0.1:8001\"}".to_string(),
         };
-        let config = KafkaConfig::from_hashmap(properties).unwrap();
+        let config = KafkaConfig::from_btreemap(properties).unwrap();
         assert_eq!(config.common.brokers, "localhost:9092");
         assert_eq!(config.common.topic, "test");
         assert_eq!(config.max_retry_num, 20);
         assert_eq!(config.retry_interval, Duration::from_millis(500));
 
         // PrivateLink fields
-        let hashmap: HashMap<String, String> = hashmap! {
+        let btreemap: BTreeMap<String, String> = btreemap! {
             "broker1".to_string() => "10.0.0.1:8001".to_string()
         };
-        assert_eq!(config.privatelink_common.broker_rewrite_map, Some(hashmap));
+        assert_eq!(config.privatelink_common.broker_rewrite_map, Some(btreemap));
 
         // Optional fields eliminated.
-        let properties: HashMap<String, String> = hashmap! {
+        let properties: BTreeMap<String, String> = btreemap! {
             // "connector".to_string() => "kafka".to_string(),
             "properties.bootstrap.server".to_string() => "localhost:9092".to_string(),
             "topic".to_string() => "test".to_string(),
             // "type".to_string() => "upsert".to_string(),
         };
-        let config = KafkaConfig::from_hashmap(properties).unwrap();
+        let config = KafkaConfig::from_btreemap(properties).unwrap();
         assert_eq!(config.max_retry_num, 3);
         assert_eq!(config.retry_interval, Duration::from_millis(100));
 
         // Invalid u32 input.
-        let properties: HashMap<String, String> = hashmap! {
+        let properties: BTreeMap<String, String> = btreemap! {
             "connector".to_string() => "kafka".to_string(),
             "properties.bootstrap.server".to_string() => "localhost:9092".to_string(),
             "topic".to_string() => "test".to_string(),
             "type".to_string() => "upsert".to_string(),
             "properties.retry.max".to_string() => "-20".to_string(),  // error!
         };
-        assert!(KafkaConfig::from_hashmap(properties).is_err());
+        assert!(KafkaConfig::from_btreemap(properties).is_err());
 
         // Invalid duration input.
-        let properties: HashMap<String, String> = hashmap! {
+        let properties: BTreeMap<String, String> = btreemap! {
             "connector".to_string() => "kafka".to_string(),
             "properties.bootstrap.server".to_string() => "localhost:9092".to_string(),
             "topic".to_string() => "test".to_string(),
             "type".to_string() => "upsert".to_string(),
             "properties.retry.interval".to_string() => "500minutes".to_string(),  // error!
         };
-        assert!(KafkaConfig::from_hashmap(properties).is_err());
+        assert!(KafkaConfig::from_btreemap(properties).is_err());
     }
 
     /// Note: Please enable the kafka by running `./risedev configure` before commenting #[ignore]
@@ -719,7 +739,7 @@ mod test {
     #[tokio::test]
     async fn test_kafka_producer() -> Result<()> {
         // Create a dummy kafka properties
-        let properties = hashmap! {
+        let properties = btreemap! {
             "connector".to_string() => "kafka".to_string(),
             "properties.bootstrap.server".to_string() => "localhost:29092".to_string(),
             "type".to_string() => "append-only".to_string(),
@@ -743,7 +763,7 @@ mod test {
             },
         ]);
 
-        let kafka_config = KafkaConfig::from_hashmap(properties)?;
+        let kafka_config = KafkaConfig::from_btreemap(properties)?;
 
         // Create the actual sink writer to Kafka
         let sink = KafkaSinkWriter::new(
diff --git a/src/connector/src/sink/kinesis.rs b/src/connector/src/sink/kinesis.rs
index 3b5533a49c76c..771d3c8a6f91d 100644
--- a/src/connector/src/sink/kinesis.rs
+++ b/src/connector/src/sink/kinesis.rs
@@ -12,12 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 
 use anyhow::{anyhow, Context};
-use aws_sdk_kinesis::operation::put_record::PutRecordOutput;
+use aws_sdk_kinesis::operation::put_records::builders::PutRecordsFluentBuilder;
 use aws_sdk_kinesis::primitives::Blob;
+use aws_sdk_kinesis::types::PutRecordsRequestEntry;
 use aws_sdk_kinesis::Client as KinesisClient;
+use futures::{FutureExt, TryFuture};
 use risingwave_common::array::StreamChunk;
 use risingwave_common::catalog::Schema;
 use risingwave_common::session_config::sink_decouple::SinkDecouple;
@@ -56,7 +58,7 @@ impl TryFrom<SinkParam> for KinesisSink {
 
     fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
         let schema = param.schema();
-        let config = KinesisSinkConfig::from_hashmap(param.properties)?;
+        let config = KinesisSinkConfig::from_btreemap(param.properties)?;
         Ok(Self {
             config,
             schema,
@@ -70,17 +72,18 @@ impl TryFrom<SinkParam> for KinesisSink {
     }
 }
 
+const KINESIS_SINK_MAX_PENDING_CHUNK_NUM: usize = 64;
+
 impl Sink for KinesisSink {
     type Coordinator = DummySinkCommitCoordinator;
     type LogSinker = AsyncTruncateLogSinkerOf<KinesisSinkWriter>;
 
     const SINK_NAME: &'static str = KINESIS_SINK;
 
-    fn is_sink_decouple(desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
+    fn is_sink_decouple(_desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
         match user_specified {
-            SinkDecouple::Default => Ok(desc.sink_type.is_append_only()),
+            SinkDecouple::Default | SinkDecouple::Enable => Ok(true),
             SinkDecouple::Disable => Ok(false),
-            SinkDecouple::Enable => Ok(true),
         }
     }
 
@@ -125,7 +128,7 @@ impl Sink for KinesisSink {
             self.sink_from_name.clone(),
         )
         .await?
-        .into_log_sinker(usize::MAX))
+        .into_log_sinker(KINESIS_SINK_MAX_PENDING_CHUNK_NUM))
     }
 }
 
@@ -137,7 +140,7 @@ pub struct KinesisSinkConfig {
 }
 
 impl KinesisSinkConfig {
-    pub fn from_hashmap(properties: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(properties: BTreeMap<String, String>) -> Result<Self> {
         let config =
             serde_json::from_value::<KinesisSinkConfig>(serde_json::to_value(properties).unwrap())
                 .map_err(|e| SinkError::Config(anyhow!(e)))?;
@@ -148,12 +151,13 @@ impl KinesisSinkConfig {
 pub struct KinesisSinkWriter {
     pub config: KinesisSinkConfig,
     formatter: SinkFormatterImpl,
-    payload_writer: KinesisSinkPayloadWriter,
+    client: KinesisClient,
 }
 
 struct KinesisSinkPayloadWriter {
-    client: KinesisClient,
-    config: KinesisSinkConfig,
+    // builder should always be `Some`. Making it an option so that we can call
+    // builder methods that take the builder ownership as input and return with a new builder.
+    builder: Option<PutRecordsFluentBuilder>,
 }
 
 impl KinesisSinkWriter {
@@ -182,29 +186,57 @@ impl KinesisSinkWriter {
         Ok(Self {
             config: config.clone(),
             formatter,
-            payload_writer: KinesisSinkPayloadWriter { client, config },
+            client,
         })
     }
+
+    fn new_payload_writer(&self) -> KinesisSinkPayloadWriter {
+        let builder = self
+            .client
+            .put_records()
+            .stream_name(&self.config.common.stream_name);
+        KinesisSinkPayloadWriter {
+            builder: Some(builder),
+        }
+    }
 }
+
+pub type KinesisSinkPayloadWriterDeliveryFuture =
+    impl TryFuture<Ok = (), Error = SinkError> + Unpin + Send + 'static;
+
 impl KinesisSinkPayloadWriter {
-    async fn put_record(&self, key: &str, payload: Vec<u8>) -> Result<PutRecordOutput> {
-        let payload = Blob::new(payload);
-        // todo: switch to put_records() for batching
-        Retry::spawn(
-            ExponentialBackoff::from_millis(100).map(jitter).take(3),
-            || async {
-                self.client
-                    .put_record()
-                    .stream_name(&self.config.common.stream_name)
+    fn put_record(&mut self, key: String, payload: Vec<u8>) {
+        self.builder = Some(
+            self.builder.take().expect("should not be None").records(
+                PutRecordsRequestEntry::builder()
                     .partition_key(key)
-                    .data(payload.clone())
-                    .send()
-                    .await
-            },
-        )
-        .await
-        .with_context(|| format!("failed to put record to {}", self.config.common.stream_name))
-        .map_err(SinkError::Kinesis)
+                    .data(Blob::new(payload))
+                    .build()
+                    .expect("should not fail because we have set `data` and `partition_key`"),
+            ),
+        );
+    }
+
+    fn finish(self) -> KinesisSinkPayloadWriterDeliveryFuture {
+        async move {
+            let builder = self.builder.expect("should not be None");
+            let context_fmt = format!(
+                "failed to put record to {}",
+                builder
+                    .get_stream_name()
+                    .as_ref()
+                    .expect("should have set stream name")
+            );
+            Retry::spawn(
+                ExponentialBackoff::from_millis(100).map(jitter).take(3),
+                || builder.clone().send(),
+            )
+            .await
+            .with_context(|| context_fmt.clone())
+            .map_err(SinkError::Kinesis)?;
+            Ok(())
+        }
+        .boxed()
     }
 }
 
@@ -214,24 +246,46 @@ impl FormattedSink for KinesisSinkPayloadWriter {
 
     async fn write_one(&mut self, k: Option<Self::K>, v: Option<Self::V>) -> Result<()> {
         self.put_record(
-            &k.ok_or_else(|| SinkError::Kinesis(anyhow!("no key provided")))?,
+            k.ok_or_else(|| SinkError::Kinesis(anyhow!("no key provided")))?,
             v.unwrap_or_default(),
-        )
-        .await
-        .map(|_| ())
+        );
+        Ok(())
     }
 }
 
 impl AsyncTruncateSinkWriter for KinesisSinkWriter {
+    type DeliveryFuture = KinesisSinkPayloadWriterDeliveryFuture;
+
     async fn write_chunk<'a>(
         &'a mut self,
         chunk: StreamChunk,
-        _add_future: DeliveryFutureManagerAddFuture<'a, Self::DeliveryFuture>,
+        mut add_future: DeliveryFutureManagerAddFuture<'a, Self::DeliveryFuture>,
     ) -> Result<()> {
+        let mut payload_writer = self.new_payload_writer();
         dispatch_sink_formatter_str_key_impl!(
             &self.formatter,
             formatter,
-            self.payload_writer.write_chunk(chunk, formatter).await
-        )
+            payload_writer.write_chunk(chunk, formatter).await
+        )?;
+
+        add_future
+            .add_future_may_await(payload_writer.finish())
+            .await?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use aws_sdk_kinesis::types::PutRecordsRequestEntry;
+    use aws_smithy_types::Blob;
+
+    #[test]
+    fn test_kinesis_entry_builder_save_unwrap() {
+        PutRecordsRequestEntry::builder()
+            .data(Blob::new(b"data"))
+            .partition_key("partition-key")
+            .build()
+            .unwrap();
     }
 }
diff --git a/src/connector/src/sink/log_store.rs b/src/connector/src/sink/log_store.rs
index c2cdf7756c598..d609aa7170d0d 100644
--- a/src/connector/src/sink/log_store.rs
+++ b/src/connector/src/sink/log_store.rs
@@ -18,11 +18,14 @@ use std::fmt::Debug;
 use std::future::{poll_fn, Future};
 use std::sync::Arc;
 use std::task::Poll;
+use std::time::Instant;
 
+use await_tree::InstrumentAwait;
 use futures::{TryFuture, TryFutureExt};
 use risingwave_common::array::StreamChunk;
 use risingwave_common::bail;
 use risingwave_common::buffer::Bitmap;
+use risingwave_common::metrics::LabelGuardedIntCounter;
 use risingwave_common::util::epoch::{EpochPair, INVALID_EPOCH};
 
 use crate::sink::SinkMetrics;
@@ -161,10 +164,7 @@ pub trait LogReader: Send + Sized + 'static {
 
     /// Mark that all items emitted so far have been consumed and it is safe to truncate the log
     /// from the current offset.
-    fn truncate(
-        &mut self,
-        offset: TruncateOffset,
-    ) -> impl Future<Output = LogStoreResult<()>> + Send + '_;
+    fn truncate(&mut self, offset: TruncateOffset) -> LogStoreResult<()>;
 
     /// Reset the log reader to after the latest truncate offset
     ///
@@ -205,10 +205,7 @@ impl<F: Fn(StreamChunk) -> StreamChunk + Send + 'static, R: LogReader> LogReader
         Ok((epoch, item))
     }
 
-    fn truncate(
-        &mut self,
-        offset: TruncateOffset,
-    ) -> impl Future<Output = LogStoreResult<()>> + Send + '_ {
+    fn truncate(&mut self, offset: TruncateOffset) -> LogStoreResult<()> {
         self.inner.truncate(offset)
     }
 
@@ -219,39 +216,102 @@ impl<F: Fn(StreamChunk) -> StreamChunk + Send + 'static, R: LogReader> LogReader
     }
 }
 
+pub struct BackpressureMonitoredLogReader<R: LogReader> {
+    inner: R,
+    /// Start time to wait for new future after poll ready
+    wait_new_future_start_time: Option<Instant>,
+    wait_new_future_duration_ns: LabelGuardedIntCounter<3>,
+}
+
+impl<R: LogReader> BackpressureMonitoredLogReader<R> {
+    fn new(inner: R, wait_new_future_duration_ns: LabelGuardedIntCounter<3>) -> Self {
+        Self {
+            inner,
+            wait_new_future_start_time: None,
+            wait_new_future_duration_ns,
+        }
+    }
+}
+
+impl<R: LogReader> LogReader for BackpressureMonitoredLogReader<R> {
+    fn init(&mut self) -> impl Future<Output = LogStoreResult<()>> + Send + '_ {
+        self.wait_new_future_start_time = None;
+        self.inner.init()
+    }
+
+    fn next_item(
+        &mut self,
+    ) -> impl Future<Output = LogStoreResult<(u64, LogStoreReadItem)>> + Send + '_ {
+        if let Some(start_time) = self.wait_new_future_start_time.take() {
+            self.wait_new_future_duration_ns
+                .inc_by(start_time.elapsed().as_nanos() as _);
+        }
+        self.inner.next_item().inspect_ok(|_| {
+            // Set start time when return ready
+            self.wait_new_future_start_time = Some(Instant::now());
+        })
+    }
+
+    fn truncate(&mut self, offset: TruncateOffset) -> LogStoreResult<()> {
+        self.inner.truncate(offset)
+    }
+
+    fn rewind(
+        &mut self,
+    ) -> impl Future<Output = LogStoreResult<(bool, Option<Bitmap>)>> + Send + '_ {
+        self.inner.rewind().inspect_ok(|_| {
+            self.wait_new_future_start_time = None;
+        })
+    }
+}
+
 pub struct MonitoredLogReader<R: LogReader> {
     inner: R,
     read_epoch: u64,
     metrics: SinkMetrics,
 }
 
+impl<R: LogReader> MonitoredLogReader<R> {
+    pub fn new(inner: R, metrics: SinkMetrics) -> Self {
+        Self {
+            inner,
+            read_epoch: INVALID_EPOCH,
+            metrics,
+        }
+    }
+}
+
 impl<R: LogReader> LogReader for MonitoredLogReader<R> {
     async fn init(&mut self) -> LogStoreResult<()> {
-        self.inner.init().await
+        self.inner.init().instrument_await("log_reader_init").await
     }
 
     async fn next_item(&mut self) -> LogStoreResult<(u64, LogStoreReadItem)> {
-        self.inner.next_item().await.inspect(|(epoch, item)| {
-            if self.read_epoch != *epoch {
-                self.read_epoch = *epoch;
-                self.metrics.log_store_latest_read_epoch.set(*epoch as _);
-            }
-            if let LogStoreReadItem::StreamChunk { chunk, .. } = item {
-                self.metrics
-                    .log_store_read_rows
-                    .inc_by(chunk.cardinality() as _);
-            }
-        })
+        self.inner
+            .next_item()
+            .instrument_await("log_reader_next_item")
+            .await
+            .inspect(|(epoch, item)| {
+                if self.read_epoch != *epoch {
+                    self.read_epoch = *epoch;
+                    self.metrics.log_store_latest_read_epoch.set(*epoch as _);
+                }
+                if let LogStoreReadItem::StreamChunk { chunk, .. } = item {
+                    self.metrics
+                        .log_store_read_rows
+                        .inc_by(chunk.cardinality() as _);
+                }
+            })
     }
 
-    async fn truncate(&mut self, offset: TruncateOffset) -> LogStoreResult<()> {
-        self.inner.truncate(offset).await
+    fn truncate(&mut self, offset: TruncateOffset) -> LogStoreResult<()> {
+        self.inner.truncate(offset)
     }
 
     fn rewind(
         &mut self,
     ) -> impl Future<Output = LogStoreResult<(bool, Option<Bitmap>)>> + Send + '_ {
-        self.inner.rewind()
+        self.inner.rewind().instrument_await("log_reader_rewind")
     }
 }
 
@@ -267,12 +327,12 @@ where
         TransformChunkLogReader { f, inner: self }
     }
 
-    pub fn monitored(self, metrics: SinkMetrics) -> MonitoredLogReader<T> {
-        MonitoredLogReader {
-            read_epoch: INVALID_EPOCH,
-            inner: self,
-            metrics,
-        }
+    pub fn monitored(self, metrics: SinkMetrics) -> impl LogReader {
+        let wait_new_future_duration = metrics.log_store_reader_wait_new_future_duration_ns.clone();
+        BackpressureMonitoredLogReader::new(
+            MonitoredLogReader::new(self, metrics),
+            wait_new_future_duration,
+        )
     }
 }
 
diff --git a/src/connector/src/sink/mod.rs b/src/connector/src/sink/mod.rs
index c430b4303f1e9..bdd923f786ec4 100644
--- a/src/connector/src/sink/mod.rs
+++ b/src/connector/src/sink/mod.rs
@@ -17,12 +17,15 @@ pub mod boxed;
 pub mod catalog;
 pub mod clickhouse;
 pub mod coordinate;
+pub mod decouple_checkpoint_log_sink;
 pub mod deltalake;
 pub mod doris;
 pub mod doris_starrocks_connector;
+pub mod dynamodb;
 pub mod elasticsearch;
 pub mod encoder;
 pub mod formatter;
+pub mod google_pubsub;
 pub mod iceberg;
 pub mod kafka;
 pub mod kinesis;
@@ -35,13 +38,14 @@ pub mod redis;
 pub mod remote;
 pub mod snowflake;
 pub mod snowflake_connector;
+pub mod sqlserver;
 pub mod starrocks;
 pub mod test_sink;
 pub mod trivial;
 pub mod utils;
 pub mod writer;
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::future::Future;
 
 use ::clickhouse::error::Error as ClickHouseError;
@@ -70,7 +74,6 @@ use crate::sink::catalog::desc::SinkDesc;
 use crate::sink::catalog::{SinkCatalog, SinkId};
 use crate::sink::log_store::{LogReader, LogStoreReadItem, LogStoreResult, TruncateOffset};
 use crate::sink::writer::SinkWriter;
-use crate::ConnectorParams;
 
 const BOUNDED_CHANNEL_SIZE: usize = 16;
 #[macro_export]
@@ -86,6 +89,7 @@ macro_rules! for_all_sinks {
                 { ClickHouse, $crate::sink::clickhouse::ClickHouseSink },
                 { Iceberg, $crate::sink::iceberg::IcebergSink },
                 { Mqtt, $crate::sink::mqtt::MqttSink },
+                { GooglePubSub, $crate::sink::google_pubsub::GooglePubSubSink },
                 { Nats, $crate::sink::nats::NatsSink },
                 { Jdbc, $crate::sink::remote::JdbcSink },
                 { ElasticSearch, $crate::sink::remote::ElasticSearchSink },
@@ -96,6 +100,8 @@ macro_rules! for_all_sinks {
                 { Snowflake, $crate::sink::snowflake::SnowflakeSink },
                 { DeltaLake, $crate::sink::deltalake::DeltaLakeSink },
                 { BigQuery, $crate::sink::big_query::BigQuerySink },
+                { DynamoDb, $crate::sink::dynamodb::DynamoDbSink },
+                { SqlServer, $crate::sink::sqlserver::SqlServerSink },
                 { Test, $crate::sink::test_sink::TestSink },
                 { Table, $crate::sink::trivial::TableSink }
             }
@@ -153,7 +159,7 @@ pub const SINK_USER_FORCE_APPEND_ONLY_OPTION: &str = "force_append_only";
 pub struct SinkParam {
     pub sink_id: SinkId,
     pub sink_name: String,
-    pub properties: HashMap<String, String>,
+    pub properties: BTreeMap<String, String>,
     pub columns: Vec<ColumnDesc>,
     pub downstream_pk: Vec<usize>,
     pub sink_type: SinkType,
@@ -254,6 +260,8 @@ pub struct SinkMetrics {
     pub log_store_latest_read_epoch: LabelGuardedIntGauge<3>,
     pub log_store_read_rows: LabelGuardedIntCounter<3>,
 
+    pub log_store_reader_wait_new_future_duration_ns: LabelGuardedIntCounter<3>,
+
     pub iceberg_write_qps: LabelGuardedIntCounter<2>,
     pub iceberg_write_latency: LabelGuardedHistogram<2>,
     pub iceberg_rolling_unflushed_data_file: LabelGuardedIntGauge<2>,
@@ -271,6 +279,8 @@ impl SinkMetrics {
             log_store_latest_read_epoch: LabelGuardedIntGauge::test_int_gauge(),
             log_store_write_rows: LabelGuardedIntCounter::test_int_counter(),
             log_store_read_rows: LabelGuardedIntCounter::test_int_counter(),
+            log_store_reader_wait_new_future_duration_ns: LabelGuardedIntCounter::test_int_counter(
+            ),
             iceberg_write_qps: LabelGuardedIntCounter::test_int_counter(),
             iceberg_write_latency: LabelGuardedHistogram::test_histogram(),
             iceberg_rolling_unflushed_data_file: LabelGuardedIntGauge::test_int_gauge(),
@@ -282,7 +292,6 @@ impl SinkMetrics {
 
 #[derive(Clone)]
 pub struct SinkWriterParam {
-    pub connector_params: ConnectorParams,
     pub executor_id: u64,
     pub vnode_bitmap: Option<Bitmap>,
     pub meta_client: Option<SinkMetaClient>,
@@ -320,7 +329,6 @@ impl SinkMetaClient {
 impl SinkWriterParam {
     pub fn for_test() -> Self {
         SinkWriterParam {
-            connector_params: Default::default(),
             executor_id: Default::default(),
             vnode_bitmap: Default::default(),
             meta_client: Default::default(),
@@ -361,10 +369,7 @@ pub trait SinkLogReader: Send + Sized + 'static {
 
     /// Mark that all items emitted so far have been consumed and it is safe to truncate the log
     /// from the current offset.
-    fn truncate(
-        &mut self,
-        offset: TruncateOffset,
-    ) -> impl Future<Output = LogStoreResult<()>> + Send + '_;
+    fn truncate(&mut self, offset: TruncateOffset) -> LogStoreResult<()>;
 }
 
 impl<R: LogReader> SinkLogReader for R {
@@ -374,17 +379,14 @@ impl<R: LogReader> SinkLogReader for R {
         <Self as LogReader>::next_item(self)
     }
 
-    fn truncate(
-        &mut self,
-        offset: TruncateOffset,
-    ) -> impl Future<Output = LogStoreResult<()>> + Send + '_ {
+    fn truncate(&mut self, offset: TruncateOffset) -> LogStoreResult<()> {
         <Self as LogReader>::truncate(self, offset)
     }
 }
 
 #[async_trait]
 pub trait LogSinker: 'static {
-    async fn consume_log_and_sink(self, log_reader: &mut impl SinkLogReader) -> Result<()>;
+    async fn consume_log_and_sink(self, log_reader: &mut impl SinkLogReader) -> Result<!>;
 }
 
 #[async_trait]
@@ -525,6 +527,12 @@ pub enum SinkError {
         #[backtrace]
         anyhow::Error,
     ),
+    #[error("Google Pub/Sub error: {0}")]
+    GooglePubSub(
+        #[source]
+        #[backtrace]
+        anyhow::Error,
+    ),
     #[error("Doris/Starrocks connect error: {0}")]
     DorisStarrocksConnect(
         #[source]
@@ -542,7 +550,11 @@ pub enum SinkError {
     #[error("Starrocks error: {0}")]
     Starrocks(String),
     #[error("Snowflake error: {0}")]
-    Snowflake(String),
+    Snowflake(
+        #[source]
+        #[backtrace]
+        anyhow::Error,
+    ),
     #[error("Pulsar error: {0}")]
     Pulsar(
         #[source]
@@ -561,6 +573,18 @@ pub enum SinkError {
         #[backtrace]
         anyhow::Error,
     ),
+    #[error("DynamoDB error: {0}")]
+    DynamoDb(
+        #[source]
+        #[backtrace]
+        anyhow::Error,
+    ),
+    #[error("SQL Server error: {0}")]
+    SqlServer(
+        #[source]
+        #[backtrace]
+        anyhow::Error,
+    ),
     #[error(transparent)]
     Connector(
         #[from]
@@ -598,3 +622,9 @@ impl From<RedisError> for SinkError {
         SinkError::Redis(value.to_report_string())
     }
 }
+
+impl From<tiberius::error::Error> for SinkError {
+    fn from(err: tiberius::error::Error) -> Self {
+        SinkError::SqlServer(anyhow!(err))
+    }
+}
diff --git a/src/connector/src/sink/mqtt.rs b/src/connector/src/sink/mqtt.rs
index 308ec450878a9..d9dfdbe03b21b 100644
--- a/src/connector/src/sink/mqtt.rs
+++ b/src/connector/src/sink/mqtt.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 use core::fmt::Debug;
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::sync::atomic::AtomicBool;
 use std::sync::Arc;
 
@@ -120,6 +120,7 @@ pub struct MqttSink {
 pub struct MqttSinkWriter {
     pub config: MqttConfig,
     payload_writer: MqttSinkPayloadWriter,
+    #[expect(dead_code)]
     schema: Schema,
     encoder: RowEncoderWrapper,
     stopped: Arc<AtomicBool>,
@@ -127,7 +128,7 @@ pub struct MqttSinkWriter {
 
 /// Basic data types for use with the mqtt interface
 impl MqttConfig {
-    pub fn from_hashmap(values: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(values: BTreeMap<String, String>) -> Result<Self> {
         let config = serde_json::from_value::<MqttConfig>(serde_json::to_value(values).unwrap())
             .map_err(|e| SinkError::Config(anyhow!(e)))?;
         if config.r#type != SINK_TYPE_APPEND_ONLY {
@@ -145,7 +146,7 @@ impl TryFrom<SinkParam> for MqttSink {
 
     fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
         let schema = param.schema();
-        let config = MqttConfig::from_hashmap(param.properties)?;
+        let config = MqttConfig::from_btreemap(param.properties)?;
         Ok(Self {
             config,
             schema,
@@ -164,11 +165,10 @@ impl Sink for MqttSink {
 
     const SINK_NAME: &'static str = MQTT_SINK;
 
-    fn is_sink_decouple(desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
+    fn is_sink_decouple(_desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
         match user_specified {
-            SinkDecouple::Default => Ok(desc.sink_type.is_append_only()),
+            SinkDecouple::Default | SinkDecouple::Enable => Ok(true),
             SinkDecouple::Disable => Ok(false),
-            SinkDecouple::Enable => Ok(true),
         }
     }
 
diff --git a/src/connector/src/sink/nats.rs b/src/connector/src/sink/nats.rs
index f4cf6ac4d7372..162aca3c4d2e8 100644
--- a/src/connector/src/sink/nats.rs
+++ b/src/connector/src/sink/nats.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 use core::fmt::Debug;
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 
 use anyhow::{anyhow, Context as _};
 use async_nats::jetstream::context::Context;
@@ -59,13 +59,14 @@ pub struct NatsSink {
 pub struct NatsSinkWriter {
     pub config: NatsConfig,
     context: Context,
+    #[expect(dead_code)]
     schema: Schema,
     json_encoder: JsonEncoder,
 }
 
 /// Basic data types for use with the nats interface
 impl NatsConfig {
-    pub fn from_hashmap(values: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(values: BTreeMap<String, String>) -> Result<Self> {
         let config = serde_json::from_value::<NatsConfig>(serde_json::to_value(values).unwrap())
             .map_err(|e| SinkError::Config(anyhow!(e)))?;
         if config.r#type != SINK_TYPE_APPEND_ONLY {
@@ -83,7 +84,7 @@ impl TryFrom<SinkParam> for NatsSink {
 
     fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
         let schema = param.schema();
-        let config = NatsConfig::from_hashmap(param.properties)?;
+        let config = NatsConfig::from_btreemap(param.properties)?;
         Ok(Self {
             config,
             schema,
@@ -98,11 +99,10 @@ impl Sink for NatsSink {
 
     const SINK_NAME: &'static str = NATS_SINK;
 
-    fn is_sink_decouple(desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
+    fn is_sink_decouple(_desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
         match user_specified {
-            SinkDecouple::Default => Ok(desc.sink_type.is_append_only()),
+            SinkDecouple::Default | SinkDecouple::Enable => Ok(true),
             SinkDecouple::Disable => Ok(false),
-            SinkDecouple::Enable => Ok(true),
         }
     }
 
diff --git a/src/connector/src/sink/pulsar.rs b/src/connector/src/sink/pulsar.rs
index 2ffb3e18fb39f..3f016ad94946d 100644
--- a/src/connector/src/sink/pulsar.rs
+++ b/src/connector/src/sink/pulsar.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::fmt::Debug;
 use std::time::Duration;
 
@@ -127,7 +127,7 @@ pub struct PulsarConfig {
 }
 
 impl PulsarConfig {
-    pub fn from_hashmap(values: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(values: BTreeMap<String, String>) -> Result<Self> {
         let config = serde_json::from_value::<PulsarConfig>(serde_json::to_value(values).unwrap())
             .map_err(|e| SinkError::Config(anyhow!(e)))?;
 
@@ -150,7 +150,7 @@ impl TryFrom<SinkParam> for PulsarSink {
 
     fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
         let schema = param.schema();
-        let config = PulsarConfig::from_hashmap(param.properties)?;
+        let config = PulsarConfig::from_btreemap(param.properties)?;
         Ok(Self {
             config,
             schema,
@@ -170,11 +170,10 @@ impl Sink for PulsarSink {
 
     const SINK_NAME: &'static str = PULSAR_SINK;
 
-    fn is_sink_decouple(desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
+    fn is_sink_decouple(_desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
         match user_specified {
-            SinkDecouple::Default => Ok(desc.sink_type.is_append_only()),
+            SinkDecouple::Default | SinkDecouple::Enable => Ok(true),
             SinkDecouple::Disable => Ok(false),
-            SinkDecouple::Enable => Ok(true),
         }
     }
 
@@ -224,6 +223,7 @@ impl Sink for PulsarSink {
 
 pub struct PulsarSinkWriter {
     formatter: SinkFormatterImpl,
+    #[expect(dead_code)]
     pulsar: Pulsar<TokioExecutor>,
     producer: Producer<TokioExecutor>,
     config: PulsarConfig,
@@ -286,7 +286,7 @@ impl<'w> PulsarPayloadWriter<'w> {
             if retry_num > 0 {
                 tracing::warn!("Failed to send message, at retry no. {retry_num}");
             }
-            match self.producer.send(message.clone()).await {
+            match self.producer.send_non_blocking(message.clone()).await {
                 // If the message is sent successfully,
                 // a SendFuture holding the message receipt
                 // or error after sending is returned
diff --git a/src/connector/src/sink/redis.rs b/src/connector/src/sink/redis.rs
index 96b524d5b7b21..9d6a33d5131a4 100644
--- a/src/connector/src/sink/redis.rs
+++ b/src/connector/src/sink/redis.rs
@@ -12,13 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeMap, HashSet};
 
 use anyhow::anyhow;
 use async_trait::async_trait;
-use redis::aio::{ConnectionLike, MultiplexedConnection};
-use redis::cluster::ClusterClient;
-use redis::cluster_async::ClusterConnection;
+use redis::aio::MultiplexedConnection;
+use redis::cluster::{ClusterClient, ClusterConnection, ClusterPipeline};
 use redis::{Client as RedisClient, Pipeline};
 use risingwave_common::array::StreamChunk;
 use risingwave_common::catalog::Schema;
@@ -49,46 +48,63 @@ pub struct RedisCommon {
     #[serde(rename = "redis.url")]
     pub url: String,
 }
-pub enum RedisConn {
-    // Redis deployed as a cluster, clusters with only one node should also use this conn
-    Cluster(ClusterConnection),
-    // Redis is not deployed as a cluster
-    Single(MultiplexedConnection),
+
+pub enum RedisPipe {
+    Cluster(ClusterPipeline),
+    Single(Pipeline),
 }
+impl RedisPipe {
+    pub async fn query<T: redis::FromRedisValue>(
+        &self,
+        conn: &mut RedisConn,
+    ) -> ConnectorResult<T> {
+        match (self, conn) {
+            (RedisPipe::Cluster(pipe), RedisConn::Cluster(conn)) => Ok(pipe.query(conn)?),
+            (RedisPipe::Single(pipe), RedisConn::Single(conn)) => {
+                Ok(pipe.query_async(conn).await?)
+            }
+            _ => Err(SinkError::Redis("RedisPipe and RedisConn not match".to_string()).into()),
+        }
+    }
 
-impl ConnectionLike for RedisConn {
-    fn req_packed_command<'a>(
-        &'a mut self,
-        cmd: &'a redis::Cmd,
-    ) -> redis::RedisFuture<'a, redis::Value> {
+    pub fn clear(&mut self) {
         match self {
-            RedisConn::Cluster(conn) => conn.req_packed_command(cmd),
-            RedisConn::Single(conn) => conn.req_packed_command(cmd),
+            RedisPipe::Cluster(pipe) => pipe.clear(),
+            RedisPipe::Single(pipe) => pipe.clear(),
         }
     }
 
-    fn req_packed_commands<'a>(
-        &'a mut self,
-        cmd: &'a redis::Pipeline,
-        offset: usize,
-        count: usize,
-    ) -> redis::RedisFuture<'a, Vec<redis::Value>> {
+    pub fn set(&mut self, k: String, v: Vec<u8>) {
         match self {
-            RedisConn::Cluster(conn) => conn.req_packed_commands(cmd, offset, count),
-            RedisConn::Single(conn) => conn.req_packed_commands(cmd, offset, count),
-        }
+            RedisPipe::Cluster(pipe) => {
+                pipe.set(k, v);
+            }
+            RedisPipe::Single(pipe) => {
+                pipe.set(k, v);
+            }
+        };
     }
 
-    fn get_db(&self) -> i64 {
+    pub fn del(&mut self, k: String) {
         match self {
-            RedisConn::Cluster(conn) => conn.get_db(),
-            RedisConn::Single(conn) => conn.get_db(),
-        }
+            RedisPipe::Cluster(pipe) => {
+                pipe.del(k);
+            }
+            RedisPipe::Single(pipe) => {
+                pipe.del(k);
+            }
+        };
     }
 }
+pub enum RedisConn {
+    // Redis deployed as a cluster, clusters with only one node should also use this conn
+    Cluster(ClusterConnection),
+    // Redis is not deployed as a cluster
+    Single(MultiplexedConnection),
+}
 
 impl RedisCommon {
-    pub async fn build_conn(&self) -> ConnectorResult<RedisConn> {
+    pub async fn build_conn_and_pipe(&self) -> ConnectorResult<(RedisConn, RedisPipe)> {
         match serde_json::from_str(&self.url).map_err(|e| SinkError::Config(anyhow!(e))) {
             Ok(v) => {
                 if let Value::Array(list) = v {
@@ -107,20 +123,25 @@ impl RedisCommon {
                         .collect::<ConnectorResult<Vec<String>>>()?;
 
                     let client = ClusterClient::new(list)?;
-                    Ok(RedisConn::Cluster(client.get_async_connection().await?))
+                    Ok((
+                        RedisConn::Cluster(client.get_connection()?),
+                        RedisPipe::Cluster(redis::cluster::cluster_pipe()),
+                    ))
                 } else {
                     Err(SinkError::Redis("redis.url must be array or string".to_string()).into())
                 }
             }
             Err(_) => {
                 let client = RedisClient::open(self.url.clone())?;
-                Ok(RedisConn::Single(
-                    client.get_multiplexed_async_connection().await?,
+                Ok((
+                    RedisConn::Single(client.get_multiplexed_async_connection().await?),
+                    RedisPipe::Single(redis::pipe()),
                 ))
             }
         }
     }
 }
+
 #[serde_as]
 #[derive(Clone, Debug, Deserialize, WithOptions)]
 pub struct RedisConfig {
@@ -129,7 +150,7 @@ pub struct RedisConfig {
 }
 
 impl RedisConfig {
-    pub fn from_hashmap(properties: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(properties: BTreeMap<String, String>) -> Result<Self> {
         let config =
             serde_json::from_value::<RedisConfig>(serde_json::to_value(properties).unwrap())
                 .map_err(|e| SinkError::Config(anyhow!(e)))?;
@@ -157,7 +178,7 @@ impl TryFrom<SinkParam> for RedisSink {
                 "Redis Sink Primary Key must be specified."
             )));
         }
-        let config = RedisConfig::from_hashmap(param.properties.clone())?;
+        let config = RedisConfig::from_btreemap(param.properties.clone())?;
         Ok(Self {
             config,
             schema: param.schema(),
@@ -191,7 +212,7 @@ impl Sink for RedisSink {
     }
 
     async fn validate(&self) -> Result<()> {
-        let _conn = self.config.common.build_conn().await?;
+        self.config.common.build_conn_and_pipe().await?;
         let all_set: HashSet<String> = self
             .schema
             .fields()
@@ -228,8 +249,11 @@ impl Sink for RedisSink {
 }
 
 pub struct RedisSinkWriter {
+    #[expect(dead_code)]
     epoch: u64,
+    #[expect(dead_code)]
     schema: Schema,
+    #[expect(dead_code)]
     pk_indices: Vec<usize>,
     formatter: SinkFormatterImpl,
     payload_writer: RedisSinkPayloadWriter,
@@ -239,13 +263,13 @@ struct RedisSinkPayloadWriter {
     // connection to redis, one per executor
     conn: Option<RedisConn>,
     // the command pipeline for write-commit
-    pipe: Pipeline,
+    pipe: RedisPipe,
 }
+
 impl RedisSinkPayloadWriter {
     pub async fn new(config: RedisConfig) -> Result<Self> {
-        let conn = config.common.build_conn().await?;
+        let (conn, pipe) = config.common.build_conn_and_pipe().await?;
         let conn = Some(conn);
-        let pipe = redis::pipe();
 
         Ok(Self { conn, pipe })
     }
@@ -253,7 +277,7 @@ impl RedisSinkPayloadWriter {
     #[cfg(test)]
     pub fn mock() -> Self {
         let conn = None;
-        let pipe = redis::pipe();
+        let pipe = RedisPipe::Single(redis::pipe());
         Self { conn, pipe }
     }
 
@@ -264,7 +288,7 @@ impl RedisSinkPayloadWriter {
                 return Ok(());
             }
         }
-        self.pipe.query_async(self.conn.as_mut().unwrap()).await?;
+        self.pipe.query(self.conn.as_mut().unwrap()).await?;
         self.pipe.clear();
         Ok(())
     }
@@ -353,11 +377,11 @@ impl AsyncTruncateSinkWriter for RedisSinkWriter {
 
 #[cfg(test)]
 mod test {
-    use std::collections::BTreeMap;
+    use core::panic;
 
     use rdkafka::message::FromBytes;
-    use risingwave_common::array::{Array, I32Array, Op, StreamChunk, Utf8Array};
-    use risingwave_common::catalog::{Field, Schema};
+    use risingwave_common::array::{Array, I32Array, Op, Utf8Array};
+    use risingwave_common::catalog::Field;
     use risingwave_common::types::DataType;
     use risingwave_common::util::iter_util::ZipEqDebug;
 
@@ -386,6 +410,7 @@ mod test {
             format: SinkFormat::AppendOnly,
             encode: SinkEncode::Json,
             options: BTreeMap::default(),
+            key_encode: None,
         };
 
         let mut redis_sink_writer = RedisSinkWriter::mock(schema, vec![0], &format_desc)
@@ -406,24 +431,33 @@ mod test {
             .write_chunk(chunk_a, manager.start_write_chunk(0, 0))
             .await
             .expect("failed to write batch");
-        let expected_a =
-            vec![
-            (0, "*3\r\n$3\r\nSET\r\n$8\r\n{\"id\":1}\r\n$23\r\n{\"id\":1,\"name\":\"Alice\"}\r\n"),
-            (1, "*3\r\n$3\r\nSET\r\n$8\r\n{\"id\":2}\r\n$21\r\n{\"id\":2,\"name\":\"Bob\"}\r\n"),
-            (2, "*3\r\n$3\r\nSET\r\n$8\r\n{\"id\":3}\r\n$23\r\n{\"id\":3,\"name\":\"Clare\"}\r\n"),
+        let expected_a = vec![
+            (
+                0,
+                "*3\r\n$3\r\nSET\r\n$8\r\n{\"id\":1}\r\n$23\r\n{\"id\":1,\"name\":\"Alice\"}\r\n",
+            ),
+            (
+                1,
+                "*3\r\n$3\r\nSET\r\n$8\r\n{\"id\":2}\r\n$21\r\n{\"id\":2,\"name\":\"Bob\"}\r\n",
+            ),
+            (
+                2,
+                "*3\r\n$3\r\nSET\r\n$8\r\n{\"id\":3}\r\n$23\r\n{\"id\":3,\"name\":\"Clare\"}\r\n",
+            ),
         ];
 
-        redis_sink_writer
-            .payload_writer
-            .pipe
-            .cmd_iter()
-            .enumerate()
-            .zip_eq_debug(expected_a.clone())
-            .for_each(|((i, cmd), (exp_i, exp_cmd))| {
-                if exp_i == i {
-                    assert_eq!(exp_cmd, str::from_bytes(&cmd.get_packed_command()).unwrap())
-                }
-            });
+        if let RedisPipe::Single(pipe) = &redis_sink_writer.payload_writer.pipe {
+            pipe.cmd_iter()
+                .enumerate()
+                .zip_eq_debug(expected_a.clone())
+                .for_each(|((i, cmd), (exp_i, exp_cmd))| {
+                    if exp_i == i {
+                        assert_eq!(exp_cmd, str::from_bytes(&cmd.get_packed_command()).unwrap())
+                    }
+                });
+        } else {
+            panic!("pipe type not match")
+        }
     }
 
     #[tokio::test]
@@ -453,6 +487,7 @@ mod test {
             format: SinkFormat::AppendOnly,
             encode: SinkEncode::Template,
             options: btree_map,
+            key_encode: None,
         };
 
         let mut redis_sink_writer = RedisSinkWriter::mock(schema, vec![0], &format_desc)
@@ -488,16 +523,17 @@ mod test {
             ),
         ];
 
-        redis_sink_writer
-            .payload_writer
-            .pipe
-            .cmd_iter()
-            .enumerate()
-            .zip_eq_debug(expected_a.clone())
-            .for_each(|((i, cmd), (exp_i, exp_cmd))| {
-                if exp_i == i {
-                    assert_eq!(exp_cmd, str::from_bytes(&cmd.get_packed_command()).unwrap())
-                }
-            });
+        if let RedisPipe::Single(pipe) = &redis_sink_writer.payload_writer.pipe {
+            pipe.cmd_iter()
+                .enumerate()
+                .zip_eq_debug(expected_a.clone())
+                .for_each(|((i, cmd), (exp_i, exp_cmd))| {
+                    if exp_i == i {
+                        assert_eq!(exp_cmd, str::from_bytes(&cmd.get_packed_command()).unwrap())
+                    }
+                });
+        } else {
+            panic!("pipe type not match")
+        };
     }
 }
diff --git a/src/connector/src/sink/remote.rs b/src/connector/src/sink/remote.rs
index 4d3c48b6ec5c7..f8b84fc64eb86 100644
--- a/src/connector/src/sink/remote.rs
+++ b/src/connector/src/sink/remote.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{HashMap, VecDeque};
+use std::collections::{BTreeMap, VecDeque};
 use std::marker::PhantomData;
 use std::ops::Deref;
 use std::pin::pin;
@@ -20,8 +20,9 @@ use std::time::Instant;
 
 use anyhow::{anyhow, Context};
 use async_trait::async_trait;
+use await_tree::InstrumentAwait;
 use futures::future::select;
-use futures::{StreamExt, TryStreamExt};
+use futures::TryStreamExt;
 use itertools::Itertools;
 use jni::JavaVM;
 use prost::Message;
@@ -41,8 +42,8 @@ use risingwave_pb::connector_service::sink_writer_stream_request::{
 use risingwave_pb::connector_service::{
     sink_coordinator_stream_request, sink_coordinator_stream_response, sink_writer_stream_response,
     PbSinkParam, SinkCoordinatorStreamRequest, SinkCoordinatorStreamResponse, SinkMetadata,
-    SinkPayloadFormat, SinkWriterStreamRequest, SinkWriterStreamResponse, TableSchema,
-    ValidateSinkRequest, ValidateSinkResponse,
+    SinkWriterStreamRequest, SinkWriterStreamResponse, TableSchema, ValidateSinkRequest,
+    ValidateSinkResponse,
 };
 use risingwave_rpc_client::error::RpcError;
 use risingwave_rpc_client::{
@@ -52,7 +53,7 @@ use risingwave_rpc_client::{
 use rw_futures_util::drop_either_future;
 use thiserror_ext::AsReport;
 use tokio::sync::mpsc;
-use tokio::sync::mpsc::{unbounded_channel, Receiver, Sender};
+use tokio::sync::mpsc::{unbounded_channel, Receiver};
 use tokio::task::spawn_blocking;
 use tokio_stream::wrappers::ReceiverStream;
 use tracing::warn;
@@ -67,7 +68,6 @@ use crate::sink::{
     DummySinkCommitCoordinator, LogSinker, Result, Sink, SinkCommitCoordinator, SinkError,
     SinkLogReader, SinkMetrics, SinkParam, SinkWriterParam,
 };
-use crate::ConnectorParams;
 
 macro_rules! def_remote_sink {
     () => {
@@ -81,7 +81,6 @@ macro_rules! def_remote_sink {
             { HttpJava, HttpJavaSink, "http" }
         }
     };
-    () => {};
     ({ $variant_name:ident, $sink_type_name:ident, $sink_name:expr }) => {
         #[derive(Debug)]
         pub struct $variant_name;
@@ -167,7 +166,7 @@ impl<R: RemoteSinkTrait> Sink for RemoteSink<R> {
 async fn validate_remote_sink(param: &SinkParam, sink_name: &str) -> ConnectorResult<()> {
     if sink_name == ElasticSearchSink::SINK_NAME
         && param.downstream_pk.len() > 1
-        && param.properties.get(ES_OPTION_DELIMITER).is_none()
+        && !param.properties.contains_key(ES_OPTION_DELIMITER)
     {
         bail!("Es sink only support single pk or pk with delimiter option");
     }
@@ -282,7 +281,7 @@ impl RemoteLogSinker {
             request_sender,
             response_stream,
         } = EmbeddedConnectorClient::new()?
-            .start_sink_writer_stream(payload_schema, sink_proto, SinkPayloadFormat::StreamChunk)
+            .start_sink_writer_stream(payload_schema, sink_proto)
             .await?;
 
         let sink_metrics = writer_param.sink_metrics;
@@ -302,7 +301,7 @@ impl RemoteLogSinker {
 
 #[async_trait]
 impl LogSinker for RemoteLogSinker {
-    async fn consume_log_and_sink(self, log_reader: &mut impl SinkLogReader) -> Result<()> {
+    async fn consume_log_and_sink(self, log_reader: &mut impl SinkLogReader) -> Result<!> {
         let mut request_tx = self.request_sender;
         let mut response_err_stream_rx = self.response_stream;
         let sink_metrics = self.sink_metrics;
@@ -311,7 +310,11 @@ impl LogSinker for RemoteLogSinker {
 
         let poll_response_stream = async move {
             loop {
-                let result = response_err_stream_rx.stream.try_next().await;
+                let result = response_err_stream_rx
+                    .stream
+                    .try_next()
+                    .instrument_await("log_sinker_wait_next_response")
+                    .await;
                 match result {
                     Ok(Some(response)) => {
                         response_tx.send(response).map_err(|err| {
@@ -325,7 +328,7 @@ impl LogSinker for RemoteLogSinker {
         };
 
         let poll_consume_log_and_sink = async move {
-            async fn truncate_matched_offset(
+            fn truncate_matched_offset(
                 queue: &mut VecDeque<(TruncateOffset, Option<Instant>)>,
                 persisted_offset: TruncateOffset,
                 log_reader: &mut impl SinkLogReader,
@@ -356,7 +359,7 @@ impl LogSinker for RemoteLogSinker {
                         .observe(start_time.elapsed().as_millis() as f64);
                 }
 
-                log_reader.truncate(persisted_offset).await?;
+                log_reader.truncate(persisted_offset)?;
                 Ok(())
             }
 
@@ -393,8 +396,7 @@ impl LogSinker for RemoteLogSinker {
                                     },
                                     log_reader,
                                     &sink_metrics,
-                                )
-                                .await?;
+                                )?;
                             }
                             SinkWriterStreamResponse {
                                 response:
@@ -413,8 +415,7 @@ impl LogSinker for RemoteLogSinker {
                                     TruncateOffset::Barrier { epoch },
                                     log_reader,
                                     &sink_metrics,
-                                )
-                                .await?;
+                                )?;
                             }
                             response => {
                                 return Err(SinkError::Remote(anyhow!(
@@ -445,6 +446,9 @@ impl LogSinker for RemoteLogSinker {
                                         batch_id: chunk_id as u64,
                                         chunk,
                                     })
+                                    .instrument_await(format!(
+                                        "log_sinker_send_chunk (chunk {chunk_id})"
+                                    ))
                                     .await?;
                                 prev_offset = Some(offset);
                                 sent_offset_queue
@@ -457,10 +461,20 @@ impl LogSinker for RemoteLogSinker {
                                 }
                                 let start_time = if is_checkpoint {
                                     let start_time = Instant::now();
-                                    request_tx.barrier(epoch, true).await?;
+                                    request_tx
+                                        .barrier(epoch, true)
+                                        .instrument_await(format!(
+                                            "log_sinker_commit_checkpoint (epoch {epoch})"
+                                        ))
+                                        .await?;
                                     Some(start_time)
                                 } else {
-                                    request_tx.barrier(epoch, false).await?;
+                                    request_tx
+                                        .barrier(epoch, false)
+                                        .instrument_await(format!(
+                                            "log_sinker_send_barrier (epoch {epoch})"
+                                        ))
+                                        .await?;
                                     None
                                 };
                                 prev_offset = Some(offset);
@@ -522,12 +536,8 @@ impl<R: RemoteSinkTrait> Sink for CoordinatedRemoteSink<R> {
                     "sink needs coordination should not have singleton input"
                 ))
             })?,
-            CoordinatedRemoteSinkWriter::new(
-                self.param.clone(),
-                writer_param.connector_params,
-                writer_param.sink_metrics.clone(),
-            )
-            .await?,
+            CoordinatedRemoteSinkWriter::new(self.param.clone(), writer_param.sink_metrics.clone())
+                .await?,
         )
         .await?
         .into_log_sinker(writer_param.sink_metrics))
@@ -539,7 +549,8 @@ impl<R: RemoteSinkTrait> Sink for CoordinatedRemoteSink<R> {
 }
 
 pub struct CoordinatedRemoteSinkWriter {
-    properties: HashMap<String, String>,
+    #[expect(dead_code)]
+    properties: BTreeMap<String, String>,
     epoch: Option<u64>,
     batch_id: u64,
     stream_handle: SinkWriterStreamHandle<JniSinkWriterStreamRequest>,
@@ -547,18 +558,10 @@ pub struct CoordinatedRemoteSinkWriter {
 }
 
 impl CoordinatedRemoteSinkWriter {
-    pub async fn new(
-        param: SinkParam,
-        connector_params: ConnectorParams,
-        sink_metrics: SinkMetrics,
-    ) -> Result<Self> {
+    pub async fn new(param: SinkParam, sink_metrics: SinkMetrics) -> Result<Self> {
         let sink_proto = param.to_proto();
         let stream_handle = EmbeddedConnectorClient::new()?
-            .start_sink_writer_stream(
-                sink_proto.table_schema.clone(),
-                sink_proto,
-                connector_params.sink_payload_format,
-            )
+            .start_sink_writer_stream(sink_proto.table_schema.clone(), sink_proto)
             .await?;
 
         Ok(Self {
@@ -570,11 +573,14 @@ impl CoordinatedRemoteSinkWriter {
         })
     }
 
+    #[cfg(test)]
     fn for_test(
         response_receiver: Receiver<ConnectorResult<SinkWriterStreamResponse>>,
-        request_sender: Sender<JniSinkWriterStreamRequest>,
+        request_sender: tokio::sync::mpsc::Sender<JniSinkWriterStreamRequest>,
     ) -> CoordinatedRemoteSinkWriter {
-        let properties = HashMap::from([("output.path".to_string(), "/tmp/rw".to_string())]);
+        use futures::StreamExt;
+
+        let properties = BTreeMap::from([("output.path".to_string(), "/tmp/rw".to_string())]);
 
         let stream_handle = SinkWriterStreamHandle::for_test(
             request_sender,
@@ -655,11 +661,7 @@ impl RemoteCoordinator {
             .start_sink_coordinator_stream(param.clone())
             .await?;
 
-        tracing::trace!(
-            "{:?} RemoteCoordinator started with properties: {:?}",
-            R::SINK_NAME,
-            &param.properties
-        );
+        tracing::trace!("{:?} RemoteCoordinator started", R::SINK_NAME,);
 
         Ok(RemoteCoordinator { stream_handle })
     }
@@ -692,13 +694,11 @@ impl EmbeddedConnectorClient {
         &self,
         payload_schema: Option<TableSchema>,
         sink_proto: PbSinkParam,
-        sink_payload_format: SinkPayloadFormat,
     ) -> Result<SinkWriterStreamHandle<JniSinkWriterStreamRequest>> {
         let (handle, first_rsp) = SinkWriterStreamHandle::initialize(
             SinkWriterStreamRequest {
                 request: Some(SinkRequest::Start(StartSink {
                     sink_param: Some(sink_proto),
-                    format: sink_payload_format as i32,
                     payload_schema,
                 })),
             },
diff --git a/src/connector/src/sink/snowflake.rs b/src/connector/src/sink/snowflake.rs
index e4dbbfa59f17b..f08519e3e8da6 100644
--- a/src/connector/src/sink/snowflake.rs
+++ b/src/connector/src/sink/snowflake.rs
@@ -12,17 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::fmt::Write;
 use std::sync::Arc;
 
-use anyhow::anyhow;
+use anyhow::{anyhow, Context};
 use async_trait::async_trait;
 use bytes::{Bytes, BytesMut};
 use risingwave_common::array::{Op, StreamChunk};
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::Schema;
-use risingwave_object_store::object::{ObjectStore, StreamingUploader};
+use risingwave_object_store::object::{ObjectStore, OpendalStreamingUploader, StreamingUploader};
 use serde::Deserialize;
 use serde_json::Value;
 use serde_with::serde_as;
@@ -39,7 +39,6 @@ use crate::sink::writer::SinkWriterExt;
 use crate::sink::{DummySinkCommitCoordinator, Result, Sink, SinkWriter, SinkWriterParam};
 
 pub const SNOWFLAKE_SINK: &str = "snowflake";
-const INITIAL_ROW_CAPACITY: usize = 1024;
 
 #[derive(Deserialize, Debug, Clone, WithOptions)]
 pub struct SnowflakeCommon {
@@ -108,7 +107,7 @@ pub struct SnowflakeConfig {
 }
 
 impl SnowflakeConfig {
-    pub fn from_hashmap(properties: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(properties: BTreeMap<String, String>) -> Result<Self> {
         let config =
             serde_json::from_value::<SnowflakeConfig>(serde_json::to_value(properties).unwrap())
                 .map_err(|e| SinkError::Config(anyhow!(e)))?;
@@ -155,7 +154,7 @@ impl TryFrom<SinkParam> for SnowflakeSink {
 
     fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
         let schema = param.schema();
-        let config = SnowflakeConfig::from_hashmap(param.properties)?;
+        let config = SnowflakeConfig::from_btreemap(param.properties)?;
         Ok(SnowflakeSink {
             config,
             schema,
@@ -166,9 +165,13 @@ impl TryFrom<SinkParam> for SnowflakeSink {
 }
 
 pub struct SnowflakeSinkWriter {
+    #[expect(dead_code)]
     config: SnowflakeConfig,
+    #[expect(dead_code)]
     schema: Schema,
+    #[expect(dead_code)]
     pk_indices: Vec<usize>,
+    #[expect(dead_code)]
     is_append_only: bool,
     /// the client used to send `insertFiles` post request
     http_client: SnowflakeHttpClient,
@@ -184,7 +187,7 @@ pub struct SnowflakeSinkWriter {
     /// note: the option here *implicitly* indicates whether we have at
     /// least call `streaming_upload` once during this epoch,
     /// which is mainly used to prevent uploading empty data.
-    streaming_uploader: Option<(Box<dyn StreamingUploader>, String)>,
+    streaming_uploader: Option<(OpendalStreamingUploader, String)>,
 }
 
 impl SnowflakeSinkWriter {
@@ -242,7 +245,7 @@ impl SnowflakeSinkWriter {
     /// and `streaming_upload` being called the first time.
     /// i.e., lazily initialization of the internal `streaming_uploader`.
     /// plus, this function is *pure*, the `&mut self` here is to make rustc (and tokio) happy.
-    async fn new_streaming_uploader(&mut self) -> Result<(Box<dyn StreamingUploader>, String)> {
+    async fn new_streaming_uploader(&mut self) -> Result<(OpendalStreamingUploader, String)> {
         let file_suffix = self.file_suffix();
         let path = generate_s3_file_name(self.s3_client.s3_path(), &file_suffix);
         let uploader = self
@@ -250,13 +253,13 @@ impl SnowflakeSinkWriter {
             .opendal_s3_engine
             .streaming_upload(&path)
             .await
-            .map_err(|err| {
-                SinkError::Snowflake(format!(
-                    "failed to create the streaming uploader of opendal s3 engine for epoch {}, error: {}",
-                    self.epoch,
-                    err
-                ))
-            })?;
+            .with_context(|| {
+                format!(
+                    "failed to create the streaming uploader of opendal s3 engine for epoch {}",
+                    self.epoch
+                )
+            })
+            .map_err(SinkError::Snowflake)?;
         Ok((uploader, file_suffix))
     }
 
@@ -276,12 +279,8 @@ impl SnowflakeSinkWriter {
         uploader
             .write_bytes(data)
             .await
-            .map_err(|err| {
-                SinkError::Snowflake(format!(
-                    "failed to write bytes when streaming uploading to s3 for snowflake sink, error: {}",
-                    err
-                ))
-            })?;
+            .context("failed to write bytes when streaming uploading to s3")
+            .map_err(SinkError::Snowflake)?;
         Ok(())
     }
 
@@ -293,12 +292,11 @@ impl SnowflakeSinkWriter {
             // there is no data to be uploaded for this epoch
             return Ok(None);
         };
-        uploader.finish().await.map_err(|err| {
-            SinkError::Snowflake(format!(
-                "failed to finish streaming upload to s3 for snowflake sink, error: {}",
-                err
-            ))
-        })?;
+        uploader
+            .finish()
+            .await
+            .context("failed to finish streaming upload to s3")
+            .map_err(SinkError::Snowflake)?;
         Ok(Some(file_suffix))
     }
 
@@ -315,12 +313,7 @@ impl SnowflakeSinkWriter {
                 "{}",
                 Value::Object(self.row_encoder.encode(row)?)
             )
-            .map_err(|err| {
-                SinkError::Snowflake(format!(
-                    "failed to write json object to `row_buf`, error: {}",
-                    err
-                ))
-            })?;
+            .unwrap(); // write to a `BytesMut` should never fail
         }
 
         // streaming upload in a chunk-by-chunk manner
diff --git a/src/connector/src/sink/snowflake_connector.rs b/src/connector/src/sink/snowflake_connector.rs
index 432d222a8b426..3adaa43bb5aa6 100644
--- a/src/connector/src/sink/snowflake_connector.rs
+++ b/src/connector/src/sink/snowflake_connector.rs
@@ -13,14 +13,14 @@
 // limitations under the License.
 
 use std::collections::HashMap;
+use std::sync::Arc;
 use std::time::{SystemTime, UNIX_EPOCH};
 
+use anyhow::{anyhow, Context};
 use jsonwebtoken::{encode, Algorithm, EncodingKey, Header};
 use reqwest::{header, Client, RequestBuilder, StatusCode};
-use risingwave_common::config::ObjectStoreConfig;
 use risingwave_object_store::object::*;
 use serde::{Deserialize, Serialize};
-use thiserror_ext::AsReport;
 
 use super::doris_starrocks_connector::POOL_IDLE_TIMEOUT;
 use super::{Result, SinkError};
@@ -55,6 +55,7 @@ pub struct SnowflakeHttpClient {
     account: String,
     user: String,
     private_key: String,
+    #[expect(dead_code)]
     header: HashMap<String, String>,
     s3_path: Option<String>,
 }
@@ -126,16 +127,12 @@ impl SnowflakeHttpClient {
         let jwt_token = encode(
             &header,
             &claims,
-            &EncodingKey::from_rsa_pem(self.private_key.as_ref()).map_err(|err| {
-                SinkError::Snowflake(format!(
-                    "failed to encode from provided rsa pem key, error: {}",
-                    err
-                ))
-            })?,
+            &EncodingKey::from_rsa_pem(self.private_key.as_ref())
+                .context("failed to encode from provided rsa pem key")
+                .map_err(SinkError::Snowflake)?,
         )
-        .map_err(|err| {
-            SinkError::Snowflake(format!("failed to encode jwt_token, error: {}", err))
-        })?;
+        .context("failed to encode jwt_token")
+        .map_err(SinkError::Snowflake)?;
         Ok(jwt_token)
     }
 
@@ -167,10 +164,10 @@ impl SnowflakeHttpClient {
         let response = builder
             .send()
             .await
-            .map_err(|err| SinkError::Snowflake(err.to_report_string()))?;
+            .map_err(|err| SinkError::Snowflake(anyhow!(err)))?;
 
         if response.status() != StatusCode::OK {
-            return Err(SinkError::Snowflake(format!(
+            return Err(SinkError::Snowflake(anyhow!(
                 "failed to make http request, error code: {}\ndetailed response: {:#?}",
                 response.status(),
                 response,
@@ -183,6 +180,7 @@ impl SnowflakeHttpClient {
 
 /// todo: refactor this part after s3 sink is available
 pub struct SnowflakeS3Client {
+    #[expect(dead_code)]
     s3_bucket: String,
     s3_path: Option<String>,
     pub opendal_s3_engine: OpendalObjectStore,
@@ -196,23 +194,20 @@ impl SnowflakeS3Client {
         aws_secret_access_key: String,
         aws_region: String,
     ) -> Result<Self> {
+        // FIXME: we should use the `ObjectStoreConfig` instead of default
         // just use default configuration here for opendal s3 engine
         let config = ObjectStoreConfig::default();
 
         // create the s3 engine for streaming upload to the intermediate s3 bucket
         let opendal_s3_engine = OpendalObjectStore::new_s3_engine_with_credentials(
             &s3_bucket,
-            config,
+            Arc::new(config),
             &aws_access_key_id,
             &aws_secret_access_key,
             &aws_region,
         )
-        .map_err(|err| {
-            SinkError::Snowflake(format!(
-                "failed to create opendal s3 engine, error: {}",
-                err
-            ))
-        })?;
+        .context("failed to create opendal s3 engine")
+        .map_err(SinkError::Snowflake)?;
 
         Ok(Self {
             s3_bucket,
diff --git a/src/connector/src/sink/sqlserver.rs b/src/connector/src/sink/sqlserver.rs
new file mode 100644
index 0000000000000..acdad3c47627e
--- /dev/null
+++ b/src/connector/src/sink/sqlserver.rs
@@ -0,0 +1,649 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{BTreeMap, HashMap};
+use std::sync::Arc;
+
+use anyhow::anyhow;
+use async_trait::async_trait;
+use risingwave_common::array::{Op, RowRef, StreamChunk};
+use risingwave_common::buffer::Bitmap;
+use risingwave_common::catalog::Schema;
+use risingwave_common::row::{OwnedRow, Row};
+use risingwave_common::types::{DataType, Decimal};
+use serde_derive::Deserialize;
+use serde_with::{serde_as, DisplayFromStr};
+use simd_json::prelude::ArrayTrait;
+use tiberius::numeric::Numeric;
+use tiberius::{AuthMethod, Client, ColumnData, Config, Query};
+use tokio::net::TcpStream;
+use tokio_util::compat::TokioAsyncWriteCompatExt;
+use with_options::WithOptions;
+
+use super::{SinkError, SINK_TYPE_APPEND_ONLY, SINK_TYPE_OPTION, SINK_TYPE_UPSERT};
+use crate::sink::writer::{LogSinkerOf, SinkWriter, SinkWriterExt};
+use crate::sink::{DummySinkCommitCoordinator, Result, Sink, SinkParam, SinkWriterParam};
+
+pub const SQLSERVER_SINK: &str = "sqlserver";
+
+fn default_max_batch_rows() -> usize {
+    1024
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, WithOptions)]
+pub struct SqlServerConfig {
+    #[serde(rename = "sqlserver.host")]
+    pub host: String,
+    #[serde(rename = "sqlserver.port")]
+    #[serde_as(as = "DisplayFromStr")]
+    pub port: u16,
+    #[serde(rename = "sqlserver.user")]
+    pub user: String,
+    #[serde(rename = "sqlserver.password")]
+    pub password: String,
+    #[serde(rename = "sqlserver.database")]
+    pub database: String,
+    #[serde(rename = "sqlserver.table")]
+    pub table: String,
+    #[serde(
+        rename = "sqlserver.max_batch_rows",
+        default = "default_max_batch_rows"
+    )]
+    #[serde_as(as = "DisplayFromStr")]
+    pub max_batch_rows: usize,
+    pub r#type: String, // accept "append-only" or "upsert"
+}
+
+impl SqlServerConfig {
+    pub fn from_btreemap(properties: BTreeMap<String, String>) -> Result<Self> {
+        let config =
+            serde_json::from_value::<SqlServerConfig>(serde_json::to_value(properties).unwrap())
+                .map_err(|e| SinkError::Config(anyhow!(e)))?;
+        if config.r#type != SINK_TYPE_APPEND_ONLY && config.r#type != SINK_TYPE_UPSERT {
+            return Err(SinkError::Config(anyhow!(
+                "`{}` must be {}, or {}",
+                SINK_TYPE_OPTION,
+                SINK_TYPE_APPEND_ONLY,
+                SINK_TYPE_UPSERT
+            )));
+        }
+        Ok(config)
+    }
+}
+
+#[derive(Debug)]
+pub struct SqlServerSink {
+    pub config: SqlServerConfig,
+    schema: Schema,
+    pk_indices: Vec<usize>,
+    is_append_only: bool,
+}
+
+impl SqlServerSink {
+    pub fn new(
+        mut config: SqlServerConfig,
+        schema: Schema,
+        pk_indices: Vec<usize>,
+        is_append_only: bool,
+    ) -> Result<Self> {
+        // Rewrite config because tiberius allows a maximum of 2100 params in one query request.
+        const TIBERIUS_PARAM_MAX: usize = 2000;
+        let params_per_op = schema.fields().len();
+        let tiberius_max_batch_rows = if params_per_op == 0 {
+            config.max_batch_rows
+        } else {
+            ((TIBERIUS_PARAM_MAX as f64 / params_per_op as f64).floor()) as usize
+        };
+        if tiberius_max_batch_rows == 0 {
+            return Err(SinkError::SqlServer(anyhow!(format!(
+                "too many column {}",
+                params_per_op
+            ))));
+        }
+        config.max_batch_rows = std::cmp::min(config.max_batch_rows, tiberius_max_batch_rows);
+        Ok(Self {
+            config,
+            schema,
+            pk_indices,
+            is_append_only,
+        })
+    }
+}
+
+impl TryFrom<SinkParam> for SqlServerSink {
+    type Error = SinkError;
+
+    fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
+        let schema = param.schema();
+        let config = SqlServerConfig::from_btreemap(param.properties)?;
+        SqlServerSink::new(
+            config,
+            schema,
+            param.downstream_pk,
+            param.sink_type.is_append_only(),
+        )
+    }
+}
+
+impl Sink for SqlServerSink {
+    type Coordinator = DummySinkCommitCoordinator;
+    type LogSinker = LogSinkerOf<SqlServerSinkWriter>;
+
+    const SINK_NAME: &'static str = SQLSERVER_SINK;
+
+    async fn validate(&self) -> Result<()> {
+        if !self.is_append_only && self.pk_indices.is_empty() {
+            return Err(SinkError::Config(anyhow!(
+                "Primary key not defined for upsert SQL Server sink (please define in `primary_key` field)")));
+        }
+
+        for f in self.schema.fields() {
+            check_data_type_compatibility(&f.data_type)?;
+        }
+
+        // Query table metadata from SQL Server.
+        let mut sql_server_table_metadata = HashMap::new();
+        let mut sql_client = SqlClient::new(&self.config).await?;
+        let query_table_metadata_error = || {
+            SinkError::SqlServer(anyhow!(format!(
+                "SQL Server table {} metadata error",
+                self.config.table
+            )))
+        };
+        static QUERY_TABLE_METADATA: &str = r#"
+SELECT
+    col.name AS ColumnName,
+    pk.index_id AS PkIndex
+FROM
+    sys.columns col
+LEFT JOIN
+    sys.index_columns ic ON ic.object_id = col.object_id AND ic.column_id = col.column_id
+LEFT JOIN
+    sys.indexes pk ON pk.object_id = col.object_id AND pk.index_id = ic.index_id AND pk.is_primary_key = 1
+WHERE
+    col.object_id = OBJECT_ID(@P1)
+ORDER BY
+    col.column_id;"#;
+        let rows = sql_client
+            .client
+            .query(QUERY_TABLE_METADATA, &[&self.config.table])
+            .await?
+            .into_results()
+            .await?;
+        for row in rows.into_iter().flatten() {
+            let mut iter = row.into_iter();
+            let ColumnData::String(Some(col_name)) =
+                iter.next().ok_or_else(query_table_metadata_error)?
+            else {
+                return Err(query_table_metadata_error());
+            };
+            let ColumnData::I32(col_pk_index) =
+                iter.next().ok_or_else(query_table_metadata_error)?
+            else {
+                return Err(query_table_metadata_error());
+            };
+            sql_server_table_metadata.insert(col_name.into_owned(), col_pk_index.is_some());
+        }
+
+        // Validate Column name and Primary Key
+        for (idx, col) in self.schema.fields().iter().enumerate() {
+            let rw_is_pk = self.pk_indices.contains(&idx);
+            match sql_server_table_metadata.get(&col.name) {
+                None => {
+                    return Err(SinkError::SqlServer(anyhow!(format!(
+                        "column {} not found in the downstream SQL Server table {}",
+                        col.name, self.config.table
+                    ))));
+                }
+                Some(sql_server_is_pk) => {
+                    if self.is_append_only {
+                        continue;
+                    }
+                    if rw_is_pk && !*sql_server_is_pk {
+                        return Err(SinkError::SqlServer(anyhow!(format!(
+                            "column {} specified in primary_key mismatches with the downstream SQL Server table {} PK",
+                            col.name, self.config.table,
+                        ))));
+                    }
+                    if !rw_is_pk && *sql_server_is_pk {
+                        return Err(SinkError::SqlServer(anyhow!(format!(
+                            "column {} unspecified in primary_key mismatches with the downstream SQL Server table {} PK",
+                            col.name, self.config.table,
+                        ))));
+                    }
+                }
+            }
+        }
+
+        if !self.is_append_only {
+            let sql_server_pk_count = sql_server_table_metadata
+                .values()
+                .filter(|is_pk| **is_pk)
+                .count();
+            if sql_server_pk_count != self.pk_indices.len() {
+                return Err(SinkError::SqlServer(anyhow!(format!(
+                    "primary key does not match between RisingWave sink ({}) and SQL Server table {} ({})",
+                    self.pk_indices.len(),
+                    self.config.table,
+                    sql_server_pk_count,
+                ))));
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn new_log_sinker(&self, writer_param: SinkWriterParam) -> Result<Self::LogSinker> {
+        Ok(SqlServerSinkWriter::new(
+            self.config.clone(),
+            self.schema.clone(),
+            self.pk_indices.clone(),
+            self.is_append_only,
+        )
+        .await?
+        .into_log_sinker(writer_param.sink_metrics))
+    }
+}
+
+enum SqlOp {
+    Insert(OwnedRow),
+    Merge(OwnedRow),
+    Delete(OwnedRow),
+}
+
+pub struct SqlServerSinkWriter {
+    config: SqlServerConfig,
+    schema: Schema,
+    pk_indices: Vec<usize>,
+    is_append_only: bool,
+    sql_client: SqlClient,
+    ops: Vec<SqlOp>,
+}
+
+impl SqlServerSinkWriter {
+    async fn new(
+        config: SqlServerConfig,
+        schema: Schema,
+        pk_indices: Vec<usize>,
+        is_append_only: bool,
+    ) -> Result<Self> {
+        let sql_client = SqlClient::new(&config).await?;
+        let writer = Self {
+            config,
+            schema,
+            pk_indices,
+            is_append_only,
+            sql_client,
+            ops: vec![],
+        };
+        Ok(writer)
+    }
+
+    async fn delete_one(&mut self, row: RowRef<'_>) -> Result<()> {
+        if self.ops.len() + 1 >= self.config.max_batch_rows {
+            self.flush().await?;
+        }
+        self.ops.push(SqlOp::Delete(row.into_owned_row()));
+        Ok(())
+    }
+
+    async fn upsert_one(&mut self, row: RowRef<'_>) -> Result<()> {
+        if self.ops.len() + 1 >= self.config.max_batch_rows {
+            self.flush().await?;
+        }
+        self.ops.push(SqlOp::Merge(row.into_owned_row()));
+        Ok(())
+    }
+
+    async fn insert_one(&mut self, row: RowRef<'_>) -> Result<()> {
+        if self.ops.len() + 1 >= self.config.max_batch_rows {
+            self.flush().await?;
+        }
+        self.ops.push(SqlOp::Insert(row.into_owned_row()));
+        Ok(())
+    }
+
+    async fn flush(&mut self) -> Result<()> {
+        use std::fmt::Write;
+        if self.ops.is_empty() {
+            return Ok(());
+        }
+        let mut query_str = String::new();
+        let col_num = self.schema.fields.len();
+        let mut next_param_id = 1;
+        let non_pk_col_indices = (0..col_num)
+            .filter(|idx| !self.pk_indices.contains(idx))
+            .collect::<Vec<usize>>();
+        let all_col_names = self
+            .schema
+            .fields
+            .iter()
+            .map(|f| format!("[{}]", f.name))
+            .collect::<Vec<_>>()
+            .join(",");
+        let all_source_col_names = self
+            .schema
+            .fields
+            .iter()
+            .map(|f| format!("[SOURCE].[{}]", f.name))
+            .collect::<Vec<_>>()
+            .join(",");
+        let pk_match = self
+            .pk_indices
+            .iter()
+            .map(|idx| {
+                format!(
+                    "[SOURCE].[{}]=[TARGET].[{}]",
+                    &self.schema[*idx].name, &self.schema[*idx].name
+                )
+            })
+            .collect::<Vec<_>>()
+            .join(" AND ");
+        let param_placeholders = |param_id: &mut usize| {
+            let params = (*param_id..(*param_id + col_num))
+                .map(|i| format!("@P{}", i))
+                .collect::<Vec<_>>()
+                .join(",");
+            *param_id += col_num;
+            params
+        };
+        let set_all_source_col = non_pk_col_indices
+            .iter()
+            .map(|idx| {
+                format!(
+                    "[{}]=[SOURCE].[{}]",
+                    &self.schema[*idx].name, &self.schema[*idx].name
+                )
+            })
+            .collect::<Vec<_>>()
+            .join(",");
+        // TODO: avoid repeating the SQL
+        for op in &self.ops {
+            match op {
+                SqlOp::Insert(_) => {
+                    write!(
+                        &mut query_str,
+                        "INSERT INTO [{}] ({}) VALUES ({});",
+                        self.config.table,
+                        all_col_names,
+                        param_placeholders(&mut next_param_id),
+                    )
+                    .unwrap();
+                }
+                SqlOp::Merge(_) => {
+                    write!(
+                        &mut query_str,
+                        r#"MERGE [{}] AS [TARGET]
+                        USING (VALUES ({})) AS [SOURCE] ({})
+                        ON {}
+                        WHEN MATCHED THEN UPDATE SET {}
+                        WHEN NOT MATCHED THEN INSERT ({}) VALUES ({});"#,
+                        self.config.table,
+                        param_placeholders(&mut next_param_id),
+                        all_col_names,
+                        pk_match,
+                        set_all_source_col,
+                        all_col_names,
+                        all_source_col_names,
+                    )
+                    .unwrap();
+                }
+                SqlOp::Delete(_) => {
+                    write!(
+                        &mut query_str,
+                        r#"DELETE FROM [{}] WHERE {};"#,
+                        self.config.table,
+                        self.pk_indices
+                            .iter()
+                            .map(|idx| {
+                                let condition =
+                                    format!("[{}]=@P{}", self.schema[*idx].name, next_param_id);
+                                next_param_id += 1;
+                                condition
+                            })
+                            .collect::<Vec<_>>()
+                            .join(" AND "),
+                    )
+                    .unwrap();
+                }
+            }
+        }
+
+        let mut query = Query::new(query_str);
+        for op in self.ops.drain(..) {
+            match op {
+                SqlOp::Insert(row) => {
+                    bind_params(&mut query, row, &self.schema, 0..col_num)?;
+                }
+                SqlOp::Merge(row) => {
+                    bind_params(&mut query, row, &self.schema, 0..col_num)?;
+                }
+                SqlOp::Delete(row) => {
+                    bind_params(
+                        &mut query,
+                        row,
+                        &self.schema,
+                        self.pk_indices.iter().copied(),
+                    )?;
+                }
+            }
+        }
+        query.execute(&mut self.sql_client.client).await?;
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl SinkWriter for SqlServerSinkWriter {
+    async fn begin_epoch(&mut self, _epoch: u64) -> Result<()> {
+        Ok(())
+    }
+
+    async fn write_batch(&mut self, chunk: StreamChunk) -> Result<()> {
+        for (op, row) in chunk.rows() {
+            match op {
+                Op::Insert => {
+                    if self.is_append_only {
+                        self.insert_one(row).await?;
+                    } else {
+                        self.upsert_one(row).await?;
+                    }
+                }
+                Op::UpdateInsert => {
+                    debug_assert!(!self.is_append_only);
+                    self.upsert_one(row).await?;
+                }
+                Op::Delete => {
+                    debug_assert!(!self.is_append_only);
+                    self.delete_one(row).await?;
+                }
+                Op::UpdateDelete => {}
+            }
+        }
+        Ok(())
+    }
+
+    async fn barrier(&mut self, is_checkpoint: bool) -> Result<Self::CommitMetadata> {
+        if is_checkpoint {
+            self.flush().await?;
+        }
+        Ok(())
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        Ok(())
+    }
+
+    async fn update_vnode_bitmap(&mut self, _vnode_bitmap: Arc<Bitmap>) -> Result<()> {
+        Ok(())
+    }
+}
+
+struct SqlClient {
+    client: Client<tokio_util::compat::Compat<TcpStream>>,
+}
+
+impl SqlClient {
+    async fn new(msconfig: &SqlServerConfig) -> Result<Self> {
+        let mut config = Config::new();
+        config.host(&msconfig.host);
+        config.port(msconfig.port);
+        config.authentication(AuthMethod::sql_server(&msconfig.user, &msconfig.password));
+        config.database(&msconfig.database);
+        config.trust_cert();
+
+        let tcp = TcpStream::connect(config.get_addr()).await.unwrap();
+        tcp.set_nodelay(true).unwrap();
+        let client = Client::connect(config, tcp.compat_write()).await?;
+        Ok(Self { client })
+    }
+}
+
+fn bind_params(
+    query: &mut Query<'_>,
+    row: impl Row,
+    schema: &Schema,
+    col_indices: impl Iterator<Item = usize>,
+) -> Result<()> {
+    use risingwave_common::types::ScalarRefImpl;
+    for col_idx in col_indices {
+        match row.datum_at(col_idx) {
+            Some(data_ref) => match data_ref {
+                ScalarRefImpl::Int16(v) => query.bind(v),
+                ScalarRefImpl::Int32(v) => query.bind(v),
+                ScalarRefImpl::Int64(v) => query.bind(v),
+                ScalarRefImpl::Float32(v) => query.bind(v.into_inner()),
+                ScalarRefImpl::Float64(v) => query.bind(v.into_inner()),
+                ScalarRefImpl::Utf8(v) => query.bind(v.to_owned()),
+                ScalarRefImpl::Bool(v) => query.bind(v),
+                ScalarRefImpl::Decimal(v) => match v {
+                    Decimal::Normalized(d) => {
+                        query.bind(decimal_to_sql(&d));
+                    }
+                    Decimal::NaN | Decimal::PositiveInf | Decimal::NegativeInf => {
+                        tracing::warn!(
+                            "Inf, -Inf, Nan in RisingWave decimal is converted into SQL Server null!"
+                        );
+                        query.bind(None as Option<Numeric>);
+                    }
+                },
+                ScalarRefImpl::Date(v) => query.bind(v.0),
+                ScalarRefImpl::Timestamp(v) => query.bind(v.0),
+                ScalarRefImpl::Timestamptz(v) => query.bind(v.timestamp_micros()),
+                ScalarRefImpl::Time(v) => query.bind(v.0),
+                ScalarRefImpl::Bytea(v) => query.bind(v.to_vec()),
+                ScalarRefImpl::Interval(_) => return Err(data_type_not_supported("Interval")),
+                ScalarRefImpl::Jsonb(_) => return Err(data_type_not_supported("Jsonb")),
+                ScalarRefImpl::Struct(_) => return Err(data_type_not_supported("Struct")),
+                ScalarRefImpl::List(_) => return Err(data_type_not_supported("List")),
+                ScalarRefImpl::Int256(_) => return Err(data_type_not_supported("Int256")),
+                ScalarRefImpl::Serial(_) => return Err(data_type_not_supported("Serial")),
+            },
+            None => match schema[col_idx].data_type {
+                DataType::Boolean => {
+                    query.bind(None as Option<bool>);
+                }
+                DataType::Int16 => {
+                    query.bind(None as Option<i16>);
+                }
+                DataType::Int32 => {
+                    query.bind(None as Option<i32>);
+                }
+                DataType::Int64 => {
+                    query.bind(None as Option<i64>);
+                }
+                DataType::Float32 => {
+                    query.bind(None as Option<f32>);
+                }
+                DataType::Float64 => {
+                    query.bind(None as Option<f64>);
+                }
+                DataType::Decimal => {
+                    query.bind(None as Option<Numeric>);
+                }
+                DataType::Date => {
+                    query.bind(None as Option<chrono::NaiveDate>);
+                }
+                DataType::Time => {
+                    query.bind(None as Option<chrono::NaiveTime>);
+                }
+                DataType::Timestamp => {
+                    query.bind(None as Option<chrono::NaiveDateTime>);
+                }
+                DataType::Timestamptz => {
+                    query.bind(None as Option<i64>);
+                }
+                DataType::Varchar => {
+                    query.bind(None as Option<String>);
+                }
+                DataType::Bytea => {
+                    query.bind(None as Option<Vec<u8>>);
+                }
+                DataType::Interval => return Err(data_type_not_supported("Interval")),
+                DataType::Struct(_) => return Err(data_type_not_supported("Struct")),
+                DataType::List(_) => return Err(data_type_not_supported("List")),
+                DataType::Jsonb => return Err(data_type_not_supported("Jsonb")),
+                DataType::Serial => return Err(data_type_not_supported("Serial")),
+                DataType::Int256 => return Err(data_type_not_supported("Int256")),
+            },
+        };
+    }
+    Ok(())
+}
+
+fn data_type_not_supported(data_type_name: &str) -> SinkError {
+    SinkError::SqlServer(anyhow!(format!(
+        "{data_type_name} is not supported in SQL Server"
+    )))
+}
+
+fn check_data_type_compatibility(data_type: &DataType) -> Result<()> {
+    match data_type {
+        DataType::Boolean
+        | DataType::Int16
+        | DataType::Int32
+        | DataType::Int64
+        | DataType::Float32
+        | DataType::Float64
+        | DataType::Decimal
+        | DataType::Date
+        | DataType::Varchar
+        | DataType::Time
+        | DataType::Timestamp
+        | DataType::Timestamptz
+        | DataType::Bytea => Ok(()),
+        DataType::Interval => Err(data_type_not_supported("Interval")),
+        DataType::Struct(_) => Err(data_type_not_supported("Struct")),
+        DataType::List(_) => Err(data_type_not_supported("List")),
+        DataType::Jsonb => Err(data_type_not_supported("Jsonb")),
+        DataType::Serial => Err(data_type_not_supported("Serial")),
+        DataType::Int256 => Err(data_type_not_supported("Int256")),
+    }
+}
+
+/// The implementation is copied from tiberius crate.
+fn decimal_to_sql(decimal: &rust_decimal::Decimal) -> Numeric {
+    let unpacked = decimal.unpack();
+
+    let mut value = (((unpacked.hi as u128) << 64)
+        + ((unpacked.mid as u128) << 32)
+        + unpacked.lo as u128) as i128;
+
+    if decimal.is_sign_negative() {
+        value = -value;
+    }
+
+    Numeric::new_with_scale(value, decimal.scale() as u8)
+}
diff --git a/src/connector/src/sink/starrocks.rs b/src/connector/src/sink/starrocks.rs
index 994e4e43d8172..bace71cd59e2c 100644
--- a/src/connector/src/sink/starrocks.rs
+++ b/src/connector/src/sink/starrocks.rs
@@ -12,7 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
+use std::num::NonZeroU64;
 use std::sync::Arc;
 
 use anyhow::anyhow;
@@ -23,28 +24,44 @@ use mysql_async::Opts;
 use risingwave_common::array::{Op, StreamChunk};
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::Schema;
+use risingwave_common::session_config::sink_decouple::SinkDecouple;
 use risingwave_common::types::DataType;
+use risingwave_pb::connector_service::sink_metadata::Metadata::Serialized;
+use risingwave_pb::connector_service::sink_metadata::SerializedMetadata;
+use risingwave_pb::connector_service::SinkMetadata;
 use serde::Deserialize;
 use serde_derive::Serialize;
 use serde_json::Value;
-use serde_with::serde_as;
+use serde_with::{serde_as, DisplayFromStr};
 use thiserror_ext::AsReport;
+use tokio::task::JoinHandle;
+use url::form_urlencoded;
 use with_options::WithOptions;
 
 use super::doris_starrocks_connector::{
-    HeaderBuilder, InserterInner, InserterInnerBuilder, DORIS_SUCCESS_STATUS, STARROCKS_DELETE_SIGN,
+    HeaderBuilder, InserterInner, StarrocksTxnRequestBuilder, STARROCKS_DELETE_SIGN,
+    STARROCKS_SUCCESS_STATUS,
 };
 use super::encoder::{JsonEncoder, RowEncoder};
-use super::writer::LogSinkerOf;
-use super::{SinkError, SinkParam, SINK_TYPE_APPEND_ONLY, SINK_TYPE_OPTION, SINK_TYPE_UPSERT};
-use crate::sink::writer::SinkWriterExt;
-use crate::sink::{DummySinkCommitCoordinator, Result, Sink, SinkWriter, SinkWriterParam};
+use super::{
+    SinkCommitCoordinator, SinkError, SinkParam, SINK_TYPE_APPEND_ONLY, SINK_TYPE_OPTION,
+    SINK_TYPE_UPSERT,
+};
+use crate::deserialize_optional_u64_from_string;
+use crate::sink::catalog::desc::SinkDesc;
+use crate::sink::coordinate::CoordinatedSinkWriter;
+use crate::sink::decouple_checkpoint_log_sink::DecoupleCheckpointLogSinkerOf;
+use crate::sink::{Result, Sink, SinkWriter, SinkWriterParam};
 
 pub const STARROCKS_SINK: &str = "starrocks";
 const STARROCK_MYSQL_PREFER_SOCKET: &str = "false";
 const STARROCK_MYSQL_MAX_ALLOWED_PACKET: usize = 1024;
 const STARROCK_MYSQL_WAIT_TIMEOUT: usize = 28800;
 
+const fn _default_stream_load_http_timeout_ms() -> u64 {
+    30 * 1000
+}
+
 #[derive(Deserialize, Debug, Clone, WithOptions)]
 pub struct StarrocksCommon {
     /// The `StarRocks` host address.
@@ -68,8 +85,6 @@ pub struct StarrocksCommon {
     /// The `StarRocks` table you want to sink data to.
     #[serde(rename = "starrocks.table")]
     pub table: String,
-    #[serde(rename = "starrocks.partial_update")]
-    pub partial_update: Option<String>,
 }
 
 #[serde_as]
@@ -78,10 +93,31 @@ pub struct StarrocksConfig {
     #[serde(flatten)]
     pub common: StarrocksCommon,
 
+    /// The timeout in milliseconds for stream load http request, defaults to 10 seconds.
+    #[serde(
+        rename = "starrocks.stream_load.http.timeout.ms",
+        default = "_default_stream_load_http_timeout_ms"
+    )]
+    #[serde_as(as = "DisplayFromStr")]
+    pub stream_load_http_timeout_ms: u64,
+
+    /// Set this option to a positive integer n, RisingWave will try to commit data
+    /// to Starrocks at every n checkpoints by leveraging the
+    /// [StreamLoad Transaction API](https://docs.starrocks.io/docs/loading/Stream_Load_transaction_interface/),
+    /// also, in this time, the `sink_decouple` option should be enabled as well.
+    /// Defaults to 1 if commit_checkpoint_interval <= 0
+    #[serde(default, deserialize_with = "deserialize_optional_u64_from_string")]
+    pub commit_checkpoint_interval: Option<u64>,
+
+    /// Enable partial update
+    #[serde(rename = "starrocks.partial_update")]
+    pub partial_update: Option<String>,
+
     pub r#type: String, // accept "append-only" or "upsert"
 }
+
 impl StarrocksConfig {
-    pub fn from_hashmap(properties: HashMap<String, String>) -> Result<Self> {
+    pub fn from_btreemap(properties: BTreeMap<String, String>) -> Result<Self> {
         let config =
             serde_json::from_value::<StarrocksConfig>(serde_json::to_value(properties).unwrap())
                 .map_err(|e| SinkError::Config(anyhow!(e)))?;
@@ -93,12 +129,18 @@ impl StarrocksConfig {
                 SINK_TYPE_UPSERT
             )));
         }
+        if config.commit_checkpoint_interval == Some(0) {
+            return Err(SinkError::Config(anyhow!(
+                "commit_checkpoint_interval must be greater than 0"
+            )));
+        }
         Ok(config)
     }
 }
 
 #[derive(Debug)]
 pub struct StarrocksSink {
+    param: SinkParam,
     pub config: StarrocksConfig,
     schema: Schema,
     pk_indices: Vec<usize>,
@@ -106,13 +148,11 @@ pub struct StarrocksSink {
 }
 
 impl StarrocksSink {
-    pub fn new(
-        config: StarrocksConfig,
-        schema: Schema,
-        pk_indices: Vec<usize>,
-        is_append_only: bool,
-    ) -> Result<Self> {
+    pub fn new(param: SinkParam, config: StarrocksConfig, schema: Schema) -> Result<Self> {
+        let pk_indices = param.downstream_pk.clone();
+        let is_append_only = param.sink_type.is_append_only();
         Ok(Self {
+            param,
             config,
             schema,
             pk_indices,
@@ -211,19 +251,33 @@ impl StarrocksSink {
 }
 
 impl Sink for StarrocksSink {
-    type Coordinator = DummySinkCommitCoordinator;
-    type LogSinker = LogSinkerOf<StarrocksSinkWriter>;
+    type Coordinator = StarrocksSinkCommitter;
+    type LogSinker = DecoupleCheckpointLogSinkerOf<CoordinatedSinkWriter<StarrocksSinkWriter>>;
 
     const SINK_NAME: &'static str = STARROCKS_SINK;
 
-    async fn new_log_sinker(&self, writer_param: SinkWriterParam) -> Result<Self::LogSinker> {
-        Ok(StarrocksSinkWriter::new(
-            self.config.clone(),
-            self.schema.clone(),
-            self.pk_indices.clone(),
-            self.is_append_only,
-        )?
-        .into_log_sinker(writer_param.sink_metrics))
+    fn is_sink_decouple(desc: &SinkDesc, user_specified: &SinkDecouple) -> Result<bool> {
+        let config_decouple = if let Some(interval) =
+            desc.properties.get("commit_checkpoint_interval")
+            && interval.parse::<u64>().unwrap_or(0) > 1
+        {
+            true
+        } else {
+            false
+        };
+
+        match user_specified {
+            SinkDecouple::Default => Ok(config_decouple),
+            SinkDecouple::Disable => {
+                if config_decouple {
+                    return Err(SinkError::Config(anyhow!(
+                        "config conflict: StarRocks config `commit_checkpoint_interval` larger than 1 means that sink decouple must be enabled, but session config sink_decouple is disabled"
+                    )));
+                }
+                Ok(false)
+            }
+            SinkDecouple::Enable => Ok(true),
+        }
     }
 
     async fn validate(&self) -> Result<()> {
@@ -263,16 +317,80 @@ impl Sink for StarrocksSink {
         self.check_column_name_and_type(starrocks_columns_desc)?;
         Ok(())
     }
+
+    async fn new_log_sinker(&self, writer_param: SinkWriterParam) -> Result<Self::LogSinker> {
+        let commit_checkpoint_interval =
+            NonZeroU64::new(self.config.commit_checkpoint_interval.unwrap_or(1)).expect(
+                "commit_checkpoint_interval should be greater than 0, and it should be checked in config validation",
+            );
+
+        let inner = StarrocksSinkWriter::new(
+            self.config.clone(),
+            self.schema.clone(),
+            self.pk_indices.clone(),
+            self.is_append_only,
+            writer_param.executor_id,
+        )?;
+        let writer = CoordinatedSinkWriter::new(
+            writer_param
+                .meta_client
+                .expect("should have meta client")
+                .sink_coordinate_client()
+                .await,
+            self.param.clone(),
+            writer_param.vnode_bitmap.ok_or_else(|| {
+                SinkError::Remote(anyhow!(
+                    "sink needs coordination should not have singleton input"
+                ))
+            })?,
+            inner,
+        )
+        .await?;
+
+        Ok(DecoupleCheckpointLogSinkerOf::new(
+            writer,
+            writer_param.sink_metrics,
+            commit_checkpoint_interval,
+        ))
+    }
+
+    async fn new_coordinator(&self) -> Result<Self::Coordinator> {
+        let header = HeaderBuilder::new()
+            .add_common_header()
+            .set_user_password(
+                self.config.common.user.clone(),
+                self.config.common.password.clone(),
+            )
+            .set_db(self.config.common.database.clone())
+            .set_table(self.config.common.table.clone())
+            .build();
+
+        let txn_request_builder = StarrocksTxnRequestBuilder::new(
+            format!(
+                "http://{}:{}",
+                self.config.common.host, self.config.common.http_port
+            ),
+            header,
+            self.config.stream_load_http_timeout_ms,
+        )?;
+        Ok(StarrocksSinkCommitter {
+            client: Arc::new(StarrocksTxnClient::new(txn_request_builder)),
+        })
+    }
 }
 
 pub struct StarrocksSinkWriter {
     pub config: StarrocksConfig,
+    #[expect(dead_code)]
     schema: Schema,
+    #[expect(dead_code)]
     pk_indices: Vec<usize>,
-    inserter_innet_builder: InserterInnerBuilder,
     is_append_only: bool,
     client: Option<StarrocksClient>,
+    txn_client: StarrocksTxnClient,
     row_encoder: JsonEncoder,
+    executor_id: u64,
+    curr_txn_label: Option<String>,
 }
 
 impl TryFrom<SinkParam> for StarrocksSink {
@@ -280,13 +398,8 @@ impl TryFrom<SinkParam> for StarrocksSink {
 
     fn try_from(param: SinkParam) -> std::result::Result<Self, Self::Error> {
         let schema = param.schema();
-        let config = StarrocksConfig::from_hashmap(param.properties)?;
-        StarrocksSink::new(
-            config,
-            schema,
-            param.downstream_pk,
-            param.sink_type.is_append_only(),
-        )
+        let config = StarrocksConfig::from_btreemap(param.properties.clone())?;
+        StarrocksSink::new(param, config, schema)
     }
 }
 
@@ -296,6 +409,7 @@ impl StarrocksSinkWriter {
         schema: Schema,
         pk_indices: Vec<usize>,
         is_append_only: bool,
+        executor_id: u64,
     ) -> Result<Self> {
         let mut fields_name = schema.names_str();
         if !is_append_only {
@@ -306,24 +420,28 @@ impl StarrocksSinkWriter {
             .add_common_header()
             .set_user_password(config.common.user.clone(), config.common.password.clone())
             .add_json_format()
-            .set_partial_update(config.common.partial_update.clone())
+            .set_partial_update(config.partial_update.clone())
             .set_columns_name(fields_name)
+            .set_db(config.common.database.clone())
+            .set_table(config.common.table.clone())
             .build();
 
-        let starrocks_insert_builder = InserterInnerBuilder::new(
+        let txn_request_builder = StarrocksTxnRequestBuilder::new(
             format!("http://{}:{}", config.common.host, config.common.http_port),
-            config.common.database.clone(),
-            config.common.table.clone(),
             header,
+            config.stream_load_http_timeout_ms,
         )?;
+
         Ok(Self {
             config,
             schema: schema.clone(),
             pk_indices,
-            inserter_innet_builder: starrocks_insert_builder,
             is_append_only,
             client: None,
+            txn_client: StarrocksTxnClient::new(txn_request_builder),
             row_encoder: JsonEncoder::new_with_starrocks(schema, None),
+            executor_id,
+            curr_txn_label: None,
         })
     }
 
@@ -403,14 +521,46 @@ impl StarrocksSinkWriter {
         }
         Ok(())
     }
+
+    /// Generating a new transaction label, should be unique across all `SinkWriters` even under rewinding.
+    #[inline(always)]
+    fn new_txn_label(&self) -> String {
+        format!(
+            "rw-txn-{}-{}",
+            self.executor_id,
+            chrono::Utc::now().timestamp_micros()
+        )
+    }
 }
 
 #[async_trait]
 impl SinkWriter for StarrocksSinkWriter {
+    type CommitMetadata = Option<SinkMetadata>;
+
+    async fn begin_epoch(&mut self, _epoch: u64) -> Result<()> {
+        Ok(())
+    }
+
     async fn write_batch(&mut self, chunk: StreamChunk) -> Result<()> {
+        // We check whether start a new transaction in `write_batch`. Therefore, if no data has been written
+        // within the `commit_checkpoint_interval` period, no meta requests will be made. Otherwise if we request
+        // `prepare` against an empty transaction, the `StarRocks` will report a `hasn't send any data yet` error.
+        if self.curr_txn_label.is_none() {
+            let txn_label = self.new_txn_label();
+            tracing::debug!(?txn_label, "begin transaction");
+            let txn_label_res = self.txn_client.begin(txn_label.clone()).await?;
+            assert_eq!(
+                txn_label, txn_label_res,
+                "label responding from StarRocks: {} differ from generated one: {}",
+                txn_label, txn_label_res
+            );
+            self.curr_txn_label = Some(txn_label.clone());
+        }
         if self.client.is_none() {
+            let txn_label = self.curr_txn_label.clone();
+            assert!(txn_label.is_some(), "transaction label is none during load");
             self.client = Some(StarrocksClient::new(
-                self.inserter_innet_builder.build().await?,
+                self.txn_client.load(txn_label.unwrap()).await?,
             ));
         }
         if self.is_append_only {
@@ -420,22 +570,45 @@ impl SinkWriter for StarrocksSinkWriter {
         }
     }
 
-    async fn begin_epoch(&mut self, _epoch: u64) -> Result<()> {
-        Ok(())
-    }
-
-    async fn abort(&mut self) -> Result<()> {
-        Ok(())
-    }
-
-    async fn barrier(&mut self, _is_checkpoint: bool) -> Result<()> {
+    async fn barrier(&mut self, is_checkpoint: bool) -> Result<Option<SinkMetadata>> {
         if self.client.is_some() {
+            // Here we finish the `/api/transaction/load` request when a barrier is received. Therefore,
+            // one or more load requests should be made within one commit_checkpoint_interval period.
+            // StarRocks will take care of merging those splits into a larger one during prepare transaction.
+            // Thus, only one version will be produced when the transaction is committed. See Stream Load
+            // transaction interface for more information.
             let client = self
                 .client
                 .take()
                 .ok_or_else(|| SinkError::Starrocks("Can't find starrocks inserter".to_string()))?;
             client.finish().await?;
         }
+
+        if is_checkpoint {
+            if self.curr_txn_label.is_some() {
+                let txn_label = self.curr_txn_label.take().unwrap();
+                tracing::debug!(?txn_label, "prepare transaction");
+                let txn_label_res = self.txn_client.prepare(txn_label.clone()).await?;
+                assert_eq!(
+                    txn_label, txn_label_res,
+                    "label responding from StarRocks differs from the current one"
+                );
+                Ok(Some(StarrocksWriteResult(Some(txn_label)).try_into()?))
+            } else {
+                // no data was written within previous epoch
+                Ok(Some(StarrocksWriteResult(None).try_into()?))
+            }
+        } else {
+            Ok(None)
+        }
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        if self.curr_txn_label.is_some() {
+            let txn_label = self.curr_txn_label.take().unwrap();
+            tracing::debug!(?txn_label, "rollback transaction");
+            self.txn_client.rollback(txn_label).await?;
+        }
         Ok(())
     }
 
@@ -459,6 +632,11 @@ impl StarrocksSchemaClient {
         user: String,
         password: String,
     ) -> Result<Self> {
+        // username & password may contain special chars, so we need to do URL encoding on them.
+        // Otherwise, Opts::from_url may report a `Parse error`
+        let user = form_urlencoded::byte_serialize(user.as_bytes()).collect::<String>();
+        let password = form_urlencoded::byte_serialize(password.as_bytes()).collect::<String>();
+
         let conn_uri = format!(
             "mysql://{}:{}@{}:{}/{}?prefer_socket={}&max_allowed_packet={}&wait_timeout={}",
             user,
@@ -472,12 +650,12 @@ impl StarrocksSchemaClient {
         );
         let pool = mysql_async::Pool::new(
             Opts::from_url(&conn_uri)
-                .map_err(|err| SinkError::DorisStarrocksConnect(err.into()))?,
+                .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?,
         );
         let conn = pool
             .get_conn()
             .await
-            .map_err(|err| SinkError::DorisStarrocksConnect(err.into()))?;
+            .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
 
         Ok(Self { table, db, conn })
     }
@@ -490,7 +668,7 @@ impl StarrocksSchemaClient {
                 query_map.insert(column_name, column_type)
             })
             .await
-            .map_err(|err| SinkError::DorisStarrocksConnect(err.into()))?;
+            .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
         Ok(query_map)
     }
 
@@ -502,7 +680,7 @@ impl StarrocksSchemaClient {
                 (table_model, primary_key)
             })
             .await
-            .map_err(|err| SinkError::DorisStarrocksConnect(err.into()))?
+            .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?
             .first()
             .ok_or_else(|| {
                 SinkError::Starrocks(format!(
@@ -518,35 +696,39 @@ impl StarrocksSchemaClient {
 #[derive(Debug, Serialize, Deserialize)]
 pub struct StarrocksInsertResultResponse {
     #[serde(rename = "TxnId")]
-    txn_id: i64,
+    pub txn_id: Option<i64>,
+    #[serde(rename = "Seq")]
+    pub seq: Option<i64>,
     #[serde(rename = "Label")]
-    label: String,
+    pub label: Option<String>,
     #[serde(rename = "Status")]
-    status: String,
+    pub status: String,
     #[serde(rename = "Message")]
-    message: String,
+    pub message: String,
     #[serde(rename = "NumberTotalRows")]
-    number_total_rows: i64,
+    pub number_total_rows: Option<i64>,
     #[serde(rename = "NumberLoadedRows")]
-    number_loaded_rows: i64,
+    pub number_loaded_rows: Option<i64>,
     #[serde(rename = "NumberFilteredRows")]
-    number_filtered_rows: i32,
+    pub number_filtered_rows: Option<i32>,
     #[serde(rename = "NumberUnselectedRows")]
-    number_unselected_rows: i32,
+    pub number_unselected_rows: Option<i32>,
     #[serde(rename = "LoadBytes")]
-    load_bytes: i64,
+    pub load_bytes: Option<i64>,
     #[serde(rename = "LoadTimeMs")]
-    load_time_ms: i32,
+    pub load_time_ms: Option<i32>,
     #[serde(rename = "BeginTxnTimeMs")]
-    begin_txn_time_ms: i32,
+    pub begin_txn_time_ms: Option<i32>,
     #[serde(rename = "ReadDataTimeMs")]
-    read_data_time_ms: i32,
+    pub read_data_time_ms: Option<i32>,
     #[serde(rename = "WriteDataTimeMs")]
-    write_data_time_ms: i32,
+    pub write_data_time_ms: Option<i32>,
     #[serde(rename = "CommitAndPublishTimeMs")]
-    commit_and_publish_time_ms: i32,
+    pub commit_and_publish_time_ms: Option<i32>,
     #[serde(rename = "StreamLoadPlanTimeMs")]
-    stream_load_plan_time_ms: Option<i32>,
+    pub stream_load_plan_time_ms: Option<i32>,
+    #[serde(rename = "ExistingJobStatus")]
+    pub existing_job_status: Option<String>,
 }
 
 pub struct StarrocksClient {
@@ -565,9 +747,9 @@ impl StarrocksClient {
     pub async fn finish(self) -> Result<StarrocksInsertResultResponse> {
         let raw = self.insert.finish().await?;
         let res: StarrocksInsertResultResponse = serde_json::from_slice(&raw)
-            .map_err(|err| SinkError::DorisStarrocksConnect(err.into()))?;
+            .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
 
-        if !DORIS_SUCCESS_STATUS.contains(&res.status.as_str()) {
+        if !STARROCKS_SUCCESS_STATUS.contains(&res.status.as_str()) {
             return Err(SinkError::DorisStarrocksConnect(anyhow::anyhow!(
                 "Insert error: {:?}",
                 res.message,
@@ -576,3 +758,140 @@ impl StarrocksClient {
         Ok(res)
     }
 }
+
+pub struct StarrocksTxnClient {
+    request_builder: StarrocksTxnRequestBuilder,
+}
+
+impl StarrocksTxnClient {
+    pub fn new(request_builder: StarrocksTxnRequestBuilder) -> Self {
+        Self { request_builder }
+    }
+
+    fn check_response_and_extract_label(&self, res: Bytes) -> Result<String> {
+        let res: StarrocksInsertResultResponse = serde_json::from_slice(&res)
+            .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
+        if !STARROCKS_SUCCESS_STATUS.contains(&res.status.as_str()) {
+            return Err(SinkError::DorisStarrocksConnect(anyhow::anyhow!(
+                "transaction error: {:?}",
+                res.message,
+            )));
+        }
+        res.label.ok_or_else(|| {
+            SinkError::DorisStarrocksConnect(anyhow::anyhow!("Can't get label from response"))
+        })
+    }
+
+    pub async fn begin(&self, label: String) -> Result<String> {
+        let res = self
+            .request_builder
+            .build_begin_request_sender(label)?
+            .send()
+            .await?;
+        self.check_response_and_extract_label(res)
+    }
+
+    pub async fn prepare(&self, label: String) -> Result<String> {
+        let res = self
+            .request_builder
+            .build_prepare_request_sender(label)?
+            .send()
+            .await?;
+        self.check_response_and_extract_label(res)
+    }
+
+    pub async fn commit(&self, label: String) -> Result<String> {
+        let res = self
+            .request_builder
+            .build_commit_request_sender(label)?
+            .send()
+            .await?;
+        self.check_response_and_extract_label(res)
+    }
+
+    pub async fn rollback(&self, label: String) -> Result<String> {
+        let res = self
+            .request_builder
+            .build_rollback_request_sender(label)?
+            .send()
+            .await?;
+        self.check_response_and_extract_label(res)
+    }
+
+    pub async fn load(&self, label: String) -> Result<InserterInner> {
+        self.request_builder.build_txn_inserter(label).await
+    }
+}
+
+struct StarrocksWriteResult(Option<String>);
+
+impl TryFrom<StarrocksWriteResult> for SinkMetadata {
+    type Error = SinkError;
+
+    fn try_from(value: StarrocksWriteResult) -> std::result::Result<Self, Self::Error> {
+        match value.0 {
+            Some(label) => {
+                let metadata = label.into_bytes();
+                Ok(SinkMetadata {
+                    metadata: Some(Serialized(SerializedMetadata { metadata })),
+                })
+            }
+            None => Ok(SinkMetadata { metadata: None }),
+        }
+    }
+}
+
+impl TryFrom<SinkMetadata> for StarrocksWriteResult {
+    type Error = SinkError;
+
+    fn try_from(value: SinkMetadata) -> std::result::Result<Self, Self::Error> {
+        if let Some(Serialized(v)) = value.metadata {
+            Ok(StarrocksWriteResult(Some(
+                String::from_utf8(v.metadata)
+                    .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?,
+            )))
+        } else {
+            Ok(StarrocksWriteResult(None))
+        }
+    }
+}
+
+pub struct StarrocksSinkCommitter {
+    client: Arc<StarrocksTxnClient>,
+}
+
+#[async_trait::async_trait]
+impl SinkCommitCoordinator for StarrocksSinkCommitter {
+    async fn init(&mut self) -> Result<()> {
+        tracing::info!("Starrocks commit coordinator inited.");
+        Ok(())
+    }
+
+    async fn commit(&mut self, epoch: u64, metadata: Vec<SinkMetadata>) -> Result<()> {
+        let write_results = metadata
+            .into_iter()
+            .map(TryFrom::try_from)
+            .collect::<Result<Vec<StarrocksWriteResult>>>()?;
+
+        let txn_labels = write_results
+            .into_iter()
+            .filter_map(|v| v.0)
+            .collect::<Vec<String>>();
+
+        tracing::debug!(?epoch, ?txn_labels, "commit transaction");
+
+        if !txn_labels.is_empty() {
+            let join_handles = txn_labels
+                .into_iter()
+                .map(|txn_label| {
+                    let client = self.client.clone();
+                    tokio::spawn(async move { client.commit(txn_label).await })
+                })
+                .collect::<Vec<JoinHandle<Result<String>>>>();
+            futures::future::try_join_all(join_handles)
+                .await
+                .map_err(|err| SinkError::DorisStarrocksConnect(anyhow!(err)))?;
+        }
+        Ok(())
+    }
+}
diff --git a/src/connector/src/sink/trivial.rs b/src/connector/src/sink/trivial.rs
index a3c5a1bd7e2ce..0cfa82c5c4d19 100644
--- a/src/connector/src/sink/trivial.rs
+++ b/src/connector/src/sink/trivial.rs
@@ -75,19 +75,15 @@ impl<T: TrivialSinkName> Sink for TrivialSink<T> {
 
 #[async_trait]
 impl<T: TrivialSinkName> LogSinker for TrivialSink<T> {
-    async fn consume_log_and_sink(self, log_reader: &mut impl SinkLogReader) -> Result<()> {
+    async fn consume_log_and_sink(self, log_reader: &mut impl SinkLogReader) -> Result<!> {
         loop {
             let (epoch, item) = log_reader.next_item().await?;
             match item {
                 LogStoreReadItem::StreamChunk { chunk_id, .. } => {
-                    log_reader
-                        .truncate(TruncateOffset::Chunk { epoch, chunk_id })
-                        .await?;
+                    log_reader.truncate(TruncateOffset::Chunk { epoch, chunk_id })?;
                 }
                 LogStoreReadItem::Barrier { .. } => {
-                    log_reader
-                        .truncate(TruncateOffset::Barrier { epoch })
-                        .await?;
+                    log_reader.truncate(TruncateOffset::Barrier { epoch })?;
                 }
                 LogStoreReadItem::UpdateVnodeBitmap(_) => {}
             }
diff --git a/src/connector/src/sink/writer.rs b/src/connector/src/sink/writer.rs
index 1d8142061f35b..00917f26c4163 100644
--- a/src/connector/src/sink/writer.rs
+++ b/src/connector/src/sink/writer.rs
@@ -126,7 +126,7 @@ impl<W> LogSinkerOf<W> {
 
 #[async_trait]
 impl<W: SinkWriter<CommitMetadata = ()>> LogSinker for LogSinkerOf<W> {
-    async fn consume_log_and_sink(self, log_reader: &mut impl SinkLogReader) -> Result<()> {
+    async fn consume_log_and_sink(self, log_reader: &mut impl SinkLogReader) -> Result<!> {
         let mut sink_writer = self.writer;
         let sink_metrics = self.sink_metrics;
         #[derive(Debug)]
@@ -199,9 +199,7 @@ impl<W: SinkWriter<CommitMetadata = ()>> LogSinker for LogSinkerOf<W> {
                         sink_metrics
                             .sink_commit_duration_metrics
                             .observe(start_time.elapsed().as_millis() as f64);
-                        log_reader
-                            .truncate(TruncateOffset::Barrier { epoch })
-                            .await?;
+                        log_reader.truncate(TruncateOffset::Barrier { epoch })?;
                     } else {
                         sink_writer.barrier(false).await?;
                     }
@@ -244,7 +242,7 @@ impl<W: AsyncTruncateSinkWriter> AsyncTruncateLogSinkerOf<W> {
 
 #[async_trait]
 impl<W: AsyncTruncateSinkWriter> LogSinker for AsyncTruncateLogSinkerOf<W> {
-    async fn consume_log_and_sink(mut self, log_reader: &mut impl SinkLogReader) -> Result<()> {
+    async fn consume_log_and_sink(mut self, log_reader: &mut impl SinkLogReader) -> Result<!> {
         loop {
             let select_result = drop_either_future(
                 select(
@@ -270,7 +268,7 @@ impl<W: AsyncTruncateSinkWriter> LogSinker for AsyncTruncateLogSinkerOf<W> {
                 }
                 Either::Right(offset_result) => {
                     let offset = offset_result?;
-                    log_reader.truncate(offset).await?;
+                    log_reader.truncate(offset)?;
                 }
             }
         }
diff --git a/src/connector/src/source/base.rs b/src/connector/src/source/base.rs
index e08ed4d56b61a..1f2444a0db4e7 100644
--- a/src/connector/src/source/base.rs
+++ b/src/connector/src/source/base.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::sync::Arc;
 
 use anyhow::anyhow;
@@ -42,7 +42,6 @@ use super::nexmark::source::message::NexmarkMeta;
 use super::{GCS_CONNECTOR, OPENDAL_S3_CONNECTOR, POSIX_FS_CONNECTOR};
 use crate::error::ConnectorResult as Result;
 use crate::parser::ParserConfig;
-pub(crate) use crate::source::common::CommonSplitReader;
 use crate::source::filesystem::FsPageItem;
 use crate::source::monitor::EnumeratorMetrics;
 use crate::source::SplitImpl::{CitusCdc, MongodbCdc, MysqlCdc, PostgresCdc};
@@ -56,15 +55,18 @@ const SPLIT_TYPE_FIELD: &str = "split_type";
 const SPLIT_INFO_FIELD: &str = "split_info";
 pub const UPSTREAM_SOURCE_KEY: &str = "connector";
 
-pub trait TryFromHashmap: Sized + UnknownFields {
+pub trait TryFromBTreeMap: Sized + UnknownFields {
     /// Used to initialize the source properties from the raw untyped `WITH` options.
-    fn try_from_hashmap(props: HashMap<String, String>, deny_unknown_fields: bool) -> Result<Self>;
+    fn try_from_btreemap(
+        props: BTreeMap<String, String>,
+        deny_unknown_fields: bool,
+    ) -> Result<Self>;
 }
 
 /// Represents `WITH` options for sources.
 ///
 /// Each instance should add a `#[derive(with_options::WithOptions)]` marker.
-pub trait SourceProperties: TryFromHashmap + Clone + WithOptions {
+pub trait SourceProperties: TryFromBTreeMap + Clone + WithOptions {
     const SOURCE_NAME: &'static str;
     type Split: SplitMetaData
         + TryFrom<SplitImpl, Error = crate::error::ConnectorError>
@@ -84,8 +86,11 @@ pub trait UnknownFields {
     fn unknown_fields(&self) -> HashMap<String, String>;
 }
 
-impl<P: DeserializeOwned + UnknownFields> TryFromHashmap for P {
-    fn try_from_hashmap(props: HashMap<String, String>, deny_unknown_fields: bool) -> Result<Self> {
+impl<P: DeserializeOwned + UnknownFields> TryFromBTreeMap for P {
+    fn try_from_btreemap(
+        props: BTreeMap<String, String>,
+        deny_unknown_fields: bool,
+    ) -> Result<Self> {
         let json_value = serde_json::to_value(props)?;
         let res = serde_json::from_value::<P>(json_value)?;
 
@@ -131,8 +136,7 @@ pub const MAX_CHUNK_SIZE: usize = 1024;
 
 #[derive(Debug, Clone)]
 pub struct SourceCtrlOpts {
-    // comes from developer::stream_chunk_size in stream scenario and developer::batch_chunk_size
-    // in batch scenario
+    /// The max size of a chunk yielded by source stream.
     pub chunk_size: usize,
     /// Rate limit of source
     pub rate_limit: Option<u32>,
@@ -315,10 +319,19 @@ pub fn extract_source_struct(info: &PbStreamSourceInfo) -> Result<SourceStruct>
     Ok(SourceStruct::new(format, encode))
 }
 
+/// Stream of [`SourceMessage`].
 pub type BoxSourceStream = BoxStream<'static, crate::error::ConnectorResult<Vec<SourceMessage>>>;
 
-pub trait ChunkSourceStream =
-    Stream<Item = crate::error::ConnectorResult<StreamChunk>> + Send + 'static;
+// Manually expand the trait alias to improve IDE experience.
+pub trait ChunkSourceStream:
+    Stream<Item = crate::error::ConnectorResult<StreamChunk>> + Send + 'static
+{
+}
+impl<T> ChunkSourceStream for T where
+    T: Stream<Item = crate::error::ConnectorResult<StreamChunk>> + Send + 'static
+{
+}
+
 pub type BoxChunkSourceStream = BoxStream<'static, crate::error::ConnectorResult<StreamChunk>>;
 pub type BoxTryStream<M> = BoxStream<'static, crate::error::ConnectorResult<M>>;
 
@@ -365,12 +378,12 @@ impl ConnectorProperties {
 impl ConnectorProperties {
     /// Creates typed source properties from the raw `WITH` properties.
     ///
-    /// It checks the `connector` field, and them dispatches to the corresponding type's `try_from_hashmap` method.
+    /// It checks the `connector` field, and them dispatches to the corresponding type's `try_from_btreemap` method.
     ///
     /// `deny_unknown_fields`: Since `WITH` options are persisted in meta, we do not deny unknown fields when restoring from
     /// existing data to avoid breaking backwards compatibility. We only deny unknown fields when creating new sources.
     pub fn extract(
-        mut with_properties: HashMap<String, String>,
+        mut with_properties: BTreeMap<String, String>,
         deny_unknown_fields: bool,
     ) -> Result<Self> {
         let connector = with_properties
@@ -379,7 +392,7 @@ impl ConnectorProperties {
         match_source_name_str!(
             connector.to_lowercase().as_str(),
             PropType,
-            PropType::try_from_hashmap(with_properties, deny_unknown_fields)
+            PropType::try_from_btreemap(with_properties, deny_unknown_fields)
                 .map(ConnectorProperties::from),
             |other| bail!("connector '{}' is not supported", other)
         )
@@ -544,6 +557,19 @@ pub struct SourceMessage {
     pub meta: SourceMeta,
 }
 
+impl SourceMessage {
+    /// Create a dummy `SourceMessage` with all fields unset for testing purposes.
+    pub fn dummy() -> Self {
+        Self {
+            key: None,
+            payload: None,
+            offset: "".to_string(),
+            split_id: "".into(),
+            meta: SourceMeta::Empty,
+        }
+    }
+}
+
 #[derive(Debug, Clone)]
 pub enum SourceMeta {
     Kafka(KafkaMeta),
@@ -654,7 +680,7 @@ mod tests {
 
     #[test]
     fn test_extract_nexmark_config() {
-        let props: HashMap<String, String> = convert_args!(hashmap!(
+        let props = convert_args!(btreemap!(
             "connector" => "nexmark",
             "nexmark.table.type" => "Person",
             "nexmark.split.num" => "1",
@@ -672,7 +698,7 @@ mod tests {
 
     #[test]
     fn test_extract_kafka_config() {
-        let props: HashMap<String, String> = convert_args!(hashmap!(
+        let props = convert_args!(btreemap!(
             "connector" => "kafka",
             "properties.bootstrap.server" => "b1,b2",
             "topic" => "test",
@@ -682,11 +708,11 @@ mod tests {
 
         let props = ConnectorProperties::extract(props, true).unwrap();
         if let ConnectorProperties::Kafka(k) = props {
-            let hashmap: HashMap<String, String> = hashmap! {
+            let btreemap = btreemap! {
                 "b-1:9092".to_string() => "dns-1".to_string(),
                 "b-2:9092".to_string() => "dns-2".to_string(),
             };
-            assert_eq!(k.privatelink_common.broker_rewrite_map, Some(hashmap));
+            assert_eq!(k.privatelink_common.broker_rewrite_map, Some(btreemap));
         } else {
             panic!("extract kafka config failed");
         }
@@ -694,7 +720,7 @@ mod tests {
 
     #[test]
     fn test_extract_cdc_properties() {
-        let user_props_mysql: HashMap<String, String> = convert_args!(hashmap!(
+        let user_props_mysql = convert_args!(btreemap!(
             "connector" => "mysql-cdc",
             "database.hostname" => "127.0.0.1",
             "database.port" => "3306",
@@ -704,7 +730,7 @@ mod tests {
             "table.name" => "products",
         ));
 
-        let user_props_postgres: HashMap<String, String> = convert_args!(hashmap!(
+        let user_props_postgres = convert_args!(btreemap!(
             "connector" => "postgres-cdc",
             "database.hostname" => "127.0.0.1",
             "database.port" => "5432",
diff --git a/src/connector/src/source/cdc/enumerator/mod.rs b/src/connector/src/source/cdc/enumerator/mod.rs
index 98018b6b6a113..bfe26c57a8341 100644
--- a/src/connector/src/source/cdc/enumerator/mod.rs
+++ b/src/connector/src/source/cdc/enumerator/mod.rs
@@ -27,7 +27,8 @@ use risingwave_pb::connector_service::{SourceType, ValidateSourceRequest, Valida
 
 use crate::error::ConnectorResult;
 use crate::source::cdc::{
-    CdcProperties, CdcSourceTypeTrait, Citus, DebeziumCdcSplit, Mongodb, Mysql, Postgres,
+    table_schema_exclude_additional_columns, CdcProperties, CdcSourceTypeTrait, Citus,
+    DebeziumCdcSplit, Mongodb, Mysql, Postgres,
 };
 use crate::source::{SourceEnumeratorContextRef, SplitEnumerator};
 
@@ -77,7 +78,7 @@ where
                 source_id: source_id as u64,
                 source_type: props.get_source_type_pb() as _,
                 properties: props.properties,
-                table_schema: Some(props.table_schema),
+                table_schema: Some(table_schema_exclude_additional_columns(&props.table_schema)),
                 is_source_job: props.is_cdc_source_job,
                 is_backfill_table: props.is_backfill_table,
             };
diff --git a/src/connector/src/source/cdc/external/mock_external_table.rs b/src/connector/src/source/cdc/external/mock_external_table.rs
index 4a652e1c1628c..7242f5614d409 100644
--- a/src/connector/src/source/cdc/external/mock_external_table.rs
+++ b/src/connector/src/source/cdc/external/mock_external_table.rs
@@ -38,10 +38,20 @@ impl MockExternalTableReader {
         }
     }
 
+    pub fn get_normalized_table_name(_table_name: &SchemaTableName) -> String {
+        "`mock_table`".to_string()
+    }
+
     pub fn get_cdc_offset_parser() -> CdcOffsetParseFunc {
-        Box::new(move |_| Ok(CdcOffset::MySql(MySqlOffset::default())))
+        Box::new(move |offset| {
+            Ok(CdcOffset::MySql(MySqlOffset::parse_debezium_offset(
+                offset,
+            )?))
+        })
     }
 
+    /// The snapshot will emit to downstream all in once, because it is too small.
+    /// After that we will emit the buffered upstream chunks all in one.
     #[try_stream(boxed, ok = OwnedRow, error = ConnectorError)]
     async fn snapshot_read_inner(&self) {
         let snap_idx = self
@@ -49,18 +59,18 @@ impl MockExternalTableReader {
             .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
         println!("snapshot read: idx {}", snap_idx);
 
-        let snap0 = vec![OwnedRow::new(vec![
-            Some(ScalarImpl::Int64(1)),
-            Some(ScalarImpl::Float64(1.0001.into())),
-        ])];
-        let snap1 = vec![
+        let snap0 = vec![
             OwnedRow::new(vec![
                 Some(ScalarImpl::Int64(1)),
-                Some(ScalarImpl::Float64(10.01.into())),
+                Some(ScalarImpl::Float64(1.0001.into())),
+            ]),
+            OwnedRow::new(vec![
+                Some(ScalarImpl::Int64(1)),
+                Some(ScalarImpl::Float64(11.00.into())),
             ]),
             OwnedRow::new(vec![
                 Some(ScalarImpl::Int64(2)),
-                Some(ScalarImpl::Float64(2.02.into())),
+                Some(ScalarImpl::Float64(22.00.into())),
             ]),
             OwnedRow::new(vec![
                 Some(ScalarImpl::Int64(5)),
@@ -76,7 +86,7 @@ impl MockExternalTableReader {
             ]),
         ];
 
-        let snapshots = [snap0, snap1];
+        let snapshots = [snap0];
         if snap_idx >= snapshots.len() {
             return Ok(());
         }
@@ -88,10 +98,6 @@ impl MockExternalTableReader {
 }
 
 impl ExternalTableReader for MockExternalTableReader {
-    fn get_normalized_table_name(&self, _table_name: &SchemaTableName) -> String {
-        "`mock_table`".to_string()
-    }
-
     async fn current_cdc_offset(&self) -> ConnectorResult<CdcOffset> {
         static IDX: AtomicUsize = AtomicUsize::new(0);
 
diff --git a/src/connector/src/source/cdc/external/mod.rs b/src/connector/src/source/cdc/external/mod.rs
index bd2b623731c7c..65bd4adb48132 100644
--- a/src/connector/src/source/cdc/external/mod.rs
+++ b/src/connector/src/source/cdc/external/mod.rs
@@ -13,33 +13,34 @@
 // limitations under the License.
 
 pub mod mock_external_table;
-mod postgres;
+pub mod postgres;
 
 #[cfg(not(madsim))]
 mod maybe_tls_connector;
+pub mod mysql;
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::fmt;
 
-use anyhow::Context;
+use anyhow::anyhow;
+use futures::pin_mut;
 use futures::stream::BoxStream;
-use futures::{pin_mut, StreamExt};
 use futures_async_stream::try_stream;
-use itertools::Itertools;
-use mysql_async::prelude::*;
-use mysql_common::params::Params;
-use mysql_common::value::Value;
 use risingwave_common::bail;
-use risingwave_common::catalog::{Schema, OFFSET_COLUMN_NAME};
+use risingwave_common::catalog::{ColumnDesc, Schema};
 use risingwave_common::row::OwnedRow;
-use risingwave_common::types::DataType;
-use risingwave_common::util::iter_util::ZipEqFast;
 use serde_derive::{Deserialize, Serialize};
 
 use crate::error::{ConnectorError, ConnectorResult};
 use crate::parser::mysql_row_to_owned_row;
 use crate::source::cdc::external::mock_external_table::MockExternalTableReader;
-use crate::source::cdc::external::postgres::{PostgresExternalTableReader, PostgresOffset};
+use crate::source::cdc::external::mysql::{
+    MySqlExternalTable, MySqlExternalTableReader, MySqlOffset,
+};
+use crate::source::cdc::external::postgres::{
+    PostgresExternalTable, PostgresExternalTableReader, PostgresOffset,
+};
+use crate::source::cdc::CdcSourceType;
 use crate::WithPropertiesExt;
 
 #[derive(Debug)]
@@ -67,15 +68,17 @@ impl CdcTableType {
 
     pub async fn create_table_reader(
         &self,
-        with_properties: HashMap<String, String>,
+        config: ExternalTableConfig,
         schema: Schema,
+        pk_indices: Vec<usize>,
+        scan_limit: u32,
     ) -> ConnectorResult<ExternalTableReaderImpl> {
         match self {
             Self::MySql => Ok(ExternalTableReaderImpl::MySql(
-                MySqlExternalTableReader::new(with_properties, schema).await?,
+                MySqlExternalTableReader::new(config, schema).await?,
             )),
             Self::Postgres => Ok(ExternalTableReaderImpl::Postgres(
-                PostgresExternalTableReader::new(with_properties, schema).await?,
+                PostgresExternalTableReader::new(config, schema, pk_indices, scan_limit).await?,
             )),
             _ => bail!("invalid external table type: {:?}", *self),
         }
@@ -94,13 +97,6 @@ pub const SCHEMA_NAME_KEY: &str = "schema.name";
 pub const DATABASE_NAME_KEY: &str = "database.name";
 
 impl SchemaTableName {
-    pub fn new(schema_name: String, table_name: String) -> Self {
-        Self {
-            schema_name,
-            table_name,
-        }
-    }
-
     pub fn from_properties(properties: &HashMap<String, String>) -> Self {
         let table_type = CdcTableType::from_properties(properties);
         let table_name = properties.get(TABLE_NAME_KEY).cloned().unwrap_or_default();
@@ -125,18 +121,6 @@ impl SchemaTableName {
     }
 }
 
-#[derive(Debug, Clone, Default, PartialEq, PartialOrd, Serialize, Deserialize)]
-pub struct MySqlOffset {
-    pub filename: String,
-    pub position: u64,
-}
-
-impl MySqlOffset {
-    pub fn new(filename: String, position: u64) -> Self {
-        Self { filename, position }
-    }
-}
-
 #[derive(Debug, Clone, PartialEq, PartialOrd, Serialize, Deserialize)]
 pub enum CdcOffset {
     MySql(MySqlOffset),
@@ -186,29 +170,9 @@ pub struct DebeziumSourceOffset {
     pub tx_usec: Option<u64>,
 }
 
-impl MySqlOffset {
-    pub fn parse_debezium_offset(offset: &str) -> ConnectorResult<Self> {
-        let dbz_offset: DebeziumOffset = serde_json::from_str(offset)
-            .with_context(|| format!("invalid upstream offset: {}", offset))?;
-
-        Ok(Self {
-            filename: dbz_offset
-                .source_offset
-                .file
-                .context("binlog file not found in offset")?,
-            position: dbz_offset
-                .source_offset
-                .pos
-                .context("binlog position not found in offset")?,
-        })
-    }
-}
-
 pub type CdcOffsetParseFunc = Box<dyn Fn(&str) -> ConnectorResult<CdcOffset> + Send>;
 
 pub trait ExternalTableReader {
-    fn get_normalized_table_name(&self, table_name: &SchemaTableName) -> String;
-
     async fn current_cdc_offset(&self) -> ConnectorResult<CdcOffset>;
 
     fn snapshot_read(
@@ -220,24 +184,16 @@ pub trait ExternalTableReader {
     ) -> BoxStream<'_, ConnectorResult<OwnedRow>>;
 }
 
-#[derive(Debug)]
 pub enum ExternalTableReaderImpl {
     MySql(MySqlExternalTableReader),
     Postgres(PostgresExternalTableReader),
     Mock(MockExternalTableReader),
 }
 
-#[derive(Debug)]
-pub struct MySqlExternalTableReader {
-    config: ExternalTableConfig,
-    rw_schema: Schema,
-    field_names: String,
-    // use mutex to provide shared mutable access to the connection
-    conn: tokio::sync::Mutex<mysql_async::Conn>,
-}
-
 #[derive(Debug, Clone, Deserialize)]
 pub struct ExternalTableConfig {
+    pub connector: String,
+
     #[serde(rename = "hostname")]
     pub host: String,
     pub port: String,
@@ -250,269 +206,48 @@ pub struct ExternalTableConfig {
     #[serde(rename = "table.name")]
     pub table: String,
     /// `ssl.mode` specifies the SSL/TLS encryption level for secure communication with Postgres.
-    /// Choices include `disable`, `prefer`, and `require`.
+    /// Choices include `disabled`, `preferred`, and `required`.
     /// This field is optional.
     #[serde(rename = "ssl.mode", default = "Default::default")]
     pub sslmode: SslMode,
 }
 
+impl ExternalTableConfig {
+    pub fn try_from_btreemap(
+        connect_properties: BTreeMap<String, String>,
+    ) -> ConnectorResult<Self> {
+        let json_value = serde_json::to_value(connect_properties)?;
+        let config = serde_json::from_value::<ExternalTableConfig>(json_value)?;
+        Ok(config)
+    }
+}
+
 #[derive(Debug, Clone, Deserialize)]
 #[serde(rename_all = "lowercase")]
 pub enum SslMode {
-    Disable,
-    Prefer,
-    Require,
+    Disabled,
+    Preferred,
+    Required,
 }
 
 impl Default for SslMode {
     fn default() -> Self {
-        // default to `disable` for backward compatibility
-        Self::Disable
+        // default to `disabled` for backward compatibility
+        Self::Disabled
     }
 }
 
 impl fmt::Display for SslMode {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str(match self {
-            SslMode::Disable => "disable",
-            SslMode::Prefer => "prefer",
-            SslMode::Require => "require",
+            SslMode::Disabled => "disabled",
+            SslMode::Preferred => "preferred",
+            SslMode::Required => "required",
         })
     }
 }
 
-impl ExternalTableReader for MySqlExternalTableReader {
-    fn get_normalized_table_name(&self, table_name: &SchemaTableName) -> String {
-        // schema name is the database name in mysql
-        format!("`{}`.`{}`", table_name.schema_name, table_name.table_name)
-    }
-
-    async fn current_cdc_offset(&self) -> ConnectorResult<CdcOffset> {
-        let mut conn = self.conn.lock().await;
-
-        let sql = "SHOW MASTER STATUS".to_string();
-        let mut rs = conn.query::<mysql_async::Row, _>(sql).await?;
-        let row = rs
-            .iter_mut()
-            .exactly_one()
-            .ok()
-            .context("expect exactly one row when reading binlog offset")?;
-
-        Ok(CdcOffset::MySql(MySqlOffset {
-            filename: row.take("File").unwrap(),
-            position: row.take("Position").unwrap(),
-        }))
-    }
-
-    fn snapshot_read(
-        &self,
-        table_name: SchemaTableName,
-        start_pk: Option<OwnedRow>,
-        primary_keys: Vec<String>,
-        limit: u32,
-    ) -> BoxStream<'_, ConnectorResult<OwnedRow>> {
-        self.snapshot_read_inner(table_name, start_pk, primary_keys, limit)
-    }
-}
-
-impl MySqlExternalTableReader {
-    pub async fn new(
-        with_properties: HashMap<String, String>,
-        rw_schema: Schema,
-    ) -> ConnectorResult<Self> {
-        tracing::debug!(?rw_schema, "create mysql external table reader");
-
-        let config = serde_json::from_value::<ExternalTableConfig>(
-            serde_json::to_value(with_properties).unwrap(),
-        )
-        .context("failed to extract mysql connector properties")?;
-
-        let database_url = format!(
-            "mysql://{}:{}@{}:{}/{}",
-            config.username, config.password, config.host, config.port, config.database
-        );
-        let opts = mysql_async::Opts::from_url(&database_url).map_err(mysql_async::Error::Url)?;
-        let conn = mysql_async::Conn::new(opts).await?;
-
-        let field_names = rw_schema
-            .fields
-            .iter()
-            .filter(|f| f.name != OFFSET_COLUMN_NAME)
-            .map(|f| Self::quote_column(f.name.as_str()))
-            .join(",");
-
-        Ok(Self {
-            config,
-            rw_schema,
-            field_names,
-            conn: tokio::sync::Mutex::new(conn),
-        })
-    }
-
-    pub fn get_cdc_offset_parser() -> CdcOffsetParseFunc {
-        Box::new(move |offset| {
-            Ok(CdcOffset::MySql(MySqlOffset::parse_debezium_offset(
-                offset,
-            )?))
-        })
-    }
-
-    #[try_stream(boxed, ok = OwnedRow, error = ConnectorError)]
-    async fn snapshot_read_inner(
-        &self,
-        table_name: SchemaTableName,
-        start_pk_row: Option<OwnedRow>,
-        primary_keys: Vec<String>,
-        limit: u32,
-    ) {
-        let order_key = primary_keys
-            .iter()
-            .map(|col| Self::quote_column(col))
-            .join(",");
-        let sql = if start_pk_row.is_none() {
-            format!(
-                "SELECT {} FROM {} ORDER BY {} LIMIT {limit}",
-                self.field_names,
-                self.get_normalized_table_name(&table_name),
-                order_key,
-            )
-        } else {
-            let filter_expr = Self::filter_expression(&primary_keys);
-            format!(
-                "SELECT {} FROM {} WHERE {} ORDER BY {} LIMIT {limit}",
-                self.field_names,
-                self.get_normalized_table_name(&table_name),
-                filter_expr,
-                order_key,
-            )
-        };
-
-        let mut conn = self.conn.lock().await;
-
-        // Set session timezone to UTC
-        conn.exec_drop("SET time_zone = \"+00:00\"", ()).await?;
-
-        if start_pk_row.is_none() {
-            let rs_stream = sql.stream::<mysql_async::Row, _>(&mut *conn).await?;
-            let row_stream = rs_stream.map(|row| {
-                // convert mysql row into OwnedRow
-                let mut row = row?;
-                Ok::<_, ConnectorError>(mysql_row_to_owned_row(&mut row, &self.rw_schema))
-            });
-
-            pin_mut!(row_stream);
-            #[for_await]
-            for row in row_stream {
-                let row = row?;
-                yield row;
-            }
-        } else {
-            let field_map = self
-                .rw_schema
-                .fields
-                .iter()
-                .map(|f| (f.name.as_str(), f.data_type.clone()))
-                .collect::<HashMap<_, _>>();
-
-            // fill in start primary key params
-            let params: Vec<_> = primary_keys
-                .iter()
-                .zip_eq_fast(start_pk_row.unwrap().into_iter())
-                .map(|(pk, datum)| {
-                    if let Some(value) = datum {
-                        let ty = field_map.get(pk.as_str()).unwrap();
-                        let val = match ty {
-                            DataType::Boolean => Value::from(value.into_bool()),
-                            DataType::Int16 => Value::from(value.into_int16()),
-                            DataType::Int32 => Value::from(value.into_int32()),
-                            DataType::Int64 => Value::from(value.into_int64()),
-                            DataType::Float32 => Value::from(value.into_float32().into_inner()),
-                            DataType::Float64 => Value::from(value.into_float64().into_inner()),
-                            DataType::Varchar => Value::from(String::from(value.into_utf8())),
-                            DataType::Date => Value::from(value.into_date().0),
-                            DataType::Time => Value::from(value.into_time().0),
-                            DataType::Timestamp => Value::from(value.into_timestamp().0),
-                            _ => bail!("unsupported primary key data type: {}", ty),
-                        };
-                        ConnectorResult::Ok((pk.clone(), val))
-                    } else {
-                        bail!("primary key {} cannot be null", pk);
-                    }
-                })
-                .try_collect::<_, _, ConnectorError>()?;
-
-            let rs_stream = sql
-                .with(Params::from(params))
-                .stream::<mysql_async::Row, _>(&mut *conn)
-                .await?;
-
-            let row_stream = rs_stream.map(|row| {
-                // convert mysql row into OwnedRow
-                let mut row = row?;
-                Ok::<_, ConnectorError>(mysql_row_to_owned_row(&mut row, &self.rw_schema))
-            });
-
-            pin_mut!(row_stream);
-            #[for_await]
-            for row in row_stream {
-                let row = row?;
-                yield row;
-            }
-        };
-    }
-
-    // mysql cannot leverage the given key to narrow down the range of scan,
-    // we need to rewrite the comparison conditions by our own.
-    // (a, b) > (x, y) => (`a` > x) OR ((`a` = x) AND (`b` > y))
-    fn filter_expression(columns: &[String]) -> String {
-        let mut conditions = vec![];
-        // push the first condition
-        conditions.push(format!(
-            "({} > :{})",
-            Self::quote_column(&columns[0]),
-            columns[0]
-        ));
-        for i in 2..=columns.len() {
-            // '=' condition
-            let mut condition = String::new();
-            for (j, col) in columns.iter().enumerate().take(i - 1) {
-                if j == 0 {
-                    condition.push_str(&format!("({} = :{})", Self::quote_column(col), col));
-                } else {
-                    condition.push_str(&format!(" AND ({} = :{})", Self::quote_column(col), col));
-                }
-            }
-            // '>' condition
-            condition.push_str(&format!(
-                " AND ({} > :{})",
-                Self::quote_column(&columns[i - 1]),
-                columns[i - 1]
-            ));
-            conditions.push(format!("({})", condition));
-        }
-        if columns.len() > 1 {
-            conditions.join(" OR ")
-        } else {
-            conditions.join("")
-        }
-    }
-
-    fn quote_column(column: &str) -> String {
-        format!("`{}`", column)
-    }
-}
-
 impl ExternalTableReader for ExternalTableReaderImpl {
-    fn get_normalized_table_name(&self, table_name: &SchemaTableName) -> String {
-        match self {
-            ExternalTableReaderImpl::MySql(mysql) => mysql.get_normalized_table_name(table_name),
-            ExternalTableReaderImpl::Postgres(postgres) => {
-                postgres.get_normalized_table_name(table_name)
-            }
-            ExternalTableReaderImpl::Mock(mock) => mock.get_normalized_table_name(table_name),
-        }
-    }
-
     async fn current_cdc_offset(&self) -> ConnectorResult<CdcOffset> {
         match self {
             ExternalTableReaderImpl::MySql(mysql) => mysql.current_cdc_offset().await,
@@ -572,94 +307,36 @@ impl ExternalTableReaderImpl {
     }
 }
 
-#[cfg(test)]
-mod tests {
-
-    use futures::pin_mut;
-    use futures_async_stream::for_await;
-    use maplit::{convert_args, hashmap};
-    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, Schema};
-    use risingwave_common::types::DataType;
-
-    use crate::source::cdc::external::{
-        CdcOffset, ExternalTableReader, MySqlExternalTableReader, MySqlOffset, SchemaTableName,
-    };
-
-    #[test]
-    fn test_mysql_filter_expr() {
-        let cols = vec!["id".to_string()];
-        let expr = MySqlExternalTableReader::filter_expression(&cols);
-        assert_eq!(expr, "(`id` > :id)");
-
-        let cols = vec!["aa".to_string(), "bb".to_string(), "cc".to_string()];
-        let expr = MySqlExternalTableReader::filter_expression(&cols);
-        assert_eq!(
-            expr,
-            "(`aa` > :aa) OR ((`aa` = :aa) AND (`bb` > :bb)) OR ((`aa` = :aa) AND (`bb` = :bb) AND (`cc` > :cc))"
-        );
-    }
+pub enum ExternalTableImpl {
+    MySql(MySqlExternalTable),
+    Postgres(PostgresExternalTable),
+}
 
-    #[test]
-    fn test_mysql_binlog_offset() {
-        let off0_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000001", "pos": 105622, "snapshot": true }, "isHeartbeat": false }"#;
-        let off1_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000007", "pos": 1062363217, "snapshot": true }, "isHeartbeat": false }"#;
-        let off2_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000007", "pos": 659687560, "snapshot": true }, "isHeartbeat": false }"#;
-        let off3_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000008", "pos": 7665875, "snapshot": true }, "isHeartbeat": false }"#;
-        let off4_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000008", "pos": 7665875, "snapshot": true }, "isHeartbeat": false }"#;
-
-        let off0 = CdcOffset::MySql(MySqlOffset::parse_debezium_offset(off0_str).unwrap());
-        let off1 = CdcOffset::MySql(MySqlOffset::parse_debezium_offset(off1_str).unwrap());
-        let off2 = CdcOffset::MySql(MySqlOffset::parse_debezium_offset(off2_str).unwrap());
-        let off3 = CdcOffset::MySql(MySqlOffset::parse_debezium_offset(off3_str).unwrap());
-        let off4 = CdcOffset::MySql(MySqlOffset::parse_debezium_offset(off4_str).unwrap());
-
-        assert!(off0 <= off1);
-        assert!(off1 > off2);
-        assert!(off2 < off3);
-        assert_eq!(off3, off4);
+impl ExternalTableImpl {
+    pub async fn connect(config: ExternalTableConfig) -> ConnectorResult<Self> {
+        let cdc_source_type = CdcSourceType::from(config.connector.as_str());
+        match cdc_source_type {
+            CdcSourceType::Mysql => Ok(ExternalTableImpl::MySql(
+                MySqlExternalTable::connect(config).await?,
+            )),
+            CdcSourceType::Postgres => Ok(ExternalTableImpl::Postgres(
+                PostgresExternalTable::connect(config).await?,
+            )),
+            _ => Err(anyhow!("Unsupported cdc connector type: {}", config.connector).into()),
+        }
     }
 
-    // manual test case
-    #[ignore]
-    #[tokio::test]
-    async fn test_mysql_table_reader() {
-        let columns = vec![
-            ColumnDesc::named("v1", ColumnId::new(1), DataType::Int32),
-            ColumnDesc::named("v2", ColumnId::new(2), DataType::Decimal),
-            ColumnDesc::named("v3", ColumnId::new(3), DataType::Varchar),
-            ColumnDesc::named("v4", ColumnId::new(4), DataType::Date),
-        ];
-        let rw_schema = Schema {
-            fields: columns.iter().map(Field::from).collect(),
-        };
-        let props = convert_args!(hashmap!(
-                "hostname" => "localhost",
-                "port" => "8306",
-                "username" => "root",
-                "password" => "123456",
-                "database.name" => "mytest",
-                "table.name" => "t1"));
-
-        let reader = MySqlExternalTableReader::new(props, rw_schema)
-            .await
-            .unwrap();
-        let offset = reader.current_cdc_offset().await.unwrap();
-        println!("BinlogOffset: {:?}", offset);
-
-        let off0_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000001", "pos": 105622, "snapshot": true }, "isHeartbeat": false }"#;
-        let parser = MySqlExternalTableReader::get_cdc_offset_parser();
-        println!("parsed offset: {:?}", parser(off0_str).unwrap());
-
-        let table_name = SchemaTableName {
-            schema_name: "mytest".to_string(),
-            table_name: "t1".to_string(),
-        };
+    pub fn column_descs(&self) -> &Vec<ColumnDesc> {
+        match self {
+            ExternalTableImpl::MySql(mysql) => mysql.column_descs(),
+            ExternalTableImpl::Postgres(postgres) => postgres.column_descs(),
+        }
+    }
 
-        let stream = reader.snapshot_read(table_name, None, vec!["v1".to_string()], 1000);
-        pin_mut!(stream);
-        #[for_await]
-        for row in stream {
-            println!("OwnedRow: {:?}", row);
+    pub fn pk_names(&self) -> &Vec<String> {
+        match self {
+            ExternalTableImpl::MySql(mysql) => mysql.pk_names(),
+            ExternalTableImpl::Postgres(postgres) => postgres.pk_names(),
         }
     }
 }
diff --git a/src/connector/src/source/cdc/external/mysql.rs b/src/connector/src/source/cdc/external/mysql.rs
new file mode 100644
index 0000000000000..37d855a9513e3
--- /dev/null
+++ b/src/connector/src/source/cdc/external/mysql.rs
@@ -0,0 +1,562 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+
+use anyhow::{anyhow, Context};
+use futures::stream::BoxStream;
+use futures::{pin_mut, StreamExt};
+use futures_async_stream::try_stream;
+use itertools::Itertools;
+use mysql_async::prelude::*;
+use mysql_common::params::Params;
+use mysql_common::value::Value;
+use risingwave_common::bail;
+use risingwave_common::catalog::{ColumnDesc, ColumnId, Schema, OFFSET_COLUMN_NAME};
+use risingwave_common::row::OwnedRow;
+use risingwave_common::types::DataType;
+use risingwave_common::util::iter_util::ZipEqFast;
+use sea_schema::mysql::def::{CharSet, Collation, ColumnKey, ColumnType, StorageEngine, TableInfo};
+use sea_schema::mysql::discovery::SchemaDiscovery;
+use serde_derive::{Deserialize, Serialize};
+use sqlx::mysql::MySqlConnectOptions;
+use sqlx::MySqlPool;
+
+use crate::error::{ConnectorError, ConnectorResult};
+use crate::source::cdc::external::{
+    mysql_row_to_owned_row, CdcOffset, CdcOffsetParseFunc, DebeziumOffset, ExternalTableConfig,
+    ExternalTableReader, SchemaTableName, SslMode,
+};
+
+#[derive(Debug, Clone, Default, PartialEq, PartialOrd, Serialize, Deserialize)]
+pub struct MySqlOffset {
+    pub filename: String,
+    pub position: u64,
+}
+
+impl MySqlOffset {
+    pub fn new(filename: String, position: u64) -> Self {
+        Self { filename, position }
+    }
+}
+
+impl MySqlOffset {
+    pub fn parse_debezium_offset(offset: &str) -> ConnectorResult<Self> {
+        let dbz_offset: DebeziumOffset = serde_json::from_str(offset)
+            .with_context(|| format!("invalid upstream offset: {}", offset))?;
+
+        Ok(Self {
+            filename: dbz_offset
+                .source_offset
+                .file
+                .context("binlog file not found in offset")?,
+            position: dbz_offset
+                .source_offset
+                .pos
+                .context("binlog position not found in offset")?,
+        })
+    }
+}
+
+pub struct MySqlExternalTable {
+    column_descs: Vec<ColumnDesc>,
+    pk_names: Vec<String>,
+}
+
+impl MySqlExternalTable {
+    pub async fn connect(config: ExternalTableConfig) -> ConnectorResult<Self> {
+        tracing::debug!("connect to mysql");
+        let options = MySqlConnectOptions::new()
+            .username(&config.username)
+            .password(&config.password)
+            .host(&config.host)
+            .port(config.port.parse::<u16>().unwrap())
+            .database(&config.database)
+            .ssl_mode(match config.sslmode {
+                SslMode::Disabled | SslMode::Preferred => sqlx::mysql::MySqlSslMode::Disabled,
+                SslMode::Required => sqlx::mysql::MySqlSslMode::Required,
+            });
+
+        let connection = MySqlPool::connect_with(options).await?;
+        let schema_discovery = SchemaDiscovery::new(connection, config.database.as_str());
+
+        let table_schema = schema_discovery
+            .discover_table(TableInfo {
+                name: config.table.clone(),
+                engine: StorageEngine::InnoDb,
+                auto_increment: None,
+                char_set: CharSet::Utf8Mb4,
+                collation: Collation::Utf8Mb40900AiCi,
+                comment: "".to_string(),
+            })
+            .await?;
+
+        let mut column_descs = vec![];
+        let mut pk_names = vec![];
+        for col in &table_schema.columns {
+            let data_type = type_to_rw_type(&col.col_type)?;
+            column_descs.push(ColumnDesc::named(
+                col.name.clone(),
+                ColumnId::placeholder(),
+                data_type,
+            ));
+            if matches!(col.key, ColumnKey::Primary) {
+                pk_names.push(col.name.clone());
+            }
+        }
+
+        Ok(Self {
+            column_descs,
+            pk_names,
+        })
+    }
+
+    pub fn column_descs(&self) -> &Vec<ColumnDesc> {
+        &self.column_descs
+    }
+
+    pub fn pk_names(&self) -> &Vec<String> {
+        &self.pk_names
+    }
+}
+
+fn type_to_rw_type(col_type: &ColumnType) -> ConnectorResult<DataType> {
+    let dtype = match col_type {
+        ColumnType::Serial => DataType::Int32,
+        ColumnType::Bit(attr) => {
+            if let Some(1) = attr.maximum {
+                DataType::Boolean
+            } else {
+                return Err(
+                    anyhow!("BIT({}) type not supported", attr.maximum.unwrap_or(0)).into(),
+                );
+            }
+        }
+        ColumnType::TinyInt(_) | ColumnType::SmallInt(_) => DataType::Int16,
+        ColumnType::Bool => DataType::Boolean,
+        ColumnType::MediumInt(_) => DataType::Int32,
+        ColumnType::Int(_) => DataType::Int32,
+        ColumnType::BigInt(_) => DataType::Int64,
+        ColumnType::Decimal(_) => DataType::Decimal,
+        ColumnType::Float(_) => DataType::Float32,
+        ColumnType::Double(_) => DataType::Float64,
+        ColumnType::Date => DataType::Date,
+        ColumnType::Time(_) => DataType::Time,
+        ColumnType::DateTime(_) => DataType::Timestamp,
+        ColumnType::Timestamp(_) => DataType::Timestamptz,
+        ColumnType::Year => DataType::Int32,
+        ColumnType::Char(_)
+        | ColumnType::NChar(_)
+        | ColumnType::Varchar(_)
+        | ColumnType::NVarchar(_) => DataType::Varchar,
+        ColumnType::Binary(_) | ColumnType::Varbinary(_) => DataType::Bytea,
+        ColumnType::Text(_)
+        | ColumnType::TinyText(_)
+        | ColumnType::MediumText(_)
+        | ColumnType::LongText(_) => DataType::Varchar,
+        ColumnType::Blob(_)
+        | ColumnType::TinyBlob
+        | ColumnType::MediumBlob
+        | ColumnType::LongBlob => DataType::Bytea,
+        ColumnType::Enum(_) => DataType::Varchar,
+        ColumnType::Json => DataType::Jsonb,
+        ColumnType::Set(_) => {
+            return Err(anyhow!("SET type not supported").into());
+        }
+        ColumnType::Geometry(_) => {
+            return Err(anyhow!("GEOMETRY type not supported").into());
+        }
+        ColumnType::Point(_) => {
+            return Err(anyhow!("POINT type not supported").into());
+        }
+        ColumnType::LineString(_) => {
+            return Err(anyhow!("LINE string type not supported").into());
+        }
+        ColumnType::Polygon(_) => {
+            return Err(anyhow!("POLYGON type not supported").into());
+        }
+        ColumnType::MultiPoint(_) => {
+            return Err(anyhow!("MULTI POINT type not supported").into());
+        }
+        ColumnType::MultiLineString(_) => {
+            return Err(anyhow!("MULTI LINE STRING type not supported").into());
+        }
+        ColumnType::MultiPolygon(_) => {
+            return Err(anyhow!("MULTI POLYGON type not supported").into());
+        }
+        ColumnType::GeometryCollection(_) => {
+            return Err(anyhow!("GEOMETRY COLLECTION type not supported").into());
+        }
+        ColumnType::Unknown(_) => {
+            return Err(anyhow!("Unknown MySQL data type").into());
+        }
+    };
+
+    Ok(dtype)
+}
+
+pub struct MySqlExternalTableReader {
+    rw_schema: Schema,
+    field_names: String,
+    // use mutex to provide shared mutable access to the connection
+    conn: tokio::sync::Mutex<mysql_async::Conn>,
+}
+
+impl ExternalTableReader for MySqlExternalTableReader {
+    async fn current_cdc_offset(&self) -> ConnectorResult<CdcOffset> {
+        let mut conn = self.conn.lock().await;
+
+        let sql = "SHOW MASTER STATUS".to_string();
+        let mut rs = conn.query::<mysql_async::Row, _>(sql).await?;
+        let row = rs
+            .iter_mut()
+            .exactly_one()
+            .ok()
+            .context("expect exactly one row when reading binlog offset")?;
+
+        Ok(CdcOffset::MySql(MySqlOffset {
+            filename: row.take("File").unwrap(),
+            position: row.take("Position").unwrap(),
+        }))
+    }
+
+    fn snapshot_read(
+        &self,
+        table_name: SchemaTableName,
+        start_pk: Option<OwnedRow>,
+        primary_keys: Vec<String>,
+        limit: u32,
+    ) -> BoxStream<'_, ConnectorResult<OwnedRow>> {
+        self.snapshot_read_inner(table_name, start_pk, primary_keys, limit)
+    }
+}
+
+impl MySqlExternalTableReader {
+    pub async fn new(config: ExternalTableConfig, rw_schema: Schema) -> ConnectorResult<Self> {
+        let mut opts_builder = mysql_async::OptsBuilder::default()
+            .user(Some(config.username))
+            .pass(Some(config.password))
+            .ip_or_hostname(config.host)
+            .tcp_port(config.port.parse::<u16>().unwrap())
+            .db_name(Some(config.database));
+
+        opts_builder = match config.sslmode {
+            SslMode::Disabled | SslMode::Preferred => opts_builder.ssl_opts(None),
+            SslMode::Required => {
+                let ssl_without_verify = mysql_async::SslOpts::default()
+                    .with_danger_accept_invalid_certs(true)
+                    .with_danger_skip_domain_validation(true);
+                opts_builder.ssl_opts(Some(ssl_without_verify))
+            }
+        };
+
+        let conn = mysql_async::Conn::new(mysql_async::Opts::from(opts_builder)).await?;
+
+        let field_names = rw_schema
+            .fields
+            .iter()
+            .filter(|f| f.name != OFFSET_COLUMN_NAME)
+            .map(|f| Self::quote_column(f.name.as_str()))
+            .join(",");
+
+        Ok(Self {
+            rw_schema,
+            field_names,
+            conn: tokio::sync::Mutex::new(conn),
+        })
+    }
+
+    pub fn get_normalized_table_name(table_name: &SchemaTableName) -> String {
+        // schema name is the database name in mysql
+        format!("`{}`.`{}`", table_name.schema_name, table_name.table_name)
+    }
+
+    pub fn get_cdc_offset_parser() -> CdcOffsetParseFunc {
+        Box::new(move |offset| {
+            Ok(CdcOffset::MySql(MySqlOffset::parse_debezium_offset(
+                offset,
+            )?))
+        })
+    }
+
+    #[try_stream(boxed, ok = OwnedRow, error = ConnectorError)]
+    async fn snapshot_read_inner(
+        &self,
+        table_name: SchemaTableName,
+        start_pk_row: Option<OwnedRow>,
+        primary_keys: Vec<String>,
+        limit: u32,
+    ) {
+        let order_key = primary_keys
+            .iter()
+            .map(|col| Self::quote_column(col))
+            .join(",");
+        let sql = if start_pk_row.is_none() {
+            format!(
+                "SELECT {} FROM {} ORDER BY {} LIMIT {limit}",
+                self.field_names,
+                Self::get_normalized_table_name(&table_name),
+                order_key,
+            )
+        } else {
+            let filter_expr = Self::filter_expression(&primary_keys);
+            format!(
+                "SELECT {} FROM {} WHERE {} ORDER BY {} LIMIT {limit}",
+                self.field_names,
+                Self::get_normalized_table_name(&table_name),
+                filter_expr,
+                order_key,
+            )
+        };
+
+        let mut conn = self.conn.lock().await;
+
+        // Set session timezone to UTC
+        conn.exec_drop("SET time_zone = \"+00:00\"", ()).await?;
+
+        if start_pk_row.is_none() {
+            let rs_stream = sql.stream::<mysql_async::Row, _>(&mut *conn).await?;
+            let row_stream = rs_stream.map(|row| {
+                // convert mysql row into OwnedRow
+                let mut row = row?;
+                Ok::<_, ConnectorError>(mysql_row_to_owned_row(&mut row, &self.rw_schema))
+            });
+
+            pin_mut!(row_stream);
+            #[for_await]
+            for row in row_stream {
+                let row = row?;
+                yield row;
+            }
+        } else {
+            let field_map = self
+                .rw_schema
+                .fields
+                .iter()
+                .map(|f| (f.name.as_str(), f.data_type.clone()))
+                .collect::<HashMap<_, _>>();
+
+            // fill in start primary key params
+            let params: Vec<_> = primary_keys
+                .iter()
+                .zip_eq_fast(start_pk_row.unwrap().into_iter())
+                .map(|(pk, datum)| {
+                    if let Some(value) = datum {
+                        let ty = field_map.get(pk.as_str()).unwrap();
+                        let val = match ty {
+                            DataType::Boolean => Value::from(value.into_bool()),
+                            DataType::Int16 => Value::from(value.into_int16()),
+                            DataType::Int32 => Value::from(value.into_int32()),
+                            DataType::Int64 => Value::from(value.into_int64()),
+                            DataType::Float32 => Value::from(value.into_float32().into_inner()),
+                            DataType::Float64 => Value::from(value.into_float64().into_inner()),
+                            DataType::Varchar => Value::from(String::from(value.into_utf8())),
+                            DataType::Date => Value::from(value.into_date().0),
+                            DataType::Time => Value::from(value.into_time().0),
+                            DataType::Timestamp => Value::from(value.into_timestamp().0),
+                            _ => bail!("unsupported primary key data type: {}", ty),
+                        };
+                        ConnectorResult::Ok((pk.to_lowercase(), val))
+                    } else {
+                        bail!("primary key {} cannot be null", pk);
+                    }
+                })
+                .try_collect::<_, _, ConnectorError>()?;
+
+            tracing::debug!("snapshot read params: {:?}", &params);
+            let rs_stream = sql
+                .with(Params::from(params))
+                .stream::<mysql_async::Row, _>(&mut *conn)
+                .await?;
+
+            let row_stream = rs_stream.map(|row| {
+                // convert mysql row into OwnedRow
+                let mut row = row?;
+                Ok::<_, ConnectorError>(mysql_row_to_owned_row(&mut row, &self.rw_schema))
+            });
+
+            pin_mut!(row_stream);
+            #[for_await]
+            for row in row_stream {
+                let row = row?;
+                yield row;
+            }
+        };
+    }
+
+    // mysql cannot leverage the given key to narrow down the range of scan,
+    // we need to rewrite the comparison conditions by our own.
+    // (a, b) > (x, y) => (`a` > x) OR ((`a` = x) AND (`b` > y))
+    fn filter_expression(columns: &[String]) -> String {
+        let mut conditions = vec![];
+        // push the first condition
+        conditions.push(format!(
+            "({} > :{})",
+            Self::quote_column(&columns[0]),
+            columns[0].to_lowercase()
+        ));
+        for i in 2..=columns.len() {
+            // '=' condition
+            let mut condition = String::new();
+            for (j, col) in columns.iter().enumerate().take(i - 1) {
+                if j == 0 {
+                    condition.push_str(&format!(
+                        "({} = :{})",
+                        Self::quote_column(col),
+                        col.to_lowercase()
+                    ));
+                } else {
+                    condition.push_str(&format!(
+                        " AND ({} = :{})",
+                        Self::quote_column(col),
+                        col.to_lowercase()
+                    ));
+                }
+            }
+            // '>' condition
+            condition.push_str(&format!(
+                " AND ({} > :{})",
+                Self::quote_column(&columns[i - 1]),
+                columns[i - 1].to_lowercase()
+            ));
+            conditions.push(format!("({})", condition));
+        }
+        if columns.len() > 1 {
+            conditions.join(" OR ")
+        } else {
+            conditions.join("")
+        }
+    }
+
+    fn quote_column(column: &str) -> String {
+        format!("`{}`", column)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use futures::pin_mut;
+    use futures_async_stream::for_await;
+    use maplit::{convert_args, hashmap};
+    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, Schema};
+    use risingwave_common::types::DataType;
+
+    use crate::source::cdc::external::mysql::MySqlExternalTable;
+    use crate::source::cdc::external::{
+        CdcOffset, ExternalTableConfig, ExternalTableReader, MySqlExternalTableReader, MySqlOffset,
+        SchemaTableName,
+    };
+
+    #[ignore]
+    #[tokio::test]
+    async fn test_mysql_schema() {
+        let config = ExternalTableConfig {
+            connector: "mysql-cdc".to_string(),
+            host: "localhost".to_string(),
+            port: "8306".to_string(),
+            username: "root".to_string(),
+            password: "123456".to_string(),
+            database: "mydb".to_string(),
+            schema: "".to_string(),
+            table: "part".to_string(),
+            sslmode: Default::default(),
+        };
+
+        let table = MySqlExternalTable::connect(config).await.unwrap();
+        println!("columns: {:?}", &table.column_descs);
+        println!("primary keys: {:?}", &table.pk_names);
+    }
+
+    #[test]
+    fn test_mysql_filter_expr() {
+        let cols = vec!["id".to_string()];
+        let expr = MySqlExternalTableReader::filter_expression(&cols);
+        assert_eq!(expr, "(`id` > :id)");
+
+        let cols = vec!["aa".to_string(), "bb".to_string(), "cc".to_string()];
+        let expr = MySqlExternalTableReader::filter_expression(&cols);
+        assert_eq!(
+            expr,
+            "(`aa` > :aa) OR ((`aa` = :aa) AND (`bb` > :bb)) OR ((`aa` = :aa) AND (`bb` = :bb) AND (`cc` > :cc))"
+        );
+    }
+
+    #[test]
+    fn test_mysql_binlog_offset() {
+        let off0_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000001", "pos": 105622, "snapshot": true }, "isHeartbeat": false }"#;
+        let off1_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000007", "pos": 1062363217, "snapshot": true }, "isHeartbeat": false }"#;
+        let off2_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000007", "pos": 659687560, "snapshot": true }, "isHeartbeat": false }"#;
+        let off3_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000008", "pos": 7665875, "snapshot": true }, "isHeartbeat": false }"#;
+        let off4_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000008", "pos": 7665875, "snapshot": true }, "isHeartbeat": false }"#;
+
+        let off0 = CdcOffset::MySql(MySqlOffset::parse_debezium_offset(off0_str).unwrap());
+        let off1 = CdcOffset::MySql(MySqlOffset::parse_debezium_offset(off1_str).unwrap());
+        let off2 = CdcOffset::MySql(MySqlOffset::parse_debezium_offset(off2_str).unwrap());
+        let off3 = CdcOffset::MySql(MySqlOffset::parse_debezium_offset(off3_str).unwrap());
+        let off4 = CdcOffset::MySql(MySqlOffset::parse_debezium_offset(off4_str).unwrap());
+
+        assert!(off0 <= off1);
+        assert!(off1 > off2);
+        assert!(off2 < off3);
+        assert_eq!(off3, off4);
+    }
+
+    // manual test case
+    #[ignore]
+    #[tokio::test]
+    async fn test_mysql_table_reader() {
+        let columns = vec![
+            ColumnDesc::named("v1", ColumnId::new(1), DataType::Int32),
+            ColumnDesc::named("v2", ColumnId::new(2), DataType::Decimal),
+            ColumnDesc::named("v3", ColumnId::new(3), DataType::Varchar),
+            ColumnDesc::named("v4", ColumnId::new(4), DataType::Date),
+        ];
+        let rw_schema = Schema {
+            fields: columns.iter().map(Field::from).collect(),
+        };
+        let props: HashMap<String, String> = convert_args!(hashmap!(
+                "hostname" => "localhost",
+                "port" => "8306",
+                "username" => "root",
+                "password" => "123456",
+                "database.name" => "mytest",
+                "table.name" => "t1"));
+
+        let config =
+            serde_json::from_value::<ExternalTableConfig>(serde_json::to_value(props).unwrap())
+                .unwrap();
+        let reader = MySqlExternalTableReader::new(config, rw_schema)
+            .await
+            .unwrap();
+        let offset = reader.current_cdc_offset().await.unwrap();
+        println!("BinlogOffset: {:?}", offset);
+
+        let off0_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000001", "pos": 105622, "snapshot": true }, "isHeartbeat": false }"#;
+        let parser = MySqlExternalTableReader::get_cdc_offset_parser();
+        println!("parsed offset: {:?}", parser(off0_str).unwrap());
+        let table_name = SchemaTableName {
+            schema_name: "mytest".to_string(),
+            table_name: "t1".to_string(),
+        };
+
+        let stream = reader.snapshot_read(table_name, None, vec!["v1".to_string()], 1000);
+        pin_mut!(stream);
+        #[for_await]
+        for row in stream {
+            println!("OwnedRow: {:?}", row);
+        }
+    }
+}
diff --git a/src/connector/src/source/cdc/external/postgres.rs b/src/connector/src/source/cdc/external/postgres.rs
index 9df4b32e3f5ec..3d827bafbccb3 100644
--- a/src/connector/src/source/cdc/external/postgres.rs
+++ b/src/connector/src/source/cdc/external/postgres.rs
@@ -15,23 +15,29 @@
 use std::cmp::Ordering;
 use std::collections::HashMap;
 
-use anyhow::Context;
+use anyhow::{anyhow, Context};
 use futures::stream::BoxStream;
 use futures::{pin_mut, StreamExt};
 use futures_async_stream::try_stream;
 use itertools::Itertools;
 use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode};
 use postgres_openssl::MakeTlsConnector;
-use risingwave_common::catalog::Schema;
+use risingwave_common::catalog::{ColumnDesc, ColumnId, Schema};
 use risingwave_common::row::{OwnedRow, Row};
-use risingwave_common::types::DatumRef;
+use risingwave_common::types::{DataType, StructType};
+use risingwave_common::util::iter_util::ZipEqFast;
+use sea_schema::postgres::def::{ColumnType, TableInfo};
+use sea_schema::postgres::discovery::SchemaDiscovery;
 use serde_derive::{Deserialize, Serialize};
+use sqlx::postgres::{PgConnectOptions, PgSslMode};
+use sqlx::PgPool;
 use thiserror_ext::AsReport;
 use tokio_postgres::types::PgLsn;
-use tokio_postgres::NoTls;
+use tokio_postgres::{NoTls, Statement};
 
 use crate::error::{ConnectorError, ConnectorResult};
 use crate::parser::postgres_row_to_owned_row;
+use crate::parser::scalar_adapter::ScalarAdapter;
 #[cfg(not(madsim))]
 use crate::source::cdc::external::maybe_tls_connector::MaybeMakeTlsConnector;
 use crate::source::cdc::external::{
@@ -72,23 +78,165 @@ impl PostgresOffset {
     }
 }
 
-#[derive(Debug)]
+pub struct PostgresExternalTable {
+    column_descs: Vec<ColumnDesc>,
+    pk_names: Vec<String>,
+}
+
+impl PostgresExternalTable {
+    pub async fn connect(config: ExternalTableConfig) -> ConnectorResult<Self> {
+        tracing::debug!("connect to postgres external table");
+        let options = PgConnectOptions::new()
+            .username(&config.username)
+            .password(&config.password)
+            .host(&config.host)
+            .port(config.port.parse::<u16>().unwrap())
+            .database(&config.database)
+            .ssl_mode(match config.sslmode {
+                SslMode::Disabled => PgSslMode::Disable,
+                SslMode::Preferred => PgSslMode::Prefer,
+                SslMode::Required => PgSslMode::Require,
+            });
+
+        let connection = PgPool::connect_with(options).await?;
+        let schema_discovery = SchemaDiscovery::new(connection, config.schema.as_str());
+        // fetch column schema and primary key
+        let empty_map = HashMap::new();
+        let table_schema = schema_discovery
+            .discover_table(
+                TableInfo {
+                    name: config.table.clone(),
+                    of_type: None,
+                },
+                &empty_map,
+            )
+            .await?;
+
+        let mut column_descs = vec![];
+        for col in &table_schema.columns {
+            let data_type = type_to_rw_type(&col.col_type)?;
+            column_descs.push(ColumnDesc::named(
+                col.name.clone(),
+                ColumnId::placeholder(),
+                data_type,
+            ));
+        }
+
+        if table_schema.primary_key_constraints.is_empty() {
+            return Err(anyhow!("Postgres table doesn't define the primary key").into());
+        }
+        let mut pk_names = vec![];
+        table_schema.primary_key_constraints.iter().for_each(|pk| {
+            pk_names.extend(pk.columns.clone());
+        });
+
+        Ok(Self {
+            column_descs,
+            pk_names,
+        })
+    }
+
+    pub fn column_descs(&self) -> &Vec<ColumnDesc> {
+        &self.column_descs
+    }
+
+    pub fn pk_names(&self) -> &Vec<String> {
+        &self.pk_names
+    }
+}
+
+fn type_to_rw_type(col_type: &ColumnType) -> ConnectorResult<DataType> {
+    let dtype = match col_type {
+        ColumnType::SmallInt | ColumnType::SmallSerial => DataType::Int16,
+        ColumnType::Integer | ColumnType::Serial => DataType::Int32,
+        ColumnType::BigInt | ColumnType::BigSerial => DataType::Int64,
+        ColumnType::Money | ColumnType::Decimal(_) | ColumnType::Numeric(_) => DataType::Decimal,
+        ColumnType::Real => DataType::Float32,
+        ColumnType::DoublePrecision => DataType::Float64,
+        ColumnType::Varchar(_) | ColumnType::Char(_) | ColumnType::Text => DataType::Varchar,
+        ColumnType::Bytea => DataType::Bytea,
+        ColumnType::Timestamp(_) => DataType::Timestamp,
+        ColumnType::TimestampWithTimeZone(_) => DataType::Timestamptz,
+        ColumnType::Date => DataType::Date,
+        ColumnType::Time(_) | ColumnType::TimeWithTimeZone(_) => DataType::Time,
+        ColumnType::Interval(_) => DataType::Interval,
+        ColumnType::Boolean => DataType::Boolean,
+        ColumnType::Point => DataType::Struct(StructType::new(vec![
+            ("x", DataType::Float32),
+            ("y", DataType::Float32),
+        ])),
+        ColumnType::Uuid => DataType::Varchar,
+        ColumnType::Xml => DataType::Varchar,
+        ColumnType::Json => DataType::Jsonb,
+        ColumnType::JsonBinary => DataType::Jsonb,
+        ColumnType::Array(def) => {
+            let item_type = match def.col_type.as_ref() {
+                Some(ty) => type_to_rw_type(ty.as_ref())?,
+                None => {
+                    return Err(anyhow!("ARRAY type missing element type").into());
+                }
+            };
+
+            DataType::List(Box::new(item_type))
+        }
+        ColumnType::PgLsn => DataType::Int64,
+        ColumnType::Cidr
+        | ColumnType::Inet
+        | ColumnType::MacAddr
+        | ColumnType::MacAddr8
+        | ColumnType::Int4Range
+        | ColumnType::Int8Range
+        | ColumnType::NumRange
+        | ColumnType::TsRange
+        | ColumnType::TsTzRange
+        | ColumnType::DateRange
+        | ColumnType::Enum(_) => DataType::Varchar,
+
+        ColumnType::Line => {
+            return Err(anyhow!("LINE type not supported").into());
+        }
+        ColumnType::Lseg => {
+            return Err(anyhow!("LSEG type not supported").into());
+        }
+        ColumnType::Box => {
+            return Err(anyhow!("BOX type not supported").into());
+        }
+        ColumnType::Path => {
+            return Err(anyhow!("PATH type not supported").into());
+        }
+        ColumnType::Polygon => {
+            return Err(anyhow!("POLYGON type not supported").into());
+        }
+        ColumnType::Circle => {
+            return Err(anyhow!("CIRCLE type not supported").into());
+        }
+        ColumnType::Bit(_) => {
+            return Err(anyhow!("BIT type not supported").into());
+        }
+        ColumnType::TsVector => {
+            return Err(anyhow!("TSVECTOR type not supported").into());
+        }
+        ColumnType::TsQuery => {
+            return Err(anyhow!("TSQUERY type not supported").into());
+        }
+        ColumnType::Unknown(name) => {
+            // NOTES: user-defined enum type is classified as `Unknown`
+            tracing::warn!("Unknown Postgres data type: {name}, map to varchar");
+            DataType::Varchar
+        }
+    };
+
+    Ok(dtype)
+}
+
 pub struct PostgresExternalTableReader {
-    config: ExternalTableConfig,
     rw_schema: Schema,
     field_names: String,
-
+    prepared_scan_stmt: Statement,
     client: tokio::sync::Mutex<tokio_postgres::Client>,
 }
 
 impl ExternalTableReader for PostgresExternalTableReader {
-    fn get_normalized_table_name(&self, table_name: &SchemaTableName) -> String {
-        format!(
-            "\"{}\".\"{}\"",
-            table_name.schema_name, table_name.table_name
-        )
-    }
-
     async fn current_cdc_offset(&self) -> ConnectorResult<CdcOffset> {
         let mut client = self.client.lock().await;
         // start a transaction to read current lsn and txid
@@ -122,41 +270,47 @@ impl ExternalTableReader for PostgresExternalTableReader {
 
 impl PostgresExternalTableReader {
     pub async fn new(
-        properties: HashMap<String, String>,
+        config: ExternalTableConfig,
         rw_schema: Schema,
+        pk_indices: Vec<usize>,
+        scan_limit: u32,
     ) -> ConnectorResult<Self> {
-        tracing::debug!(?rw_schema, "create postgres external table reader");
-
-        let config = serde_json::from_value::<ExternalTableConfig>(
-            serde_json::to_value(properties).unwrap(),
-        )
-        .context("failed to extract postgres connector properties")?;
-
-        let database_url = format!(
-            "postgresql://{}:{}@{}:{}/{}?sslmode={}",
-            config.username,
-            config.password,
-            config.host,
-            config.port,
-            config.database,
-            config.sslmode
+        tracing::info!(
+            ?rw_schema,
+            ?pk_indices,
+            "create postgres external table reader"
         );
 
+        let mut pg_config = tokio_postgres::Config::new();
+        pg_config
+            .user(&config.username)
+            .password(&config.password)
+            .host(&config.host)
+            .port(config.port.parse::<u16>().unwrap())
+            .dbname(&config.database);
+
         #[cfg(not(madsim))]
         let connector = match config.sslmode {
-            SslMode::Disable => MaybeMakeTlsConnector::NoTls(NoTls),
-            SslMode::Prefer => match SslConnector::builder(SslMethod::tls()) {
-                Ok(mut builder) => {
-                    // disable certificate verification for `prefer`
-                    builder.set_verify(SslVerifyMode::NONE);
-                    MaybeMakeTlsConnector::Tls(MakeTlsConnector::new(builder.build()))
-                }
-                Err(e) => {
-                    tracing::warn!(error = %e.as_report(), "SSL connector error");
-                    MaybeMakeTlsConnector::NoTls(NoTls)
+            SslMode::Disabled => {
+                pg_config.ssl_mode(tokio_postgres::config::SslMode::Disable);
+                MaybeMakeTlsConnector::NoTls(NoTls)
+            }
+            SslMode::Preferred => {
+                pg_config.ssl_mode(tokio_postgres::config::SslMode::Prefer);
+                match SslConnector::builder(SslMethod::tls()) {
+                    Ok(mut builder) => {
+                        // disable certificate verification for `prefer`
+                        builder.set_verify(SslVerifyMode::NONE);
+                        MaybeMakeTlsConnector::Tls(MakeTlsConnector::new(builder.build()))
+                    }
+                    Err(e) => {
+                        tracing::warn!(error = %e.as_report(), "SSL connector error");
+                        MaybeMakeTlsConnector::NoTls(NoTls)
+                    }
                 }
-            },
-            SslMode::Require => {
+            }
+            SslMode::Required => {
+                pg_config.ssl_mode(tokio_postgres::config::SslMode::Require);
                 let mut builder = SslConnector::builder(SslMethod::tls())?;
                 // disable certificate verification for `require`
                 builder.set_verify(SslVerifyMode::NONE);
@@ -166,7 +320,7 @@ impl PostgresExternalTableReader {
         #[cfg(madsim)]
         let connector = NoTls;
 
-        let (client, connection) = tokio_postgres::connect(&database_url, connector).await?;
+        let (client, connection) = pg_config.connect(connector).await?;
 
         tokio::spawn(async move {
             if let Err(e) = connection.await {
@@ -180,14 +334,43 @@ impl PostgresExternalTableReader {
             .map(|f| Self::quote_column(&f.name))
             .join(",");
 
+        // prepare once
+        let prepared_scan_stmt = {
+            let primary_keys = pk_indices
+                .iter()
+                .map(|i| rw_schema.fields[*i].name.clone())
+                .collect_vec();
+
+            let table_name = SchemaTableName {
+                schema_name: config.schema.clone(),
+                table_name: config.table.clone(),
+            };
+            let order_key = primary_keys.iter().join(",");
+            let scan_sql = format!(
+                "SELECT {} FROM {} WHERE {} ORDER BY {} LIMIT {scan_limit}",
+                field_names,
+                Self::get_normalized_table_name(&table_name),
+                Self::filter_expression(&primary_keys),
+                order_key,
+            );
+            client.prepare(&scan_sql).await?
+        };
+
         Ok(Self {
-            config,
             rw_schema,
             field_names,
+            prepared_scan_stmt,
             client: tokio::sync::Mutex::new(client),
         })
     }
 
+    pub fn get_normalized_table_name(table_name: &SchemaTableName) -> String {
+        format!(
+            "\"{}\".\"{}\"",
+            table_name.schema_name, table_name.table_name
+        )
+    }
+
     pub fn get_cdc_offset_parser() -> CdcOffsetParseFunc {
         Box::new(move |offset| {
             Ok(CdcOffset::Postgres(PostgresOffset::parse_debezium_offset(
@@ -205,33 +388,35 @@ impl PostgresExternalTableReader {
         limit: u32,
     ) {
         let order_key = primary_keys.iter().join(",");
-        let sql = if start_pk_row.is_none() {
-            format!(
-                "SELECT {} FROM {} ORDER BY {} LIMIT {limit}",
-                self.field_names,
-                self.get_normalized_table_name(&table_name),
-                order_key,
-            )
-        } else {
-            let filter_expr = Self::filter_expression(&primary_keys);
-            format!(
-                "SELECT {} FROM {} WHERE {} ORDER BY {} LIMIT {limit}",
-                self.field_names,
-                self.get_normalized_table_name(&table_name),
-                filter_expr,
-                order_key,
-            )
-        };
-
         let client = self.client.lock().await;
         client.execute("set time zone '+00:00'", &[]).await?;
 
-        let params: Vec<DatumRef<'_>> = match start_pk_row {
-            Some(ref pk_row) => pk_row.iter().collect_vec(),
-            None => Vec::new(),
+        let stream = match start_pk_row {
+            Some(ref pk_row) => {
+                let params: Vec<Option<ScalarAdapter>> = pk_row
+                    .iter()
+                    .zip_eq_fast(self.prepared_scan_stmt.params())
+                    .map(|(datum, ty)| {
+                        datum
+                            .map(|scalar| ScalarAdapter::from_scalar(scalar, ty))
+                            .transpose()
+                    })
+                    .try_collect()?;
+
+                client.query_raw(&self.prepared_scan_stmt, &params).await?
+            }
+            None => {
+                let sql = format!(
+                    "SELECT {} FROM {} ORDER BY {} LIMIT {limit}",
+                    self.field_names,
+                    Self::get_normalized_table_name(&table_name),
+                    order_key,
+                );
+                let params: Vec<Option<ScalarAdapter>> = vec![];
+                client.query_raw(&sql, &params).await?
+            }
         };
 
-        let stream = client.query_raw(&sql, &params).await?;
         let row_stream = stream.map(|row| {
             let row = row?;
             Ok::<_, crate::error::ConnectorError>(postgres_row_to_owned_row(row, &self.rw_schema))
@@ -267,6 +452,9 @@ impl PostgresExternalTableReader {
 
 #[cfg(test)]
 mod tests {
+
+    use std::collections::HashMap;
+
     use futures::pin_mut;
     use futures_async_stream::for_await;
     use maplit::{convert_args, hashmap};
@@ -274,8 +462,31 @@ mod tests {
     use risingwave_common::row::OwnedRow;
     use risingwave_common::types::{DataType, ScalarImpl};
 
-    use crate::source::cdc::external::postgres::{PostgresExternalTableReader, PostgresOffset};
-    use crate::source::cdc::external::{ExternalTableReader, SchemaTableName};
+    use crate::source::cdc::external::postgres::{
+        PostgresExternalTable, PostgresExternalTableReader, PostgresOffset,
+    };
+    use crate::source::cdc::external::{ExternalTableConfig, ExternalTableReader, SchemaTableName};
+
+    #[ignore]
+    #[tokio::test]
+    async fn test_postgres_schema() {
+        let config = ExternalTableConfig {
+            connector: "postgres-cdc".to_string(),
+            host: "localhost".to_string(),
+            port: "8432".to_string(),
+            username: "myuser".to_string(),
+            password: "123456".to_string(),
+            database: "mydb".to_string(),
+            schema: "public".to_string(),
+            table: "mytest".to_string(),
+            sslmode: Default::default(),
+        };
+
+        let table = PostgresExternalTable::connect(config).await.unwrap();
+
+        println!("columns: {:?}", &table.column_descs);
+        println!("primary keys: {:?}", &table.pk_names);
+    }
 
     #[test]
     fn test_postgres_offset() {
@@ -317,7 +528,7 @@ mod tests {
             fields: columns.iter().map(Field::from).collect(),
         };
 
-        let props = convert_args!(hashmap!(
+        let props: HashMap<String, String> = convert_args!(hashmap!(
                 "hostname" => "localhost",
                 "port" => "8432",
                 "username" => "myuser",
@@ -325,7 +536,11 @@ mod tests {
                 "database.name" => "mydb",
                 "schema.name" => "public",
                 "table.name" => "t1"));
-        let reader = PostgresExternalTableReader::new(props, rw_schema)
+
+        let config =
+            serde_json::from_value::<ExternalTableConfig>(serde_json::to_value(props).unwrap())
+                .unwrap();
+        let reader = PostgresExternalTableReader::new(config, rw_schema, vec![0, 1], 1000)
             .await
             .unwrap();
 
diff --git a/src/connector/src/source/cdc/mod.rs b/src/connector/src/source/cdc/mod.rs
index decbcffebc36d..2a4200b5cc4a0 100644
--- a/src/connector/src/source/cdc/mod.rs
+++ b/src/connector/src/source/cdc/mod.rs
@@ -18,20 +18,20 @@ pub mod jni_source;
 pub mod source;
 pub mod split;
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::marker::PhantomData;
 
 pub use enumerator::*;
 use itertools::Itertools;
-use risingwave_common::catalog::{ColumnDesc, Field, Schema};
 use risingwave_pb::catalog::PbSource;
 use risingwave_pb::connector_service::{PbSourceType, PbTableSchema, SourceType, TableSchema};
+use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn;
 use risingwave_pb::plan_common::ExternalTableDesc;
 use simd_json::prelude::ArrayTrait;
 pub use source::*;
 
 use crate::error::ConnectorResult;
-use crate::source::{SourceProperties, SplitImpl, TryFromHashmap};
+use crate::source::{SourceProperties, SplitImpl, TryFromBTreeMap};
 use crate::{for_all_classified_sources, impl_cdc_source_type};
 
 pub const CDC_CONNECTOR_NAME_SUFFIX: &str = "-cdc";
@@ -40,8 +40,11 @@ pub const CDC_SNAPSHOT_BACKFILL: &str = "rw_cdc_backfill";
 pub const CDC_SHARING_MODE_KEY: &str = "rw.sharing.mode.enable";
 // User can set snapshot='false' to disable cdc backfill
 pub const CDC_BACKFILL_ENABLE_KEY: &str = "snapshot";
+pub const CDC_BACKFILL_SNAPSHOT_INTERVAL_KEY: &str = "snapshot.interval";
+pub const CDC_BACKFILL_SNAPSHOT_BATCH_SIZE_KEY: &str = "snapshot.batch_size";
 // We enable transaction for shared cdc source by default
 pub const CDC_TRANSACTIONAL_KEY: &str = "transactional";
+pub const CDC_WAIT_FOR_STREAMING_START_TIMEOUT: &str = "cdc.source.wait.streaming.start.timeout";
 
 pub const MYSQL_CDC_CONNECTOR: &str = Mysql::CDC_CONNECTOR_NAME;
 pub const POSTGRES_CDC_CONNECTOR: &str = Postgres::CDC_CONNECTOR_NAME;
@@ -82,7 +85,7 @@ impl CdcSourceType {
 #[derive(Clone, Debug, Default)]
 pub struct CdcProperties<T: CdcSourceTypeTrait> {
     /// Properties specified in the WITH clause by user
-    pub properties: HashMap<String, String>,
+    pub properties: BTreeMap<String, String>,
 
     /// Schema of the source specified by users
     pub table_schema: TableSchema,
@@ -96,9 +99,25 @@ pub struct CdcProperties<T: CdcSourceTypeTrait> {
     pub _phantom: PhantomData<T>,
 }
 
-impl<T: CdcSourceTypeTrait> TryFromHashmap for CdcProperties<T> {
-    fn try_from_hashmap(
-        properties: HashMap<String, String>,
+pub fn table_schema_exclude_additional_columns(table_schema: &TableSchema) -> TableSchema {
+    TableSchema {
+        columns: table_schema
+            .columns
+            .iter()
+            .filter(|col| {
+                col.additional_column
+                    .as_ref()
+                    .is_some_and(|val| val.column_type.is_none())
+            })
+            .cloned()
+            .collect(),
+        pk_indices: table_schema.pk_indices.clone(),
+    }
+}
+
+impl<T: CdcSourceTypeTrait> TryFromBTreeMap for CdcProperties<T> {
+    fn try_from_btreemap(
+        properties: BTreeMap<String, String>,
         _deny_unknown_fields: bool,
     ) -> ConnectorResult<Self> {
         let is_share_source = properties
@@ -144,6 +163,12 @@ where
                 .columns
                 .iter()
                 .flat_map(|col| &col.column_desc)
+                .filter(|col| {
+                    !matches!(
+                        col.generated_or_default_column,
+                        Some(GeneratedOrDefaultColumn::GeneratedColumn(_))
+                    )
+                })
                 .cloned()
                 .collect(),
             pk_indices,
@@ -155,11 +180,20 @@ where
     }
 
     fn init_from_pb_cdc_table_desc(&mut self, table_desc: &ExternalTableDesc) {
-        let properties: HashMap<String, String> =
-            table_desc.connect_properties.clone().into_iter().collect();
+        let properties = table_desc.connect_properties.clone();
 
         let table_schema = TableSchema {
-            columns: table_desc.columns.clone(),
+            columns: table_desc
+                .columns
+                .iter()
+                .filter(|col| {
+                    !matches!(
+                        col.generated_or_default_column,
+                        Some(GeneratedOrDefaultColumn::GeneratedColumn(_))
+                    )
+                })
+                .cloned()
+                .collect(),
             pk_indices: table_desc.stream_key.clone(),
         };
 
@@ -181,16 +215,4 @@ impl<T: CdcSourceTypeTrait> CdcProperties<T> {
     pub fn get_source_type_pb(&self) -> SourceType {
         SourceType::from(T::source_type())
     }
-
-    pub fn schema(&self) -> Schema {
-        Schema {
-            fields: self
-                .table_schema
-                .columns
-                .iter()
-                .map(ColumnDesc::from)
-                .map(Field::from)
-                .collect(),
-        }
-    }
 }
diff --git a/src/connector/src/source/cdc/source/message.rs b/src/connector/src/source/cdc/source/message.rs
index 5df937a83fbe8..f12d18339b527 100644
--- a/src/connector/src/source/cdc/source/message.rs
+++ b/src/connector/src/source/cdc/source/message.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use risingwave_common::types::{DatumRef, ScalarRefImpl, Timestamptz};
 use risingwave_pb::connector_service::CdcMessage;
 
 use crate::source::base::SourceMessage;
@@ -19,6 +20,8 @@ use crate::source::SourceMeta;
 
 #[derive(Debug, Clone)]
 pub struct DebeziumCdcMeta {
+    db_name_prefix_len: usize,
+
     pub full_table_name: String,
     // extracted from `payload.source.ts_ms`, the time that the change event was made in the database
     pub source_ts_ms: i64,
@@ -26,6 +29,37 @@ pub struct DebeziumCdcMeta {
     pub is_transaction_meta: bool,
 }
 
+impl DebeziumCdcMeta {
+    pub fn extract_timestamp(&self) -> DatumRef<'_> {
+        Some(ScalarRefImpl::Timestamptz(
+            Timestamptz::from_millis(self.source_ts_ms).unwrap(),
+        ))
+    }
+
+    pub fn extract_database_name(&self) -> DatumRef<'_> {
+        Some(ScalarRefImpl::Utf8(
+            &self.full_table_name.as_str()[0..self.db_name_prefix_len],
+        ))
+    }
+
+    pub fn extract_table_name(&self) -> DatumRef<'_> {
+        Some(ScalarRefImpl::Utf8(
+            &self.full_table_name.as_str()[self.db_name_prefix_len..],
+        ))
+    }
+
+    pub fn new(full_table_name: String, source_ts_ms: i64, is_transaction_meta: bool) -> Self {
+        // full_table_name is in the format of `database_name.table_name`
+        let db_name_prefix_len = full_table_name.as_str().find('.').unwrap_or(0);
+        Self {
+            db_name_prefix_len,
+            full_table_name,
+            source_ts_ms,
+            is_transaction_meta,
+        }
+    }
+}
+
 impl From<CdcMessage> for SourceMessage {
     fn from(message: CdcMessage) -> Self {
         SourceMessage {
@@ -41,11 +75,11 @@ impl From<CdcMessage> for SourceMessage {
             },
             offset: message.offset,
             split_id: message.partition.into(),
-            meta: SourceMeta::DebeziumCdc(DebeziumCdcMeta {
-                full_table_name: message.full_table_name,
-                source_ts_ms: message.source_ts_ms,
-                is_transaction_meta: message.is_transaction_meta,
-            }),
+            meta: SourceMeta::DebeziumCdc(DebeziumCdcMeta::new(
+                message.full_table_name,
+                message.source_ts_ms,
+                message.is_transaction_meta,
+            )),
         }
     }
 }
diff --git a/src/connector/src/source/cdc/source/reader.rs b/src/connector/src/source/cdc/source/reader.rs
index cf2b5c3d17e00..1f66c5d3d11ea 100644
--- a/src/connector/src/source/cdc/source/reader.rs
+++ b/src/connector/src/source/cdc/source/reader.rs
@@ -33,19 +33,23 @@ use crate::parser::ParserConfig;
 use crate::source::base::SourceMessage;
 use crate::source::cdc::{CdcProperties, CdcSourceType, CdcSourceTypeTrait, DebeziumCdcSplit};
 use crate::source::{
-    into_chunk_stream, BoxChunkSourceStream, Column, CommonSplitReader, SourceContextRef, SplitId,
-    SplitMetaData, SplitReader,
+    into_chunk_stream, BoxChunkSourceStream, Column, SourceContextRef, SplitId, SplitMetaData,
+    SplitReader,
 };
 
 pub struct CdcSplitReader<T: CdcSourceTypeTrait> {
     source_id: u64,
+    #[expect(dead_code)]
     start_offset: Option<String>,
     // host address of worker node for a Citus cluster
+    #[expect(dead_code)]
     server_addr: Option<String>,
+    #[expect(dead_code)]
     conn_props: CdcProperties<T>,
-
+    #[expect(dead_code)]
     split_id: SplitId,
     // whether the full snapshot phase is done
+    #[expect(dead_code)]
     snapshot_done: bool,
     parser_config: ParserConfig,
     source_ctx: SourceContextRef,
@@ -190,11 +194,11 @@ impl<T: CdcSourceTypeTrait> SplitReader for CdcSplitReader<T> {
     fn into_stream(self) -> BoxChunkSourceStream {
         let parser_config = self.parser_config.clone();
         let source_context = self.source_ctx.clone();
-        into_chunk_stream(self, parser_config, source_context)
+        into_chunk_stream(self.into_data_stream(), parser_config, source_context)
     }
 }
 
-impl<T: CdcSourceTypeTrait> CommonSplitReader for CdcSplitReader<T> {
+impl<T: CdcSourceTypeTrait> CdcSplitReader<T> {
     #[try_stream(ok = Vec<SourceMessage>, error = ConnectorError)]
     async fn into_data_stream(self) {
         let source_type = T::source_type();
diff --git a/src/connector/src/source/common.rs b/src/connector/src/source/common.rs
index 8fcd1318401f2..abb80221e8fae 100644
--- a/src/connector/src/source/common.rs
+++ b/src/connector/src/source/common.rs
@@ -18,15 +18,13 @@ use risingwave_common::array::StreamChunk;
 
 use crate::error::{ConnectorError, ConnectorResult};
 use crate::parser::ParserConfig;
-use crate::source::{SourceContextRef, SourceMessage, SplitReader};
-
-pub(crate) trait CommonSplitReader: SplitReader + 'static {
-    fn into_data_stream(self) -> impl Stream<Item = ConnectorResult<Vec<SourceMessage>>> + Send;
-}
+use crate::source::{SourceContextRef, SourceMessage};
 
+/// Utility function to convert [`SourceMessage`] stream (got from specific connector's [`SplitReader`](super::SplitReader))
+/// into [`StreamChunk`] stream (by invoking [`ByteStreamSourceParserImpl`](crate::parser::ByteStreamSourceParserImpl)).
 #[try_stream(boxed, ok = StreamChunk, error = ConnectorError)]
 pub(crate) async fn into_chunk_stream(
-    reader: impl CommonSplitReader,
+    data_stream: impl Stream<Item = ConnectorResult<Vec<SourceMessage>>> + Send + 'static,
     parser_config: ParserConfig,
     source_ctx: SourceContextRef,
 ) {
@@ -36,8 +34,7 @@ pub(crate) async fn into_chunk_stream(
     let source_name = source_ctx.source_name.to_string();
     let metrics = source_ctx.metrics.clone();
 
-    let data_stream = reader.into_data_stream();
-
+    // add metrics to the data stream
     let data_stream = data_stream
         .inspect_ok(move |data_batch| {
             let mut by_split_id = std::collections::HashMap::new();
diff --git a/src/connector/src/source/datagen/source/reader.rs b/src/connector/src/source/datagen/source/reader.rs
index 87f798d59f38b..d3115f504f32e 100644
--- a/src/connector/src/source/datagen/source/reader.rs
+++ b/src/connector/src/source/datagen/source/reader.rs
@@ -27,12 +27,13 @@ use crate::source::data_gen_util::spawn_data_generation_stream;
 use crate::source::datagen::source::SEQUENCE_FIELD_KIND;
 use crate::source::datagen::{DatagenProperties, DatagenSplit, FieldDesc};
 use crate::source::{
-    into_chunk_stream, BoxChunkSourceStream, Column, CommonSplitReader, DataType, SourceContextRef,
-    SourceMessage, SplitId, SplitMetaData, SplitReader,
+    into_chunk_stream, BoxChunkSourceStream, Column, DataType, SourceContextRef, SourceMessage,
+    SplitId, SplitMetaData, SplitReader,
 };
 
 pub struct DatagenSplitReader {
     generator: DatagenEventGenerator,
+    #[expect(dead_code)]
     assigned_split: DatagenSplit,
 
     split_id: SplitId,
@@ -177,13 +178,13 @@ impl SplitReader for DatagenSplitReader {
             _ => {
                 let parser_config = self.parser_config.clone();
                 let source_context = self.source_ctx.clone();
-                into_chunk_stream(self, parser_config, source_context)
+                into_chunk_stream(self.into_data_stream(), parser_config, source_context)
             }
         }
     }
 }
 
-impl CommonSplitReader for DatagenSplitReader {
+impl DatagenSplitReader {
     fn into_data_stream(self) -> impl Stream<Item = ConnectorResult<Vec<SourceMessage>>> {
         // Will buffer at most 4 event chunks.
         const BUFFER_SIZE: usize = 4;
diff --git a/src/connector/src/source/filesystem/opendal_source/opendal_enumerator.rs b/src/connector/src/source/filesystem/opendal_source/opendal_enumerator.rs
index 96646ade0e1df..f65eacd9a9696 100644
--- a/src/connector/src/source/filesystem/opendal_source/opendal_enumerator.rs
+++ b/src/connector/src/source/filesystem/opendal_source/opendal_enumerator.rs
@@ -56,10 +56,7 @@ impl<Src: OpendalSource> SplitEnumerator for OpendalEnumerator<Src> {
 
 impl<Src: OpendalSource> OpendalEnumerator<Src> {
     pub async fn list(&self) -> ConnectorResult<ObjectMetadataIter> {
-        let prefix = match &self.prefix {
-            Some(prefix) => prefix,
-            None => "",
-        };
+        let prefix = self.prefix.as_deref().unwrap_or("");
 
         let object_lister = self
             .op
diff --git a/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs b/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs
index 0fb53ce8a0d1c..07765df40a383 100644
--- a/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs
+++ b/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs
@@ -25,15 +25,14 @@ use tokio_util::io::{ReaderStream, StreamReader};
 use super::opendal_enumerator::OpendalEnumerator;
 use super::OpendalSource;
 use crate::error::ConnectorResult;
-use crate::parser::{ByteStreamSourceParserImpl, ParserConfig};
+use crate::parser::ParserConfig;
 use crate::source::filesystem::nd_streaming::need_nd_streaming;
 use crate::source::filesystem::{nd_streaming, OpendalFsSplit};
 use crate::source::{
-    BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SourceMeta, SplitMetaData,
-    SplitReader,
+    into_chunk_stream, BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SourceMeta,
+    SplitMetaData, SplitReader,
 };
 
-const MAX_CHANNEL_BUFFER_SIZE: usize = 2048;
 const STREAM_READER_CAPACITY: usize = 4096;
 #[derive(Debug, Clone)]
 pub struct OpendalReader<Src: OpendalSource> {
@@ -65,46 +64,30 @@ impl<Src: OpendalSource> SplitReader for OpendalReader<Src> {
     }
 
     fn into_stream(self) -> BoxChunkSourceStream {
-        self.into_chunk_stream()
+        self.into_stream_inner()
     }
 }
 
 impl<Src: OpendalSource> OpendalReader<Src> {
     #[try_stream(boxed, ok = StreamChunk, error = crate::error::ConnectorError)]
-    async fn into_chunk_stream(self) {
+    async fn into_stream_inner(self) {
         for split in self.splits {
-            let actor_id = self.source_ctx.actor_id.to_string();
-            let fragment_id = self.source_ctx.fragment_id.to_string();
-            let source_id = self.source_ctx.source_id.to_string();
-            let source_name = self.source_ctx.source_name.to_string();
-            let source_ctx = self.source_ctx.clone();
-
-            let split_id = split.id();
-
             let data_stream =
                 Self::stream_read_object(self.connector.op.clone(), split, self.source_ctx.clone());
-
-            let parser =
-                ByteStreamSourceParserImpl::create(self.parser_config.clone(), source_ctx).await?;
-            let msg_stream = if need_nd_streaming(&self.parser_config.specific.encoding_config) {
-                parser.into_stream(nd_streaming::split_stream(data_stream))
+            let data_stream = if need_nd_streaming(&self.parser_config.specific.encoding_config) {
+                nd_streaming::split_stream(data_stream)
             } else {
-                parser.into_stream(data_stream)
+                data_stream
             };
+
+            let msg_stream = into_chunk_stream(
+                data_stream,
+                self.parser_config.clone(),
+                self.source_ctx.clone(),
+            );
             #[for_await]
             for msg in msg_stream {
                 let msg = msg?;
-                self.source_ctx
-                    .metrics
-                    .partition_input_count
-                    .with_label_values(&[
-                        &actor_id,
-                        &source_id,
-                        &split_id,
-                        &source_name,
-                        &fragment_id,
-                    ])
-                    .inc_by(msg.cardinality() as u64);
                 yield msg;
             }
         }
diff --git a/src/connector/src/source/filesystem/opendal_source/s3_source.rs b/src/connector/src/source/filesystem/opendal_source/s3_source.rs
index f4a548306885a..fd41c44e1f7d6 100644
--- a/src/connector/src/source/filesystem/opendal_source/s3_source.rs
+++ b/src/connector/src/source/filesystem/opendal_source/s3_source.rs
@@ -58,8 +58,6 @@ impl<Src: OpendalSource> OpendalEnumerator<Src> {
             );
         }
 
-        builder.enable_virtual_host_style();
-
         if let Some(assume_role) = assume_role {
             builder.role_arn(&assume_role);
         }
diff --git a/src/connector/src/source/filesystem/s3/source/reader.rs b/src/connector/src/source/filesystem/s3/source/reader.rs
index 129b708a61521..c3c800d6a5317 100644
--- a/src/connector/src/source/filesystem/s3/source/reader.rs
+++ b/src/connector/src/source/filesystem/s3/source/reader.rs
@@ -33,19 +33,21 @@ use tokio_util::io::ReaderStream;
 use crate::aws_utils::{default_conn_config, s3_client};
 use crate::connector_common::AwsAuthProps;
 use crate::error::ConnectorResult;
-use crate::parser::{ByteStreamSourceParserImpl, ParserConfig};
+use crate::parser::ParserConfig;
 use crate::source::base::{SplitMetaData, SplitReader};
 use crate::source::filesystem::file_common::FsSplit;
 use crate::source::filesystem::nd_streaming;
 use crate::source::filesystem::nd_streaming::need_nd_streaming;
 use crate::source::filesystem::s3::S3Properties;
-use crate::source::{BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SourceMeta};
+use crate::source::{
+    into_chunk_stream, BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SourceMeta,
+};
 
-const MAX_CHANNEL_BUFFER_SIZE: usize = 2048;
 const STREAM_READER_CAPACITY: usize = 4096;
 
 #[derive(Debug)]
 pub struct S3FileReader {
+    #[expect(dead_code)]
     split_offset: HashMap<String, u64>,
     bucket_name: String,
     s3_client: s3_client::Client,
@@ -204,50 +206,34 @@ impl SplitReader for S3FileReader {
     }
 
     fn into_stream(self) -> BoxChunkSourceStream {
-        self.into_chunk_stream()
+        self.into_stream_inner()
     }
 }
 
 impl S3FileReader {
     #[try_stream(boxed, ok = StreamChunk, error = crate::error::ConnectorError)]
-    async fn into_chunk_stream(self) {
+    async fn into_stream_inner(self) {
         for split in self.splits {
-            let actor_id = self.source_ctx.actor_id.to_string();
-            let fragment_id = self.source_ctx.fragment_id.to_string();
-            let source_id = self.source_ctx.source_id.to_string();
-            let source_name = self.source_ctx.source_name.to_string();
-            let source_ctx = self.source_ctx.clone();
-
-            let split_id = split.id();
-
             let data_stream = Self::stream_read_object(
                 self.s3_client.clone(),
                 self.bucket_name.clone(),
                 split,
                 self.source_ctx.clone(),
             );
-
-            let parser =
-                ByteStreamSourceParserImpl::create(self.parser_config.clone(), source_ctx).await?;
-            let msg_stream = if need_nd_streaming(&self.parser_config.specific.encoding_config) {
-                parser.into_stream(nd_streaming::split_stream(data_stream))
+            let data_stream = if need_nd_streaming(&self.parser_config.specific.encoding_config) {
+                nd_streaming::split_stream(data_stream)
             } else {
-                parser.into_stream(data_stream)
+                data_stream
             };
+
+            let msg_stream = into_chunk_stream(
+                data_stream,
+                self.parser_config.clone(),
+                self.source_ctx.clone(),
+            );
             #[for_await]
             for msg in msg_stream {
                 let msg = msg?;
-                self.source_ctx
-                    .metrics
-                    .partition_input_count
-                    .with_label_values(&[
-                        &actor_id,
-                        &source_id,
-                        &split_id,
-                        &source_name,
-                        &fragment_id,
-                    ])
-                    .inc_by(msg.cardinality() as u64);
                 yield msg;
             }
         }
@@ -313,7 +299,7 @@ mod tests {
             .await
             .unwrap();
 
-        let msg_stream = reader.into_chunk_stream();
+        let msg_stream = reader.into_stream_inner();
         #[for_await]
         for msg in msg_stream {
             println!("msg {:?}", msg);
diff --git a/src/connector/src/source/google_pubsub/enumerator/client.rs b/src/connector/src/source/google_pubsub/enumerator/client.rs
index 25cb28909c479..3bb37845763ea 100644
--- a/src/connector/src/source/google_pubsub/enumerator/client.rs
+++ b/src/connector/src/source/google_pubsub/enumerator/client.rs
@@ -15,8 +15,7 @@
 use anyhow::Context;
 use async_trait::async_trait;
 use chrono::{TimeZone, Utc};
-use google_cloud_pubsub::client::{Client, ClientConfig};
-use google_cloud_pubsub::subscription::{SeekTo, SubscriptionConfig};
+use google_cloud_pubsub::subscription::SeekTo;
 use risingwave_common::bail;
 
 use crate::error::ConnectorResult;
@@ -39,41 +38,22 @@ impl SplitEnumerator for PubsubSplitEnumerator {
         properties: Self::Properties,
         _context: SourceEnumeratorContextRef,
     ) -> ConnectorResult<PubsubSplitEnumerator> {
-        let subscription = properties.subscription.to_owned();
+        let split_count = properties.parallelism.unwrap_or(1);
+        if split_count < 1 {
+            bail!("parallelism must be >= 1");
+        };
 
         if properties.credentials.is_none() && properties.emulator_host.is_none() {
             bail!("credentials must be set if not using the pubsub emulator")
         }
 
-        properties.initialize_env();
-
-        // Validate config
-        let config = ClientConfig::default().with_auth().await?;
-        let client = Client::new(config)
-            .await
-            .context("error initializing pubsub client")?;
-
-        let sub = client.subscription(&subscription);
+        let sub = properties.subscription_client().await?;
         if !sub
             .exists(None)
             .await
             .context("error checking subscription validity")?
         {
-            bail!("subscription {} does not exist", &subscription)
-        }
-
-        // We need the `retain_acked_messages` configuration to be true to seek back to timestamps
-        // as done in the [`PubsubSplitReader`] and here.
-        let (_, subscription_config) = sub
-            .config(None)
-            .await
-            .context("failed to fetch subscription config")?;
-        if let SubscriptionConfig {
-            retain_acked_messages: false,
-            ..
-        } = subscription_config
-        {
-            bail!("subscription must be configured with retain_acked_messages set to true")
+            bail!("subscription {} does not exist", &sub.id())
         }
 
         let seek_to = match (properties.start_offset, properties.start_snapshot) {
@@ -98,8 +78,8 @@ impl SplitEnumerator for PubsubSplitEnumerator {
         }
 
         Ok(Self {
-            subscription,
-            split_count: 1,
+            subscription: properties.subscription.to_owned(),
+            split_count,
         })
     }
 
@@ -109,8 +89,8 @@ impl SplitEnumerator for PubsubSplitEnumerator {
             .map(|i| PubsubSplit {
                 index: i,
                 subscription: self.subscription.to_owned(),
-                start_offset: None,
-                stop_offset: None,
+                __deprecated_start_offset: None,
+                __deprecated_stop_offset: None,
             })
             .collect();
 
diff --git a/src/connector/src/source/google_pubsub/mod.rs b/src/connector/src/source/google_pubsub/mod.rs
index 0a49fa6467f66..a0794bb1205df 100644
--- a/src/connector/src/source/google_pubsub/mod.rs
+++ b/src/connector/src/source/google_pubsub/mod.rs
@@ -14,26 +14,39 @@
 
 use std::collections::HashMap;
 
+use anyhow::Context;
+use google_cloud_pubsub::client::{Client, ClientConfig};
+use google_cloud_pubsub::subscription::Subscription;
 use serde::Deserialize;
 
 pub mod enumerator;
 pub mod source;
 pub mod split;
-
 pub use enumerator::*;
+use serde_with::{serde_as, DisplayFromStr};
 pub use source::*;
 pub use split::*;
 use with_options::WithOptions;
 
+use crate::error::ConnectorResult;
 use crate::source::SourceProperties;
 
 pub const GOOGLE_PUBSUB_CONNECTOR: &str = "google_pubsub";
 
+/// # Implementation Notes
+/// Pub/Sub does not rely on persisted state (`SplitImpl`) to start from a position.
+/// It rely on Pub/Sub to load-balance messages between all Readers.
+/// We `ack` received messages after checkpoint (see `WaitCheckpointWorker`) to achieve at-least-once delivery.
+#[serde_as]
 #[derive(Clone, Debug, Deserialize, WithOptions)]
 pub struct PubsubProperties {
-    /// pubsub subscription to consume messages from
-    /// The subscription should be configured with the `retain-on-ack` property to enable
-    /// message recovery within risingwave.
+    /// Pub/Sub subscription to consume messages from.
+    ///
+    /// Note that we rely on Pub/Sub to load-balance messages between all Readers pulling from
+    /// the same subscription. So one `subscription` (i.e., one `Source`) can only used for one MV
+    /// (shared between the actors of its fragment).
+    /// Otherwise, different MVs on the same Source will both receive part of the messages.
+    /// TODO: check and enforce this on Meta.
     #[serde(rename = "pubsub.subscription")]
     pub subscription: String,
 
@@ -62,12 +75,18 @@ pub struct PubsubProperties {
     /// in pub/sub because they guarantee retention of:
     /// - All unacknowledged messages at the time of their creation.
     /// - All messages created after their creation.
-    /// Besides retention guarantees, timestamps are also more precise than timestamp-based seeks.
+    /// Besides retention guarantees, snapshots are also more precise than timestamp-based seeks.
     /// See [Seeking to a snapshot](https://cloud.google.com/pubsub/docs/replay-overview#seeking_to_a_timestamp) for
     /// more details.
     #[serde(rename = "pubsub.start_snapshot")]
     pub start_snapshot: Option<String>,
 
+    /// `parallelism` is the number of parallel consumers to run for the subscription.
+    /// TODO: use system parallelism if not set
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    #[serde(rename = "pubsub.parallelism")]
+    pub parallelism: Option<u32>,
+
     #[serde(flatten)]
     pub unknown_fields: HashMap<String, String>,
 }
@@ -97,6 +116,18 @@ impl PubsubProperties {
             std::env::set_var("GOOGLE_APPLICATION_CREDENTIALS_JSON", credentials);
         }
     }
+
+    pub(crate) async fn subscription_client(&self) -> ConnectorResult<Subscription> {
+        self.initialize_env();
+
+        // Validate config
+        let config = ClientConfig::default().with_auth().await?;
+        let client = Client::new(config)
+            .await
+            .context("error initializing pubsub client")?;
+
+        Ok(client.subscription(&self.subscription))
+    }
 }
 
 #[cfg(test)]
@@ -123,7 +154,7 @@ mod tests {
             start_offset: None,
             start_snapshot: None,
             subscription: String::from("test-subscription"),
-
+            parallelism: None,
             unknown_fields: Default::default(),
         };
 
diff --git a/src/connector/src/source/google_pubsub/source/message.rs b/src/connector/src/source/google_pubsub/source/message.rs
index cf14757b3668b..10194453bebbc 100644
--- a/src/connector/src/source/google_pubsub/source/message.rs
+++ b/src/connector/src/source/google_pubsub/source/message.rs
@@ -31,6 +31,7 @@ impl From<TaggedReceivedMessage> for SourceMessage {
     fn from(tagged_message: TaggedReceivedMessage) -> Self {
         let TaggedReceivedMessage(split_id, message) = tagged_message;
 
+        let ack_id = message.ack_id().to_string();
         let timestamp = message
             .message
             .publish_time
@@ -50,8 +51,9 @@ impl From<TaggedReceivedMessage> for SourceMessage {
                     _ => Some(payload),
                 }
             },
-            offset: timestamp.timestamp_nanos_opt().unwrap().to_string(),
+            offset: ack_id,
             split_id,
+            // What's the usage of this?
             meta: SourceMeta::GooglePubsub(GooglePubsubMeta {
                 timestamp: Some(timestamp.timestamp_millis()),
             }),
diff --git a/src/connector/src/source/google_pubsub/source/reader.rs b/src/connector/src/source/google_pubsub/source/reader.rs
index 0887cb06594f9..abc3ecd7743e3 100644
--- a/src/connector/src/source/google_pubsub/source/reader.rs
+++ b/src/connector/src/source/google_pubsub/source/reader.rs
@@ -12,12 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use anyhow::{anyhow, Context};
 use async_trait::async_trait;
-use chrono::{NaiveDateTime, TimeZone, Utc};
 use futures_async_stream::try_stream;
-use google_cloud_pubsub::client::{Client, ClientConfig};
-use google_cloud_pubsub::subscription::{SeekTo, Subscription};
+use google_cloud_pubsub::subscription::Subscription;
 use risingwave_common::{bail, ensure};
 use tonic::Code;
 
@@ -26,22 +23,21 @@ use crate::error::{ConnectorError, ConnectorResult as Result};
 use crate::parser::ParserConfig;
 use crate::source::google_pubsub::{PubsubProperties, PubsubSplit};
 use crate::source::{
-    into_chunk_stream, BoxChunkSourceStream, Column, CommonSplitReader, SourceContextRef,
-    SourceMessage, SplitId, SplitMetaData, SplitReader,
+    into_chunk_stream, BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SplitId,
+    SplitMetaData, SplitReader,
 };
 
 const PUBSUB_MAX_FETCH_MESSAGES: usize = 1024;
 
 pub struct PubsubSplitReader {
     subscription: Subscription,
-    stop_offset: Option<NaiveDateTime>,
 
     split_id: SplitId,
     parser_config: ParserConfig,
     source_ctx: SourceContextRef,
 }
 
-impl CommonSplitReader for PubsubSplitReader {
+impl PubsubSplitReader {
     #[try_stream(ok = Vec<SourceMessage>, error = ConnectorError)]
     async fn into_data_stream(self) {
         loop {
@@ -67,41 +63,16 @@ impl CommonSplitReader for PubsubSplitReader {
                 continue;
             }
 
-            let latest_offset: NaiveDateTime = raw_chunk
-                .last()
-                .map(|m| m.message.publish_time.clone().unwrap_or_default())
-                .map(|t| {
-                    let mut t = t;
-                    t.normalize();
-                    NaiveDateTime::from_timestamp_opt(t.seconds, t.nanos as u32).unwrap_or_default()
-                })
-                .unwrap_or_default();
-
             let mut chunk: Vec<SourceMessage> = Vec::with_capacity(raw_chunk.len());
-            let mut ack_ids: Vec<String> = Vec::with_capacity(raw_chunk.len());
 
             for message in raw_chunk {
-                ack_ids.push(message.ack_id().into());
                 chunk.push(SourceMessage::from(TaggedReceivedMessage(
                     self.split_id.clone(),
                     message,
                 )));
             }
 
-            self.subscription
-                .ack(ack_ids)
-                .await
-                .map_err(|e| anyhow!(e))
-                .context("failed to ack pubsub messages")?;
-
             yield chunk;
-
-            // Stop if we've approached the stop_offset
-            if let Some(stop_offset) = self.stop_offset
-                && latest_offset >= stop_offset
-            {
-                return Ok(());
-            }
         }
     }
 }
@@ -124,42 +95,11 @@ impl SplitReader for PubsubSplitReader {
         );
         let split = splits.into_iter().next().unwrap();
 
-        // Set environment variables consumed by `google_cloud_pubsub`
-        properties.initialize_env();
-
-        let config = ClientConfig::default().with_auth().await?;
-        let client = Client::new(config).await.map_err(|e| anyhow!(e))?;
-        let subscription = client.subscription(&properties.subscription);
-
-        if let Some(ref offset) = split.start_offset {
-            let timestamp = offset
-                .as_str()
-                .parse::<i64>()
-                .map(|nanos| Utc.timestamp_nanos(nanos))
-                .context("error parsing offset")?;
-
-            subscription
-                .seek(SeekTo::Timestamp(timestamp.into()), None)
-                .await
-                .context("error seeking to pubsub offset")?;
-        }
-
-        let stop_offset = if let Some(ref offset) = split.stop_offset {
-            Some(
-                offset
-                    .as_str()
-                    .parse::<i64>()
-                    .map_err(|e| anyhow!(e))
-                    .map(|nanos| NaiveDateTime::from_timestamp_opt(nanos, 0).unwrap_or_default())?,
-            )
-        } else {
-            None
-        };
+        let subscription = properties.subscription_client().await?;
 
         Ok(Self {
             subscription,
             split_id: split.id(),
-            stop_offset,
             parser_config,
             source_ctx,
         })
@@ -168,6 +108,6 @@ impl SplitReader for PubsubSplitReader {
     fn into_stream(self) -> BoxChunkSourceStream {
         let parser_config = self.parser_config.clone();
         let source_context = self.source_ctx.clone();
-        into_chunk_stream(self, parser_config, source_context)
+        into_chunk_stream(self.into_data_stream(), parser_config, source_context)
     }
 }
diff --git a/src/connector/src/source/google_pubsub/split.rs b/src/connector/src/source/google_pubsub/split.rs
index 14c40150488a1..85d32f479bc2e 100644
--- a/src/connector/src/source/google_pubsub/split.rs
+++ b/src/connector/src/source/google_pubsub/split.rs
@@ -20,19 +20,18 @@ use crate::source::{SplitId, SplitMetaData};
 
 #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)]
 pub struct PubsubSplit {
+    // XXX: `index` and `subscription` seems also not useful. It's only for `SplitMetaData::id`.
+    // Is the split id useful?
     pub(crate) index: u32,
     pub(crate) subscription: String,
 
-    /// `start_offset` is a numeric timestamp.
-    /// When not `None`, the `PubsubReader` seeks to the timestamp described by the `start_offset`.
-    /// These offsets are taken from the `offset` property of the `SourceMessage` yielded by the
-    /// pubsub reader.
-    pub(crate) start_offset: Option<String>,
+    #[serde(rename = "start_offset")]
+    #[serde(skip_serializing)]
+    pub(crate) __deprecated_start_offset: Option<String>,
 
-    /// `stop_offset` is a numeric timestamp.
-    /// When not `None`, the `PubsubReader` stops reading messages when the `offset` property of
-    /// the `SourceMessage` is greater than or equal to the `stop_offset`.
-    pub(crate) stop_offset: Option<String>,
+    #[serde(rename = "stop_offset")]
+    #[serde(skip_serializing)]
+    pub(crate) __deprecated_stop_offset: Option<String>,
 }
 
 impl SplitMetaData for PubsubSplit {
@@ -48,8 +47,11 @@ impl SplitMetaData for PubsubSplit {
         format!("{}-{}", self.subscription, self.index).into()
     }
 
-    fn update_offset(&mut self, last_seen_offset: String) -> ConnectorResult<()> {
-        self.start_offset = Some(last_seen_offset);
+    /// No-op. Actually `PubsubSplit` doesn't maintain any state. It's fully managed by Pubsub.
+    /// One subscription is like one Kafka consumer group.
+    fn update_offset(&mut self, _last_seen_offset: String) -> ConnectorResult<()> {
+        // forcefully set previously persisted start_offset to None
+        self.__deprecated_start_offset = None;
         Ok(())
     }
 }
diff --git a/src/connector/src/source/kafka/client_context.rs b/src/connector/src/source/kafka/client_context.rs
new file mode 100644
index 0000000000000..d788411ff15f5
--- /dev/null
+++ b/src/connector/src/source/kafka/client_context.rs
@@ -0,0 +1,227 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::BTreeMap;
+use std::sync::Arc;
+use std::thread;
+
+use anyhow::anyhow;
+use aws_config::Region;
+use aws_sdk_s3::config::SharedCredentialsProvider;
+use rdkafka::client::{BrokerAddr, OAuthToken};
+use rdkafka::consumer::ConsumerContext;
+use rdkafka::message::DeliveryResult;
+use rdkafka::producer::ProducerContext;
+use rdkafka::{ClientContext, Statistics};
+
+use super::private_link::{BrokerAddrRewriter, PrivateLinkContextRole};
+use super::stats::RdKafkaStats;
+use crate::connector_common::AwsAuthProps;
+use crate::error::ConnectorResult;
+
+struct IamAuthEnv {
+    credentials_provider: SharedCredentialsProvider,
+    region: Region,
+    // XXX(runji): madsim does not support `Handle` for now
+    #[cfg(not(madsim))]
+    rt: tokio::runtime::Handle,
+}
+
+pub struct KafkaContextCommon {
+    // For VPC PrivateLink support
+    addr_rewriter: BrokerAddrRewriter,
+
+    // identifier is required when reporting metrics as a label, usually it is compose by connector
+    // format (source or sink) and corresponding id (source_id or sink_id)
+    // identifier and metrics should be set at the same time
+    identifier: Option<String>,
+    metrics: Option<Arc<RdKafkaStats>>,
+
+    /// Credential and region for AWS MSK
+    auth: Option<IamAuthEnv>,
+}
+
+impl KafkaContextCommon {
+    pub async fn new(
+        broker_rewrite_map: Option<BTreeMap<String, String>>,
+        identifier: Option<String>,
+        metrics: Option<Arc<RdKafkaStats>>,
+        auth: AwsAuthProps,
+        is_aws_msk_iam: bool,
+    ) -> ConnectorResult<Self> {
+        let addr_rewriter =
+            BrokerAddrRewriter::new(PrivateLinkContextRole::Consumer, broker_rewrite_map)?;
+        let auth = if is_aws_msk_iam {
+            let config = auth.build_config().await?;
+            let credentials_provider = config
+                .credentials_provider()
+                .ok_or_else(|| anyhow!("missing aws credentials_provider"))?;
+            let region = config
+                .region()
+                .ok_or_else(|| anyhow!("missing aws region"))?
+                .clone();
+            Some(IamAuthEnv {
+                credentials_provider,
+                region,
+                #[cfg(not(madsim))]
+                rt: tokio::runtime::Handle::current(),
+            })
+        } else {
+            None
+        };
+        Ok(Self {
+            addr_rewriter,
+            identifier,
+            metrics,
+            auth,
+        })
+    }
+}
+
+impl KafkaContextCommon {
+    fn stats(&self, statistics: Statistics) {
+        if let Some(metrics) = &self.metrics
+            && let Some(id) = &self.identifier
+        {
+            metrics.report(id.as_str(), &statistics);
+        }
+    }
+
+    fn rewrite_broker_addr(&self, addr: BrokerAddr) -> BrokerAddr {
+        self.addr_rewriter.rewrite_broker_addr(addr)
+    }
+
+    // XXX(runji): oauth is ignored in simulation
+    #[cfg_or_panic::cfg_or_panic(not(madsim))]
+    fn generate_oauth_token(
+        &self,
+        _oauthbearer_config: Option<&str>,
+    ) -> Result<OAuthToken, Box<dyn std::error::Error>> {
+        use aws_msk_iam_sasl_signer::generate_auth_token_from_credentials_provider;
+        use tokio::time::{timeout, Duration};
+
+        if let Some(IamAuthEnv {
+            credentials_provider,
+            region,
+            rt,
+        }) = &self.auth
+        {
+            let region = region.clone();
+            let credentials_provider = credentials_provider.clone();
+            let rt = rt.clone();
+            let (token, expiration_time_ms) = {
+                let handle = thread::spawn(move || {
+                    rt.block_on(async {
+                        timeout(
+                            Duration::from_secs(10),
+                            generate_auth_token_from_credentials_provider(
+                                region,
+                                credentials_provider,
+                            ),
+                        )
+                        .await
+                    })
+                });
+                handle.join().unwrap()??
+            };
+            Ok(OAuthToken {
+                token,
+                principal_name: "".to_string(),
+                lifetime_ms: expiration_time_ms,
+            })
+        } else {
+            Err("must provide AWS IAM credential".into())
+        }
+    }
+
+    fn enable_refresh_oauth_token(&self) -> bool {
+        self.auth.is_some()
+    }
+}
+
+pub type BoxConsumerContext = Box<dyn ConsumerContext>;
+
+/// Kafka consumer context used for private link, IAM auth, and metrics
+pub struct RwConsumerContext {
+    common: KafkaContextCommon,
+}
+
+impl RwConsumerContext {
+    pub fn new(common: KafkaContextCommon) -> Self {
+        Self { common }
+    }
+}
+
+impl ClientContext for RwConsumerContext {
+    /// this func serves as a callback when `poll` is completed.
+    fn stats(&self, statistics: Statistics) {
+        self.common.stats(statistics);
+    }
+
+    fn rewrite_broker_addr(&self, addr: BrokerAddr) -> BrokerAddr {
+        self.common.rewrite_broker_addr(addr)
+    }
+
+    fn generate_oauth_token(
+        &self,
+        oauthbearer_config: Option<&str>,
+    ) -> Result<OAuthToken, Box<dyn std::error::Error>> {
+        self.common.generate_oauth_token(oauthbearer_config)
+    }
+
+    fn enable_refresh_oauth_token(&self) -> bool {
+        self.common.enable_refresh_oauth_token()
+    }
+}
+
+// required by the trait bound of BaseConsumer
+impl ConsumerContext for RwConsumerContext {}
+
+/// Kafka producer context used for private link, IAM auth, and metrics
+pub struct RwProducerContext {
+    common: KafkaContextCommon,
+}
+
+impl RwProducerContext {
+    pub fn new(common: KafkaContextCommon) -> Self {
+        Self { common }
+    }
+}
+
+impl ClientContext for RwProducerContext {
+    fn stats(&self, statistics: Statistics) {
+        self.common.stats(statistics);
+    }
+
+    fn rewrite_broker_addr(&self, addr: BrokerAddr) -> BrokerAddr {
+        self.common.rewrite_broker_addr(addr)
+    }
+
+    fn generate_oauth_token(
+        &self,
+        oauthbearer_config: Option<&str>,
+    ) -> Result<OAuthToken, Box<dyn std::error::Error>> {
+        self.common.generate_oauth_token(oauthbearer_config)
+    }
+
+    fn enable_refresh_oauth_token(&self) -> bool {
+        self.common.enable_refresh_oauth_token()
+    }
+}
+
+impl ProducerContext for RwProducerContext {
+    type DeliveryOpaque = ();
+
+    fn delivery(&self, _: &DeliveryResult<'_>, _: Self::DeliveryOpaque) {}
+}
diff --git a/src/connector/src/source/kafka/enumerator/client.rs b/src/connector/src/source/kafka/enumerator/client.rs
index 697fe421a5fb2..d4dddad67bfdf 100644
--- a/src/connector/src/source/kafka/enumerator/client.rs
+++ b/src/connector/src/source/kafka/enumerator/client.rs
@@ -25,7 +25,9 @@ use risingwave_common::bail;
 use crate::error::ConnectorResult;
 use crate::source::base::SplitEnumerator;
 use crate::source::kafka::split::KafkaSplit;
-use crate::source::kafka::{KafkaProperties, PrivateLinkConsumerContext, KAFKA_ISOLATION_LEVEL};
+use crate::source::kafka::{
+    KafkaContextCommon, KafkaProperties, RwConsumerContext, KAFKA_ISOLATION_LEVEL,
+};
 use crate::source::SourceEnumeratorContextRef;
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
@@ -40,7 +42,7 @@ pub struct KafkaSplitEnumerator {
     context: SourceEnumeratorContextRef,
     broker_address: String,
     topic: String,
-    client: BaseConsumer<PrivateLinkConsumerContext>,
+    client: BaseConsumer<RwConsumerContext>,
     start_offset: KafkaEnumeratorOffset,
 
     // maybe used in the future for batch processing
@@ -90,10 +92,30 @@ impl SplitEnumerator for KafkaSplitEnumerator {
         }
 
         // don't need kafka metrics from enumerator
-        let client_ctx = PrivateLinkConsumerContext::new(broker_rewrite_map, None, None)?;
-        let client: BaseConsumer<PrivateLinkConsumerContext> =
+        let ctx_common = KafkaContextCommon::new(
+            broker_rewrite_map,
+            None,
+            None,
+            properties.aws_auth_props,
+            common_props.is_aws_msk_iam(),
+        )
+        .await?;
+        let client_ctx = RwConsumerContext::new(ctx_common);
+        let client: BaseConsumer<RwConsumerContext> =
             config.create_with_context(client_ctx).await?;
 
+        // Note that before any SASL/OAUTHBEARER broker connection can succeed the application must call
+        // rd_kafka_oauthbearer_set_token() once – either directly or, more typically, by invoking either
+        // rd_kafka_poll(), rd_kafka_consumer_poll(), rd_kafka_queue_poll(), etc, in order to cause retrieval
+        // of an initial token to occur.
+        // https://docs.confluent.io/platform/current/clients/librdkafka/html/rdkafka_8h.html#a988395722598f63396d7a1bedb22adaf
+        if common_props.is_aws_msk_iam() {
+            #[cfg(not(madsim))]
+            client.poll(Duration::from_secs(10)); // note: this is a blocking call
+            #[cfg(madsim)]
+            client.poll(Duration::from_secs(10)).await;
+        }
+
         Ok(Self {
             context,
             broker_address,
@@ -129,6 +151,7 @@ impl SplitEnumerator for KafkaSplitEnumerator {
                 partition,
                 start_offset: start_offsets.remove(&partition).unwrap(),
                 stop_offset: stop_offsets.remove(&partition).unwrap(),
+                hack_seek_to_latest: false,
             })
             .collect();
 
@@ -232,6 +255,7 @@ impl KafkaSplitEnumerator {
                     partition: *partition,
                     start_offset: Some(start_offset),
                     stop_offset: Some(stop_offset),
+                    hack_seek_to_latest:false
                 }
             })
             .collect::<Vec<KafkaSplit>>())
diff --git a/src/connector/src/source/kafka/mod.rs b/src/connector/src/source/kafka/mod.rs
index 91d4ccce5ca88..2360f9fb8a337 100644
--- a/src/connector/src/source/kafka/mod.rs
+++ b/src/connector/src/source/kafka/mod.rs
@@ -17,16 +17,17 @@ use std::collections::HashMap;
 use serde::Deserialize;
 use serde_with::{serde_as, DisplayFromStr};
 
-use crate::connector_common::KafkaPrivateLinkCommon;
+use crate::connector_common::{AwsAuthProps, KafkaPrivateLinkCommon};
 
+mod client_context;
 pub mod enumerator;
 pub mod private_link;
 pub mod source;
 pub mod split;
 pub mod stats;
 
+pub use client_context::*;
 pub use enumerator::*;
-pub use private_link::*;
 pub use source::*;
 pub use split::*;
 use with_options::WithOptions;
@@ -137,6 +138,9 @@ pub struct KafkaProperties {
     #[serde(flatten)]
     pub privatelink_common: KafkaPrivateLinkCommon,
 
+    #[serde(flatten)]
+    pub aws_auth_props: AwsAuthProps,
+
     #[serde(flatten)]
     pub unknown_fields: HashMap<String, String>,
 }
@@ -159,8 +163,6 @@ impl KafkaProperties {
     pub fn set_client(&self, c: &mut rdkafka::ClientConfig) {
         self.rdkafka_properties_common.set_client(c);
         self.rdkafka_properties_consumer.set_client(c);
-
-        tracing::info!("kafka client starts with: {:?}", c);
     }
 }
 
@@ -191,15 +193,15 @@ impl RdKafkaPropertiesConsumer {
 
 #[cfg(test)]
 mod test {
-    use std::collections::HashMap;
+    use std::collections::BTreeMap;
 
-    use maplit::hashmap;
+    use maplit::btreemap;
 
     use super::*;
 
     #[test]
     fn test_parse_config_consumer_common() {
-        let config: HashMap<String, String> = hashmap! {
+        let config: BTreeMap<String, String> = btreemap! {
             // common
             "properties.bootstrap.server".to_string() => "127.0.0.1:9092".to_string(),
             "topic".to_string() => "test".to_string(),
@@ -255,7 +257,7 @@ mod test {
             props.rdkafka_properties_consumer.fetch_queue_backoff_ms,
             Some(114514)
         );
-        let hashmap: HashMap<String, String> = hashmap! {
+        let hashmap: BTreeMap<String, String> = btreemap! {
             "broker1".to_string() => "10.0.0.1:8001".to_string()
         };
         assert_eq!(props.privatelink_common.broker_rewrite_map, Some(hashmap));
diff --git a/src/connector/src/source/kafka/private_link.rs b/src/connector/src/source/kafka/private_link.rs
index 6187078ae24fb..6aeebde87b516 100644
--- a/src/connector/src/source/kafka/private_link.rs
+++ b/src/connector/src/source/kafka/private_link.rs
@@ -14,14 +14,10 @@
 
 use std::collections::{BTreeMap, HashMap};
 use std::str::FromStr;
-use std::sync::Arc;
 
 use anyhow::{anyhow, Context};
 use itertools::Itertools;
 use rdkafka::client::BrokerAddr;
-use rdkafka::consumer::ConsumerContext;
-use rdkafka::producer::{DeliveryResult, ProducerContext};
-use rdkafka::{ClientContext, Statistics};
 use risingwave_common::bail;
 use risingwave_common::util::addr::HostAddr;
 use risingwave_common::util::iter_util::ZipEqFast;
@@ -31,15 +27,15 @@ use crate::connector_common::{
     AwsPrivateLinkItem, PRIVATE_LINK_BROKER_REWRITE_MAP_KEY, PRIVATE_LINK_TARGETS_KEY,
 };
 use crate::error::ConnectorResult;
-use crate::source::kafka::stats::RdKafkaStats;
 use crate::source::kafka::{KAFKA_PROPS_BROKER_KEY, KAFKA_PROPS_BROKER_KEY_ALIAS};
 
 pub const PRIVATELINK_ENDPOINT_KEY: &str = "privatelink.endpoint";
 pub const CONNECTION_NAME_KEY: &str = "connection.name";
 
 #[derive(Debug)]
-enum PrivateLinkContextRole {
+pub(super) enum PrivateLinkContextRole {
     Consumer,
+    #[expect(dead_code)]
     Producer,
 }
 
@@ -52,13 +48,14 @@ impl std::fmt::Display for PrivateLinkContextRole {
     }
 }
 
-struct BrokerAddrRewriter {
+pub(super) struct BrokerAddrRewriter {
+    #[expect(dead_code)]
     role: PrivateLinkContextRole,
     rewrite_map: BTreeMap<BrokerAddr, BrokerAddr>,
 }
 
 impl BrokerAddrRewriter {
-    fn rewrite_broker_addr(&self, addr: BrokerAddr) -> BrokerAddr {
+    pub(super) fn rewrite_broker_addr(&self, addr: BrokerAddr) -> BrokerAddr {
         let rewrote_addr = match self.rewrite_map.get(&addr) {
             None => addr,
             Some(new_addr) => new_addr.clone(),
@@ -68,7 +65,7 @@ impl BrokerAddrRewriter {
 
     pub fn new(
         role: PrivateLinkContextRole,
-        broker_rewrite_map: Option<HashMap<String, String>>,
+        broker_rewrite_map: Option<BTreeMap<String, String>>,
     ) -> ConnectorResult<Self> {
         let rewrite_map: ConnectorResult<BTreeMap<BrokerAddr, BrokerAddr>> = broker_rewrite_map
             .map_or(Ok(BTreeMap::new()), |addr_map| {
@@ -95,94 +92,6 @@ impl BrokerAddrRewriter {
     }
 }
 
-pub struct PrivateLinkConsumerContext {
-    inner: BrokerAddrRewriter,
-
-    // identifier is required when reporting metrics as a label, usually it is compose by connector
-    // format (source or sink) and corresponding id (source_id or sink_id)
-    // identifier and metrics should be set at the same time
-    identifier: Option<String>,
-    metrics: Option<Arc<RdKafkaStats>>,
-}
-
-impl PrivateLinkConsumerContext {
-    pub fn new(
-        broker_rewrite_map: Option<HashMap<String, String>>,
-        identifier: Option<String>,
-        metrics: Option<Arc<RdKafkaStats>>,
-    ) -> ConnectorResult<Self> {
-        let inner = BrokerAddrRewriter::new(PrivateLinkContextRole::Consumer, broker_rewrite_map)?;
-        Ok(Self {
-            inner,
-            identifier,
-            metrics,
-        })
-    }
-}
-
-impl ClientContext for PrivateLinkConsumerContext {
-    /// this func serves as a callback when `poll` is completed.
-    fn stats(&self, statistics: Statistics) {
-        if let Some(metrics) = &self.metrics
-            && let Some(id) = &self.identifier
-        {
-            metrics.report(id.as_str(), &statistics);
-        }
-    }
-
-    fn rewrite_broker_addr(&self, addr: BrokerAddr) -> BrokerAddr {
-        self.inner.rewrite_broker_addr(addr)
-    }
-}
-
-// required by the trait bound of BaseConsumer
-impl ConsumerContext for PrivateLinkConsumerContext {}
-
-pub struct PrivateLinkProducerContext {
-    inner: BrokerAddrRewriter,
-
-    // identifier is required when reporting metrics as a label, usually it is compose by connector
-    // format (source or sink) and corresponding id (source_id or sink_id)
-    // identifier and metrics should be set at the same time
-    identifier: Option<String>,
-    metrics: Option<Arc<RdKafkaStats>>,
-}
-
-impl PrivateLinkProducerContext {
-    pub fn new(
-        broker_rewrite_map: Option<HashMap<String, String>>,
-        identifier: Option<String>,
-        metrics: Option<Arc<RdKafkaStats>>,
-    ) -> ConnectorResult<Self> {
-        let inner = BrokerAddrRewriter::new(PrivateLinkContextRole::Producer, broker_rewrite_map)?;
-        Ok(Self {
-            inner,
-            identifier,
-            metrics,
-        })
-    }
-}
-
-impl ClientContext for PrivateLinkProducerContext {
-    fn stats(&self, statistics: Statistics) {
-        if let Some(metrics) = &self.metrics
-            && let Some(id) = &self.identifier
-        {
-            metrics.report(id.as_str(), &statistics);
-        }
-    }
-
-    fn rewrite_broker_addr(&self, addr: BrokerAddr) -> BrokerAddr {
-        self.inner.rewrite_broker_addr(addr)
-    }
-}
-
-impl ProducerContext for PrivateLinkProducerContext {
-    type DeliveryOpaque = ();
-
-    fn delivery(&self, _: &DeliveryResult<'_>, _: Self::DeliveryOpaque) {}
-}
-
 #[inline(always)]
 fn kafka_props_broker_key(with_properties: &BTreeMap<String, String>) -> &str {
     if with_properties.contains_key(KAFKA_PROPS_BROKER_KEY) {
diff --git a/src/connector/src/source/kafka/source/message.rs b/src/connector/src/source/kafka/source/message.rs
index 0ef55dc79132d..247166a156763 100644
--- a/src/connector/src/source/kafka/source/message.rs
+++ b/src/connector/src/source/kafka/source/message.rs
@@ -12,10 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::borrow::Cow;
+
 use itertools::Itertools;
 use rdkafka::message::{BorrowedMessage, Headers, OwnedHeaders};
 use rdkafka::Message;
-use risingwave_common::types::{Datum, ListValue, Scalar, ScalarImpl, StructValue};
+use risingwave_common::types::{
+    Datum, DatumCow, DatumRef, ListValue, ScalarImpl, ScalarRefImpl, StructValue,
+};
 use risingwave_pb::data::data_type::TypeName as PbTypeName;
 use risingwave_pb::data::DataType as PbDataType;
 
@@ -31,39 +35,42 @@ pub struct KafkaMeta {
 }
 
 impl KafkaMeta {
-    pub fn extract_timestamp(&self) -> Option<Datum> {
-        self.timestamp
-            .map(|ts| {
-                risingwave_common::cast::i64_to_timestamptz(ts)
-                    .unwrap()
-                    .to_scalar_value()
-            })
-            .into()
+    pub fn extract_timestamp(&self) -> Option<DatumRef<'_>> {
+        self.timestamp.map(|ts| {
+            Some(ScalarRefImpl::Timestamptz(
+                risingwave_common::cast::i64_to_timestamptz(ts).unwrap(),
+            ))
+        })
     }
 
-    pub fn extract_header_inner(
-        &self,
+    pub fn extract_header_inner<'a>(
+        &'a self,
         inner_field: &str,
         data_type: Option<&PbDataType>,
-    ) -> Option<Datum> {
-        let target_value = self
-            .headers
-            .as_ref()
-            .iter()
-            .find_map(|headers| {
-                headers
-                    .iter()
-                    .find(|header| header.key == inner_field)
-                    .map(|header| header.value)
-            })
-            .unwrap_or(None); // if not found the specified column, return None
-        if let Some(data_type) = data_type
+    ) -> Option<DatumCow<'a>> {
+        let target_value = self.headers.as_ref().iter().find_map(|headers| {
+            headers
+                .iter()
+                .find(|header| header.key == inner_field)
+                .map(|header| header.value)
+        })?; // if not found the specified column, return None
+
+        let Some(target_value) = target_value else {
+            return Some(Datum::None.into());
+        };
+
+        let datum = if let Some(data_type) = data_type
             && data_type.type_name == PbTypeName::Varchar as i32
         {
-            Some(target_value.map(|byte| ScalarImpl::Utf8(String::from_utf8_lossy(byte).into())))
+            match String::from_utf8_lossy(target_value) {
+                Cow::Borrowed(str) => Some(ScalarRefImpl::Utf8(str)).into(),
+                Cow::Owned(string) => Some(ScalarImpl::Utf8(string.into())).into(),
+            }
         } else {
-            Some(target_value.map(|byte| ScalarImpl::Bytea(byte.into())))
-        }
+            Some(ScalarRefImpl::Bytea(target_value)).into()
+        };
+
+        Some(datum)
     }
 
     pub fn extract_headers(&self) -> Option<Datum> {
diff --git a/src/connector/src/source/kafka/source/reader.rs b/src/connector/src/source/kafka/source/reader.rs
index 73c24dd5f810d..c54b0ac27655a 100644
--- a/src/connector/src/source/kafka/source/reader.rs
+++ b/src/connector/src/source/kafka/source/reader.rs
@@ -31,15 +31,15 @@ use crate::error::ConnectorResult as Result;
 use crate::parser::ParserConfig;
 use crate::source::base::SourceMessage;
 use crate::source::kafka::{
-    KafkaProperties, KafkaSplit, PrivateLinkConsumerContext, KAFKA_ISOLATION_LEVEL,
+    KafkaContextCommon, KafkaProperties, KafkaSplit, RwConsumerContext, KAFKA_ISOLATION_LEVEL,
 };
 use crate::source::{
-    into_chunk_stream, BoxChunkSourceStream, Column, CommonSplitReader, SourceContextRef, SplitId,
-    SplitMetaData, SplitReader,
+    into_chunk_stream, BoxChunkSourceStream, Column, SourceContextRef, SplitId, SplitMetaData,
+    SplitReader,
 };
 
 pub struct KafkaSplitReader {
-    consumer: StreamConsumer<PrivateLinkConsumerContext>,
+    consumer: StreamConsumer<RwConsumerContext>,
     offsets: HashMap<SplitId, (Option<i64>, Option<i64>)>,
     bytes_per_second: usize,
     max_num_messages: usize,
@@ -78,7 +78,7 @@ impl SplitReader for KafkaSplitReader {
             format!("rw-consumer-{}", source_ctx.fragment_id),
         );
 
-        let client_ctx = PrivateLinkConsumerContext::new(
+        let ctx_common = KafkaContextCommon::new(
             broker_rewrite_map,
             Some(format!(
                 "fragment-{}-source-{}-actor-{}",
@@ -87,8 +87,13 @@ impl SplitReader for KafkaSplitReader {
             // thread consumer will keep polling in the background, we don't need to call `poll`
             // explicitly
             Some(source_ctx.metrics.rdkafka_native_metric.clone()),
-        )?;
-        let consumer: StreamConsumer<PrivateLinkConsumerContext> = config
+            properties.aws_auth_props,
+            properties.common.is_aws_msk_iam(),
+        )
+        .await?;
+
+        let client_ctx = RwConsumerContext::new(ctx_common);
+        let consumer: StreamConsumer<RwConsumerContext> = config
             .set_log_level(RDKafkaLogLevel::Info)
             .create_with_context(client_ctx)
             .await
@@ -101,7 +106,9 @@ impl SplitReader for KafkaSplitReader {
         for split in splits {
             offsets.insert(split.id(), (split.start_offset, split.stop_offset));
 
-            if let Some(offset) = split.start_offset {
+            if split.hack_seek_to_latest {
+                tpl.add_partition_offset(split.topic.as_str(), split.partition, Offset::End)?;
+            } else if let Some(offset) = split.start_offset {
                 tpl.add_partition_offset(
                     split.topic.as_str(),
                     split.partition,
@@ -142,7 +149,7 @@ impl SplitReader for KafkaSplitReader {
     fn into_stream(self) -> BoxChunkSourceStream {
         let parser_config = self.parser_config.clone();
         let source_context = self.source_ctx.clone();
-        into_chunk_stream(self, parser_config, source_context)
+        into_chunk_stream(self.into_data_stream(), parser_config, source_context)
     }
 }
 
@@ -161,7 +168,7 @@ impl KafkaSplitReader {
     }
 }
 
-impl CommonSplitReader for KafkaSplitReader {
+impl KafkaSplitReader {
     #[try_stream(ok = Vec<SourceMessage>, error = crate::error::ConnectorError)]
     async fn into_data_stream(self) {
         if self.offsets.values().all(|(start_offset, stop_offset)| {
diff --git a/src/connector/src/source/kafka/split.rs b/src/connector/src/source/kafka/split.rs
index a649a729c0397..7ad0d52619b4a 100644
--- a/src/connector/src/source/kafka/split.rs
+++ b/src/connector/src/source/kafka/split.rs
@@ -24,6 +24,12 @@ pub struct KafkaSplit {
     pub(crate) partition: i32,
     pub(crate) start_offset: Option<i64>,
     pub(crate) stop_offset: Option<i64>,
+    #[serde(skip)]
+    /// Used by shared source to hackily seek to the latest offset without fetching start offset first.
+    /// XXX: But why do we fetch low watermark for latest start offset..?
+    ///
+    /// When this is `true`, `start_offset` will be ignored.
+    pub(crate) hack_seek_to_latest: bool,
 }
 
 impl SplitMetaData for KafkaSplit {
@@ -58,10 +64,16 @@ impl KafkaSplit {
             partition,
             start_offset,
             stop_offset,
+            hack_seek_to_latest: false,
         }
     }
 
     pub fn get_topic_and_partition(&self) -> (String, i32) {
         (self.topic.clone(), self.partition)
     }
+
+    /// This should only be used for a fresh split, not persisted in state table yet.
+    pub fn seek_to_latest_offset(&mut self) {
+        self.hack_seek_to_latest = true;
+    }
 }
diff --git a/src/connector/src/source/kinesis/enumerator/client.rs b/src/connector/src/source/kinesis/enumerator/client.rs
index b8966f99ae1ac..840def08f6855 100644
--- a/src/connector/src/source/kinesis/enumerator/client.rs
+++ b/src/connector/src/source/kinesis/enumerator/client.rs
@@ -19,7 +19,7 @@ use aws_sdk_kinesis::Client as kinesis_client;
 use risingwave_common::bail;
 
 use crate::error::ConnectorResult as Result;
-use crate::source::kinesis::split::{KinesisOffset, KinesisSplit};
+use crate::source::kinesis::split::KinesisOffset;
 use crate::source::kinesis::*;
 use crate::source::{SourceEnumeratorContextRef, SplitEnumerator};
 
diff --git a/src/connector/src/source/kinesis/mod.rs b/src/connector/src/source/kinesis/mod.rs
index 019cefb500643..de393a5121961 100644
--- a/src/connector/src/source/kinesis/mod.rs
+++ b/src/connector/src/source/kinesis/mod.rs
@@ -65,8 +65,6 @@ impl crate::source::UnknownFields for KinesisProperties {
 
 #[cfg(test)]
 mod test {
-    use std::collections::HashMap;
-
     use maplit::hashmap;
 
     use super::*;
diff --git a/src/connector/src/source/kinesis/source/message.rs b/src/connector/src/source/kinesis/source/message.rs
index 51a3a9f2ed1ec..c12679437c1f2 100644
--- a/src/connector/src/source/kinesis/source/message.rs
+++ b/src/connector/src/source/kinesis/source/message.rs
@@ -17,12 +17,10 @@ use aws_smithy_types_convert::date_time::DateTimeExt;
 
 use crate::source::{SourceMessage, SourceMeta, SplitId};
 
-#[derive(Clone, Debug)]
-pub struct KinesisMessage {}
-
 #[derive(Clone, Debug)]
 pub struct KinesisMeta {
     // from `approximate_arrival_timestamp` of type `Option<aws_smithy_types::DateTime>`
+    #[expect(dead_code)]
     timestamp: Option<i64>,
 }
 
diff --git a/src/connector/src/source/kinesis/source/reader.rs b/src/connector/src/source/kinesis/source/reader.rs
index c9026428d1df0..b728270dbd821 100644
--- a/src/connector/src/source/kinesis/source/reader.rs
+++ b/src/connector/src/source/kinesis/source/reader.rs
@@ -24,7 +24,6 @@ use aws_sdk_kinesis::Client as KinesisClient;
 use futures_async_stream::try_stream;
 use risingwave_common::bail;
 use thiserror_ext::AsReport;
-use tokio_retry;
 
 use crate::error::ConnectorResult as Result;
 use crate::parser::ParserConfig;
@@ -32,8 +31,8 @@ use crate::source::kinesis::source::message::from_kinesis_record;
 use crate::source::kinesis::split::{KinesisOffset, KinesisSplit};
 use crate::source::kinesis::KinesisProperties;
 use crate::source::{
-    into_chunk_stream, BoxChunkSourceStream, Column, CommonSplitReader, SourceContextRef,
-    SourceMessage, SplitId, SplitMetaData, SplitReader,
+    into_chunk_stream, BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SplitId,
+    SplitMetaData, SplitReader,
 };
 
 #[derive(Debug, Clone)]
@@ -44,6 +43,7 @@ pub struct KinesisSplitReader {
     latest_offset: Option<String>,
     shard_iter: Option<String>,
     start_position: KinesisOffset,
+    #[expect(dead_code)]
     end_position: KinesisOffset,
 
     split_id: SplitId,
@@ -115,11 +115,11 @@ impl SplitReader for KinesisSplitReader {
     fn into_stream(self) -> BoxChunkSourceStream {
         let parser_config = self.parser_config.clone();
         let source_context = self.source_ctx.clone();
-        into_chunk_stream(self, parser_config, source_context)
+        into_chunk_stream(self.into_data_stream(), parser_config, source_context)
     }
 }
 
-impl CommonSplitReader for KinesisSplitReader {
+impl KinesisSplitReader {
     #[try_stream(ok = Vec < SourceMessage >, error = crate::error::ConnectorError)]
     async fn into_data_stream(mut self) {
         self.new_shard_iter().await?;
@@ -306,7 +306,6 @@ mod tests {
 
     use super::*;
     use crate::connector_common::KinesisCommon;
-    use crate::source::kinesis::split::KinesisSplit;
     use crate::source::SourceContext;
 
     #[tokio::test]
diff --git a/src/connector/src/source/mod.rs b/src/connector/src/source/mod.rs
index f965d373d9306..ed8842e70825f 100644
--- a/src/connector/src/source/mod.rs
+++ b/src/connector/src/source/mod.rs
@@ -25,8 +25,10 @@ pub mod mqtt;
 pub mod nats;
 pub mod nexmark;
 pub mod pulsar;
+
 pub use base::{UPSTREAM_SOURCE_KEY, *};
 pub(crate) use common::*;
+use google_cloud_pubsub::subscription::Subscription;
 pub use google_pubsub::GOOGLE_PUBSUB_CONNECTOR;
 pub use kafka::KAFKA_CONNECTOR;
 pub use kinesis::KINESIS_CONNECTOR;
@@ -39,6 +41,8 @@ pub mod reader;
 pub mod test_source;
 
 pub use manager::{SourceColumnDesc, SourceColumnType};
+use risingwave_common::array::{Array, ArrayRef};
+use thiserror_ext::AsReport;
 
 pub use crate::source::filesystem::opendal_source::{
     GCS_CONNECTOR, OPENDAL_S3_CONNECTOR, POSIX_FS_CONNECTOR,
@@ -68,3 +72,57 @@ pub fn should_copy_to_format_encode_options(key: &str, connector: &str) -> bool
     PREFIXES.iter().any(|prefix| key.starts_with(prefix))
         || (key == "endpoint" && !connector.eq_ignore_ascii_case(KINESIS_CONNECTOR))
 }
+
+/// Tasks executed by `WaitCheckpointWorker`
+pub enum WaitCheckpointTask {
+    CommitCdcOffset(Option<(SplitId, String)>),
+    AckPubsubMessage(Subscription, Vec<ArrayRef>),
+}
+
+impl WaitCheckpointTask {
+    pub async fn run(self) {
+        use std::str::FromStr;
+        match self {
+            WaitCheckpointTask::CommitCdcOffset(updated_offset) => {
+                if let Some((split_id, offset)) = updated_offset {
+                    let source_id: u64 = u64::from_str(split_id.as_ref()).unwrap();
+                    // notify cdc connector to commit offset
+                    match cdc::jni_source::commit_cdc_offset(source_id, offset.clone()) {
+                        Ok(()) => {}
+                        Err(e) => {
+                            tracing::error!(
+                                error = %e.as_report(),
+                                "source#{source_id}: failed to commit cdc offset: {offset}.",
+                            )
+                        }
+                    }
+                }
+            }
+            WaitCheckpointTask::AckPubsubMessage(subscription, ack_id_arrs) => {
+                async fn ack(subscription: &Subscription, ack_ids: Vec<String>) {
+                    tracing::trace!("acking pubsub messages {:?}", ack_ids);
+                    match subscription.ack(ack_ids).await {
+                        Ok(()) => {}
+                        Err(e) => {
+                            tracing::error!(
+                                error = %e.as_report(),
+                                "failed to ack pubsub messages",
+                            )
+                        }
+                    }
+                }
+                const MAX_ACK_BATCH_SIZE: usize = 1000;
+                let mut ack_ids: Vec<String> = vec![];
+                for arr in ack_id_arrs {
+                    for ack_id in arr.as_utf8().iter().flatten() {
+                        ack_ids.push(ack_id.to_string());
+                        if ack_ids.len() >= MAX_ACK_BATCH_SIZE {
+                            ack(&subscription, std::mem::take(&mut ack_ids)).await;
+                        }
+                    }
+                }
+                ack(&subscription, ack_ids).await;
+            }
+        }
+    }
+}
diff --git a/src/connector/src/source/mqtt/enumerator/mod.rs b/src/connector/src/source/mqtt/enumerator/mod.rs
index 8d4dc636cd00a..f2b949fca17b6 100644
--- a/src/connector/src/source/mqtt/enumerator/mod.rs
+++ b/src/connector/src/source/mqtt/enumerator/mod.rs
@@ -29,7 +29,9 @@ use crate::error::ConnectorResult;
 use crate::source::{SourceEnumeratorContextRef, SplitEnumerator};
 
 pub struct MqttSplitEnumerator {
+    #[expect(dead_code)]
     topic: String,
+    #[expect(dead_code)]
     client: rumqttc::v5::AsyncClient,
     topics: Arc<RwLock<HashSet<String>>>,
     connected: Arc<AtomicBool>,
diff --git a/src/connector/src/source/mqtt/source/reader.rs b/src/connector/src/source/mqtt/source/reader.rs
index 50f90c816390c..2c57c8b9966b1 100644
--- a/src/connector/src/source/mqtt/source/reader.rs
+++ b/src/connector/src/source/mqtt/source/reader.rs
@@ -23,7 +23,7 @@ use super::message::MqttMessage;
 use super::MqttSplit;
 use crate::error::ConnectorResult as Result;
 use crate::parser::ParserConfig;
-use crate::source::common::{into_chunk_stream, CommonSplitReader};
+use crate::source::common::into_chunk_stream;
 use crate::source::mqtt::MqttProperties;
 use crate::source::{BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SplitReader};
 
@@ -32,6 +32,7 @@ pub struct MqttSplitReader {
     client: rumqttc::v5::AsyncClient,
     qos: QoS,
     splits: Vec<MqttSplit>,
+    #[expect(dead_code)]
     properties: MqttProperties,
     parser_config: ParserConfig,
     source_ctx: SourceContextRef,
@@ -78,11 +79,11 @@ impl SplitReader for MqttSplitReader {
     fn into_stream(self) -> BoxChunkSourceStream {
         let parser_config = self.parser_config.clone();
         let source_context = self.source_ctx.clone();
-        into_chunk_stream(self, parser_config, source_context)
+        into_chunk_stream(self.into_data_stream(), parser_config, source_context)
     }
 }
 
-impl CommonSplitReader for MqttSplitReader {
+impl MqttSplitReader {
     #[try_stream(ok = Vec<SourceMessage>, error = crate::error::ConnectorError)]
     async fn into_data_stream(self) {
         let mut eventloop = self.eventloop;
diff --git a/src/connector/src/source/nats/enumerator/mod.rs b/src/connector/src/source/nats/enumerator/mod.rs
index 557921747b8f0..29c4f93ef603f 100644
--- a/src/connector/src/source/nats/enumerator/mod.rs
+++ b/src/connector/src/source/nats/enumerator/mod.rs
@@ -25,6 +25,7 @@ use crate::source::{SourceEnumeratorContextRef, SplitEnumerator, SplitId};
 #[derive(Debug, Clone)]
 pub struct NatsSplitEnumerator {
     subject: String,
+    #[expect(dead_code)]
     split_id: SplitId,
     client: async_nats::Client,
 }
diff --git a/src/connector/src/source/nats/source/message.rs b/src/connector/src/source/nats/source/message.rs
index 3944cd6bd4c71..55c91457d4bfc 100644
--- a/src/connector/src/source/nats/source/message.rs
+++ b/src/connector/src/source/nats/source/message.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use async_nats;
 use async_nats::jetstream::Message;
 
 use crate::source::base::SourceMessage;
diff --git a/src/connector/src/source/nats/source/reader.rs b/src/connector/src/source/nats/source/reader.rs
index 05954538b91cf..916378a263979 100644
--- a/src/connector/src/source/nats/source/reader.rs
+++ b/src/connector/src/source/nats/source/reader.rs
@@ -23,7 +23,7 @@ use super::message::NatsMessage;
 use super::{NatsOffset, NatsSplit};
 use crate::error::ConnectorResult as Result;
 use crate::parser::ParserConfig;
-use crate::source::common::{into_chunk_stream, CommonSplitReader};
+use crate::source::common::into_chunk_stream;
 use crate::source::nats::NatsProperties;
 use crate::source::{
     BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SplitId, SplitReader,
@@ -31,9 +31,11 @@ use crate::source::{
 
 pub struct NatsSplitReader {
     consumer: consumer::Consumer<consumer::pull::Config>,
+    #[expect(dead_code)]
     properties: NatsProperties,
     parser_config: ParserConfig,
     source_ctx: SourceContextRef,
+    #[expect(dead_code)]
     start_position: NatsOffset,
     split_id: SplitId,
 }
@@ -98,11 +100,11 @@ impl SplitReader for NatsSplitReader {
     fn into_stream(self) -> BoxChunkSourceStream {
         let parser_config = self.parser_config.clone();
         let source_context = self.source_ctx.clone();
-        into_chunk_stream(self, parser_config, source_context)
+        into_chunk_stream(self.into_data_stream(), parser_config, source_context)
     }
 }
 
-impl CommonSplitReader for NatsSplitReader {
+impl NatsSplitReader {
     #[try_stream(ok = Vec<SourceMessage>, error = crate::error::ConnectorError)]
     async fn into_data_stream(self) {
         let capacity = self.source_ctx.source_ctrl_opts.chunk_size;
diff --git a/src/connector/src/source/nexmark/source/reader.rs b/src/connector/src/source/nexmark/source/reader.rs
index 6441baa154ae4..aee7c178d116a 100644
--- a/src/connector/src/source/nexmark/source/reader.rs
+++ b/src/connector/src/source/nexmark/source/reader.rs
@@ -41,6 +41,7 @@ use crate::source::{
 #[derive(Debug)]
 pub struct NexmarkSplitReader {
     generator: EventGenerator,
+    #[expect(dead_code)]
     assigned_split: NexmarkSplit,
     event_num: u64,
     event_type: Option<EventType>,
@@ -210,7 +211,7 @@ impl NexmarkSplitReader {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::source::nexmark::{NexmarkProperties, NexmarkSplitEnumerator};
+    use crate::source::nexmark::NexmarkSplitEnumerator;
     use crate::source::{SourceContext, SourceEnumeratorContext, SplitEnumerator};
 
     #[tokio::test]
diff --git a/src/connector/src/source/pulsar/source/reader.rs b/src/connector/src/source/pulsar/source/reader.rs
index ce808b96200cf..105eb70fbeda8 100644
--- a/src/connector/src/source/pulsar/source/reader.rs
+++ b/src/connector/src/source/pulsar/source/reader.rs
@@ -27,7 +27,8 @@ use itertools::Itertools;
 use pulsar::consumer::InitialPosition;
 use pulsar::message::proto::MessageIdData;
 use pulsar::{Consumer, ConsumerBuilder, ConsumerOptions, Pulsar, SubType, TokioExecutor};
-use risingwave_common::array::{DataChunk, StreamChunk};
+use risingwave_common::array::arrow::{FromArrow, IcebergArrowConvert};
+use risingwave_common::array::StreamChunk;
 use risingwave_common::catalog::ROWID_PREFIX;
 use risingwave_common::{bail, ensure};
 use thiserror_ext::AsReport;
@@ -37,8 +38,8 @@ use crate::parser::ParserConfig;
 use crate::source::pulsar::split::PulsarSplit;
 use crate::source::pulsar::{PulsarEnumeratorOffset, PulsarProperties};
 use crate::source::{
-    into_chunk_stream, BoxChunkSourceStream, Column, CommonSplitReader, SourceContextRef,
-    SourceMessage, SplitId, SplitMetaData, SplitReader,
+    into_chunk_stream, BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SplitId,
+    SplitMetaData, SplitReader,
 };
 
 pub enum PulsarSplitReader {
@@ -88,7 +89,11 @@ impl SplitReader for PulsarSplitReader {
             Self::Broker(reader) => {
                 let (parser_config, source_context) =
                     (reader.parser_config.clone(), reader.source_ctx.clone());
-                Box::pin(into_chunk_stream(reader, parser_config, source_context))
+                Box::pin(into_chunk_stream(
+                    reader.into_data_stream(),
+                    parser_config,
+                    source_context,
+                ))
             }
             Self::Iceberg(reader) => Box::pin(reader.into_stream()),
         }
@@ -97,10 +102,12 @@ impl SplitReader for PulsarSplitReader {
 
 /// This reader reads from pulsar broker
 pub struct PulsarBrokerReader {
+    #[expect(dead_code)]
     pulsar: Pulsar<TokioExecutor>,
     consumer: Consumer<Vec<u8>, TokioExecutor>,
+    #[expect(dead_code)]
     split: PulsarSplit,
-
+    #[expect(dead_code)]
     split_id: SplitId,
     parser_config: ParserConfig,
     source_ctx: SourceContextRef,
@@ -180,12 +187,16 @@ impl SplitReader for PulsarBrokerReader {
                     )
                 } else {
                     builder.with_options(
-                        ConsumerOptions::default().with_initial_position(InitialPosition::Earliest),
+                        ConsumerOptions::default()
+                            .with_initial_position(InitialPosition::Earliest)
+                            .durable(false),
                     )
                 }
             }
             PulsarEnumeratorOffset::Latest => builder.with_options(
-                ConsumerOptions::default().with_initial_position(InitialPosition::Latest),
+                ConsumerOptions::default()
+                    .with_initial_position(InitialPosition::Latest)
+                    .durable(false),
             ),
             PulsarEnumeratorOffset::MessageId(m) => {
                 if topic.starts_with("non-persistent://") {
@@ -226,11 +237,11 @@ impl SplitReader for PulsarBrokerReader {
     fn into_stream(self) -> BoxChunkSourceStream {
         let parser_config = self.parser_config.clone();
         let source_context = self.source_ctx.clone();
-        into_chunk_stream(self, parser_config, source_context)
+        into_chunk_stream(self.into_data_stream(), parser_config, source_context)
     }
 }
 
-impl CommonSplitReader for PulsarBrokerReader {
+impl PulsarBrokerReader {
     #[try_stream(ok = Vec<SourceMessage>, error = crate::error::ConnectorError)]
     async fn into_data_stream(self) {
         let max_chunk_size = self.source_ctx.source_ctrl_opts.chunk_size;
@@ -246,7 +257,9 @@ impl CommonSplitReader for PulsarBrokerReader {
     }
 }
 
+#[expect(dead_code)]
 const META_COLUMN_TOPIC: &str = "__topic";
+#[expect(dead_code)]
 const META_COLUMN_KEY: &str = "__key";
 const META_COLUMN_LEDGER_ID: &str = "__ledgerId";
 const META_COLUMN_ENTRY_ID: &str = "__entryId";
@@ -508,7 +521,8 @@ impl PulsarIcebergReader {
             offsets.push(offset);
         }
 
-        let data_chunk = DataChunk::try_from(&record_batch.project(&field_indices)?)
+        let data_chunk = IcebergArrowConvert
+            .from_record_batch(&record_batch.project(&field_indices)?)
             .context("failed to convert arrow record batch to data chunk")?;
 
         let stream_chunk = StreamChunk::from(data_chunk);
diff --git a/src/connector/src/source/pulsar/topic.rs b/src/connector/src/source/pulsar/topic.rs
index 352c7e47d8da2..f4cb95bd01437 100644
--- a/src/connector/src/source/pulsar/topic.rs
+++ b/src/connector/src/source/pulsar/topic.rs
@@ -108,6 +108,7 @@ pub fn get_partition_index(topic: &str) -> Result<Option<i32>> {
 /// The short topic name can be:
 /// - `<topic>`
 /// - `<tenant>/<namespace>/<topic>`
+///
 /// The fully qualified topic name can be:
 /// `<domain>://<tenant>/<namespace>/<topic>`
 pub fn parse_topic(topic: &str) -> Result<Topic> {
diff --git a/src/connector/src/source/reader/desc.rs b/src/connector/src/source/reader/desc.rs
index 808ef1232c50c..d9076e6a78fa9 100644
--- a/src/connector/src/source/reader/desc.rs
+++ b/src/connector/src/source/reader/desc.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::sync::Arc;
 
 use itertools::Itertools;
@@ -26,11 +26,10 @@ use risingwave_pb::plan_common::PbColumnCatalog;
 use super::fs_reader::FsSourceReader;
 use super::reader::SourceReader;
 use crate::error::ConnectorResult;
-use crate::parser::additional_columns::add_partition_offset_cols;
+use crate::parser::additional_columns::source_add_partition_offset_cols;
 use crate::parser::{EncodingProperties, ProtocolProperties, SpecificParserConfig};
 use crate::source::monitor::SourceMetrics;
 use crate::source::{SourceColumnDesc, SourceColumnType, UPSTREAM_SOURCE_KEY};
-use crate::ConnectorParams;
 
 pub const DEFAULT_CONNECTOR_MESSAGE_BUFFER_SIZE: usize = 16;
 
@@ -57,9 +56,8 @@ pub struct SourceDescBuilder {
     columns: Vec<PbColumnCatalog>,
     metrics: Arc<SourceMetrics>,
     row_id_index: Option<usize>,
-    with_properties: HashMap<String, String>,
+    with_properties: BTreeMap<String, String>,
     source_info: PbStreamSourceInfo,
-    connector_params: ConnectorParams,
     connector_message_buffer_size: usize,
     pk_indices: Vec<usize>,
 }
@@ -70,9 +68,8 @@ impl SourceDescBuilder {
         columns: Vec<PbColumnCatalog>,
         metrics: Arc<SourceMetrics>,
         row_id_index: Option<usize>,
-        with_properties: HashMap<String, String>,
+        with_properties: BTreeMap<String, String>,
         source_info: PbStreamSourceInfo,
-        connector_params: ConnectorParams,
         connector_message_buffer_size: usize,
         pk_indices: Vec<usize>,
     ) -> Self {
@@ -82,7 +79,6 @@ impl SourceDescBuilder {
             row_id_index,
             with_properties,
             source_info,
-            connector_params,
             connector_message_buffer_size,
             pk_indices,
         }
@@ -102,7 +98,7 @@ impl SourceDescBuilder {
             .map(|c| ColumnCatalog::from(c.clone()))
             .collect_vec();
         let (columns_exist, additional_columns) =
-            add_partition_offset_cols(&columns, &connector_name);
+            source_add_partition_offset_cols(&columns, &connector_name);
 
         let mut columns: Vec<_> = self
             .columns
@@ -186,7 +182,7 @@ impl SourceDescBuilder {
 }
 
 pub mod test_utils {
-    use std::collections::HashMap;
+    use std::collections::BTreeMap;
 
     use risingwave_common::catalog::{ColumnDesc, Schema};
     use risingwave_pb::catalog::StreamSourceInfo;
@@ -198,7 +194,7 @@ pub mod test_utils {
         schema: &Schema,
         row_id_index: Option<usize>,
         source_info: StreamSourceInfo,
-        with_properties: HashMap<String, String>,
+        with_properties: BTreeMap<String, String>,
         pk_indices: Vec<usize>,
     ) -> SourceDescBuilder {
         let columns = schema
@@ -223,7 +219,6 @@ pub mod test_utils {
             row_id_index,
             with_properties,
             source_info,
-            connector_params: Default::default(),
             connector_message_buffer_size: DEFAULT_CONNECTOR_MESSAGE_BUFFER_SIZE,
             pk_indices,
         }
diff --git a/src/connector/src/source/reader/fs_reader.rs b/src/connector/src/source/reader/fs_reader.rs
index 5ec47cea98cee..5bd139e70983d 100644
--- a/src/connector/src/source/reader/fs_reader.rs
+++ b/src/connector/src/source/reader/fs_reader.rs
@@ -14,7 +14,7 @@
 
 #![deprecated = "will be replaced by new fs source (list + fetch)"]
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::sync::Arc;
 
 use anyhow::Context;
@@ -34,20 +34,19 @@ use crate::source::{
 pub struct FsSourceReader {
     pub config: ConnectorProperties,
     pub columns: Vec<SourceColumnDesc>,
-    pub properties: HashMap<String, String>,
+    pub properties: BTreeMap<String, String>,
     pub parser_config: SpecificParserConfig,
 }
 
 impl FsSourceReader {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
-        properties: HashMap<String, String>,
+        properties: BTreeMap<String, String>,
         columns: Vec<SourceColumnDesc>,
         parser_config: SpecificParserConfig,
     ) -> ConnectorResult<Self> {
         // Store the connector node address to properties for later use.
-        let source_props: HashMap<String, String> = HashMap::from_iter(properties.clone());
-        let config = ConnectorProperties::extract(source_props, false)?;
+        let config = ConnectorProperties::extract(properties.clone(), false)?;
 
         Ok(Self {
             config,
diff --git a/src/connector/src/source/reader/reader.rs b/src/connector/src/source/reader/reader.rs
index e8c20f66dedc4..02012841c5a48 100644
--- a/src/connector/src/source/reader/reader.rs
+++ b/src/connector/src/source/reader/reader.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::sync::Arc;
 
 use anyhow::Context;
@@ -36,7 +36,7 @@ use crate::source::filesystem::opendal_source::{
 use crate::source::filesystem::{FsPageItem, OpendalFsSplit};
 use crate::source::{
     create_split_reader, BoxChunkSourceStream, BoxTryStream, Column, ConnectorProperties,
-    ConnectorState, SourceColumnDesc, SourceContext, SplitReader,
+    ConnectorState, SourceColumnDesc, SourceContext, SplitReader, WaitCheckpointTask,
 };
 
 #[derive(Clone, Debug)]
@@ -49,7 +49,7 @@ pub struct SourceReader {
 
 impl SourceReader {
     pub fn new(
-        properties: HashMap<String, String>,
+        properties: BTreeMap<String, String>,
         columns: Vec<SourceColumnDesc>,
         connector_message_buffer_size: usize,
         parser_config: SpecificParserConfig,
@@ -105,18 +105,22 @@ impl SourceReader {
         }
     }
 
-    /// Postgres and Oracle connectors need to commit the offset to upstream.
-    pub fn need_commit_offset_to_upstream(&self) -> bool {
-        matches!(
-            &self.config,
-            ConnectorProperties::PostgresCdc(_)
-                | ConnectorProperties::MysqlCdc(_)
-                | ConnectorProperties::MongodbCdc(_)
-                | ConnectorProperties::CitusCdc(_)
-        )
+    /// Refer to `WaitCheckpointWorker` for more details.
+    pub async fn create_wait_checkpoint_task(&self) -> ConnectorResult<Option<WaitCheckpointTask>> {
+        Ok(match &self.config {
+            ConnectorProperties::PostgresCdc(_prop) => {
+                Some(WaitCheckpointTask::CommitCdcOffset(None))
+            }
+            ConnectorProperties::GooglePubsub(prop) => Some(WaitCheckpointTask::AckPubsubMessage(
+                prop.subscription_client().await?,
+                vec![],
+            )),
+            _ => None,
+        })
     }
 
-    pub async fn to_stream(
+    /// Build `SplitReader`s and then `BoxChunkSourceStream` from the given `ConnectorState` (`SplitImpl`s).
+    pub async fn build_stream(
         &self,
         state: ConnectorState,
         column_ids: Vec<ColumnId>,
@@ -161,7 +165,7 @@ impl SourceReader {
             } else {
                 let to_reader_splits = splits.into_iter().map(|split| vec![split]);
                 try_join_all(to_reader_splits.into_iter().map(|splits| {
-                    tracing::debug!(?splits, ?prop, "spawning connector split reader");
+                    tracing::debug!(?splits, "spawning connector split reader");
                     let props = prop.clone();
                     let data_gen_columns = data_gen_columns.clone();
                     let parser_config = parser_config.clone();
diff --git a/src/connector/src/source/test_source.rs b/src/connector/src/source/test_source.rs
index e8379fafeeb71..57f659873936b 100644
--- a/src/connector/src/source/test_source.rs
+++ b/src/connector/src/source/test_source.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::sync::{Arc, OnceLock};
 
 use async_trait::async_trait;
@@ -26,7 +26,7 @@ use crate::error::ConnectorResult;
 use crate::parser::ParserConfig;
 use crate::source::{
     BoxChunkSourceStream, Column, SourceContextRef, SourceEnumeratorContextRef, SourceProperties,
-    SplitEnumerator, SplitId, SplitMetaData, SplitReader, TryFromHashmap,
+    SplitEnumerator, SplitId, SplitMetaData, SplitReader, TryFromBTreeMap,
 };
 
 pub type BoxListSplits = Box<
@@ -118,12 +118,12 @@ pub const TEST_CONNECTOR: &str = "test";
 
 #[derive(Clone, Debug, Default, WithOptions)]
 pub struct TestSourceProperties {
-    properties: HashMap<String, String>,
+    properties: BTreeMap<String, String>,
 }
 
-impl TryFromHashmap for TestSourceProperties {
-    fn try_from_hashmap(
-        props: HashMap<String, String>,
+impl TryFromBTreeMap for TestSourceProperties {
+    fn try_from_btreemap(
+        props: BTreeMap<String, String>,
         _deny_unknown_fields: bool,
     ) -> ConnectorResult<Self> {
         if cfg!(any(madsim, test)) {
diff --git a/src/connector/src/with_options.rs b/src/connector/src/with_options.rs
index 5b7e75a47bcd6..3207a7bbbde2f 100644
--- a/src/connector/src/with_options.rs
+++ b/src/connector/src/with_options.rs
@@ -48,10 +48,12 @@ impl<T: crate::source::cdc::CdcSourceTypeTrait> WithOptions
 impl<T: WithOptions> WithOptions for Option<T> {}
 impl WithOptions for Vec<String> {}
 impl WithOptions for HashMap<String, String> {}
+impl WithOptions for BTreeMap<String, String> {}
 
 impl WithOptions for String {}
 impl WithOptions for bool {}
 impl WithOptions for usize {}
+impl WithOptions for u16 {}
 impl WithOptions for u32 {}
 impl WithOptions for u64 {}
 impl WithOptions for i32 {}
diff --git a/src/connector/src/with_options_test.rs b/src/connector/src/with_options_test.rs
index 155932ce90452..cd2adc11d0718 100644
--- a/src/connector/src/with_options_test.rs
+++ b/src/connector/src/with_options_test.rs
@@ -63,7 +63,7 @@ pub fn generate_with_options_yaml_sink() -> String {
 /// the generated `yaml` might be inconsistent with the actual parsing logic.
 /// TODO: improve the test to check whether serde is used.
 ///
-/// - For sources, the parsing logic is in `TryFromHashMap`.
+/// - For sources, the parsing logic is in `TryFromBTreeMap`.
 /// - For sinks, the parsing logic is in `TryFrom<SinkParam>`.
 fn generate_with_options_yaml_inner(path: &Path) -> String {
     let mut structs = vec![];
@@ -191,15 +191,15 @@ struct FieldInfo {
     #[serde(skip_serializing_if = "Option::is_none")]
     default: Option<String>,
 
-    #[serde(skip_serializing_if = "Option::is_none")]
-    alias: Option<String>,
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    alias: Vec<String>,
 }
 
 #[derive(Default)]
 struct SerdeProperties {
     default_func: Option<String>,
     rename: Option<String>,
-    alias: Option<String>,
+    alias: Vec<String>,
 }
 
 #[derive(Debug, Serialize, Default)]
@@ -270,7 +270,7 @@ fn extract_serde_properties(field: &Field) -> SerdeProperties {
                                 }
                             } else if path.is_ident("alias") {
                                 if let Lit::Str(lit_str) = lit {
-                                    serde_props.alias = Some(lit_str.value());
+                                    serde_props.alias.push(lit_str.value());
                                 }
                             } else if path.is_ident("default") {
                                 if let Lit::Str(lit_str) = lit {
diff --git a/src/connector/with_options_sink.yaml b/src/connector/with_options_sink.yaml
index 27daa718b64f9..731bb900335ee 100644
--- a/src/connector/with_options_sink.yaml
+++ b/src/connector/with_options_sink.yaml
@@ -21,32 +21,49 @@ BigQueryConfig:
     field_type: usize
     required: false
     default: '1024'
-  - name: region
+  - name: aws.region
     field_type: String
     required: false
-  - name: endpoint
+    alias:
+    - region
+  - name: aws.endpoint_url
     field_type: String
     required: false
-    alias: endpoint_url
-  - name: access_key
+    alias:
+    - endpoint_url
+    - endpoint
+  - name: aws.credentials.access_key_id
     field_type: String
     required: false
-  - name: secret_key
+    alias:
+    - access_key
+  - name: aws.credentials.secret_access_key
     field_type: String
     required: false
-  - name: session_token
+    alias:
+    - secret_key
+  - name: aws.credentials.session_token
     field_type: String
     required: false
-  - name: arn
+    alias:
+    - session_token
+  - name: aws.credentials.role.arn
     field_type: String
+    comments: IAM role
     required: false
-  - name: external_id
+    alias:
+    - arn
+  - name: aws.credentials.role.external_id
     field_type: String
-    comments: This field was added for kinesis. Not sure if it's useful for other connectors.  Please ignore it in the documentation for now.
+    comments: external ID in IAM role trust policy
     required: false
-  - name: profile
+    alias:
+    - external_id
+  - name: aws.profile
     field_type: String
     required: false
+    alias:
+    - profile
   - name: r#type
     field_type: String
     required: true
@@ -90,6 +107,11 @@ DeltaLakeConfig:
   - name: gcs.service.account
     field_type: String
     required: false
+  - name: commit_checkpoint_interval
+    field_type: u64
+    comments: Commit every n(>0) checkpoints, if n is not set, we will commit every checkpoint.
+    required: false
+    default: Default::default
   - name: r#type
     field_type: String
     required: true
@@ -113,6 +135,82 @@ DorisConfig:
   - name: r#type
     field_type: String
     required: true
+DynamoDbConfig:
+  fields:
+  - name: table
+    field_type: String
+    required: true
+    alias:
+    - dynamodb.table
+  - name: dynamodb.max_batch_rows
+    field_type: usize
+    required: false
+    default: '1024'
+  - name: aws.region
+    field_type: String
+    required: false
+    alias:
+    - region
+  - name: aws.endpoint_url
+    field_type: String
+    required: false
+    alias:
+    - endpoint_url
+    - endpoint
+  - name: aws.credentials.access_key_id
+    field_type: String
+    required: false
+    alias:
+    - access_key
+  - name: aws.credentials.secret_access_key
+    field_type: String
+    required: false
+    alias:
+    - secret_key
+  - name: aws.credentials.session_token
+    field_type: String
+    required: false
+    alias:
+    - session_token
+  - name: aws.credentials.role.arn
+    field_type: String
+    comments: IAM role
+    required: false
+    alias:
+    - arn
+  - name: aws.credentials.role.external_id
+    field_type: String
+    comments: external ID in IAM role trust policy
+    required: false
+    alias:
+    - external_id
+  - name: aws.profile
+    field_type: String
+    required: false
+    alias:
+    - profile
+GooglePubSubConfig:
+  fields:
+  - name: pubsub.project_id
+    field_type: String
+    comments: The Google Pub/Sub Project ID
+    required: true
+  - name: pubsub.topic
+    field_type: String
+    comments: Specifies the Pub/Sub topic to publish messages
+    required: true
+  - name: pubsub.endpoint
+    field_type: String
+    comments: The Google Pub/Sub endpoint URL
+    required: true
+  - name: pubsub.emulator_host
+    field_type: String
+    comments: use the connector with a pubsub emulator  <https://cloud.google.com/pubsub/docs/emulator>
+    required: false
+  - name: pubsub.credentials
+    field_type: String
+    comments: A JSON string containing the service account credentials for authorization,  see the [service-account](https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account) credentials guide.  The provided account credential must have the  `pubsub.publisher` [role](https://cloud.google.com/pubsub/docs/access-control#roles)
+    required: false
 IcebergConfig:
   fields:
   - name: connector
@@ -171,11 +269,13 @@ KafkaConfig:
   - name: properties.bootstrap.server
     field_type: String
     required: true
-    alias: kafka.brokers
+    alias:
+    - kafka.brokers
   - name: topic
     field_type: String
     required: true
-    alias: kafka.topic
+    alias:
+    - kafka.topic
   - name: properties.sync.call.timeout
     field_type: Duration
     required: false
@@ -205,7 +305,7 @@ KafkaConfig:
     required: false
   - name: properties.sasl.mechanism
     field_type: String
-    comments: SASL mechanism if SASL is enabled. Currently support PLAIN, SCRAM and GSSAPI.
+    comments: SASL mechanism if SASL is enabled. Currently support PLAIN, SCRAM, GSSAPI, and AWS_MSK_IAM.
     required: false
   - name: properties.sasl.username
     field_type: String
@@ -319,44 +419,98 @@ KafkaConfig:
     comments: The maximum number of unacknowledged requests the client will send on a single connection before blocking.
     required: false
     default: '5'
+  - name: properties.request.required.acks
+    field_type: i32
+    required: false
   - name: broker.rewrite.endpoints
-    field_type: HashMap<String,String>
+    field_type: BTreeMap<String,String>
     comments: This is generated from `private_link_targets` and `private_link_endpoint` in frontend, instead of given by users.
     required: false
+  - name: aws.region
+    field_type: String
+    required: false
+    alias:
+    - region
+  - name: aws.endpoint_url
+    field_type: String
+    required: false
+    alias:
+    - endpoint_url
+    - endpoint
+  - name: aws.credentials.access_key_id
+    field_type: String
+    required: false
+    alias:
+    - access_key
+  - name: aws.credentials.secret_access_key
+    field_type: String
+    required: false
+    alias:
+    - secret_key
+  - name: aws.credentials.session_token
+    field_type: String
+    required: false
+    alias:
+    - session_token
+  - name: aws.credentials.role.arn
+    field_type: String
+    comments: IAM role
+    required: false
+    alias:
+    - arn
+  - name: aws.credentials.role.external_id
+    field_type: String
+    comments: external ID in IAM role trust policy
+    required: false
+    alias:
+    - external_id
+  - name: aws.profile
+    field_type: String
+    required: false
+    alias:
+    - profile
 KinesisSinkConfig:
   fields:
   - name: stream
     field_type: String
     required: true
-    alias: kinesis.stream.name
+    alias:
+    - kinesis.stream.name
   - name: aws.region
     field_type: String
     required: true
-    alias: kinesis.stream.region
+    alias:
+    - kinesis.stream.region
   - name: endpoint
     field_type: String
     required: false
-    alias: kinesis.endpoint
+    alias:
+    - kinesis.endpoint
   - name: aws.credentials.access_key_id
     field_type: String
     required: false
-    alias: kinesis.credentials.access
+    alias:
+    - kinesis.credentials.access
   - name: aws.credentials.secret_access_key
     field_type: String
     required: false
-    alias: kinesis.credentials.secret
+    alias:
+    - kinesis.credentials.secret
   - name: aws.credentials.session_token
     field_type: String
     required: false
-    alias: kinesis.credentials.session_token
+    alias:
+    - kinesis.credentials.session_token
   - name: aws.credentials.role.arn
     field_type: String
     required: false
-    alias: kinesis.assumerole.arn
+    alias:
+    - kinesis.assumerole.arn
   - name: aws.credentials.role.external_id
     field_type: String
     required: false
-    alias: kinesis.assumerole.external_id
+    alias:
+    - kinesis.assumerole.external_id
 MqttConfig:
   fields:
   - name: url
@@ -469,11 +623,13 @@ PulsarConfig:
   - name: topic
     field_type: String
     required: true
-    alias: pulsar.topic
+    alias:
+    - pulsar.topic
   - name: service.url
     field_type: String
     required: true
-    alias: pulsar.service.url
+    alias:
+    - pulsar.service.url
   - name: auth.token
     field_type: String
     required: false
@@ -489,32 +645,49 @@ PulsarConfig:
   - name: oauth.scope
     field_type: String
     required: false
-  - name: region
+  - name: aws.region
     field_type: String
     required: false
-  - name: endpoint
+    alias:
+    - region
+  - name: aws.endpoint_url
     field_type: String
     required: false
-    alias: endpoint_url
-  - name: access_key
+    alias:
+    - endpoint_url
+    - endpoint
+  - name: aws.credentials.access_key_id
     field_type: String
     required: false
-  - name: secret_key
+    alias:
+    - access_key
+  - name: aws.credentials.secret_access_key
     field_type: String
     required: false
-  - name: session_token
+    alias:
+    - secret_key
+  - name: aws.credentials.session_token
     field_type: String
     required: false
-  - name: arn
+    alias:
+    - session_token
+  - name: aws.credentials.role.arn
     field_type: String
+    comments: IAM role
     required: false
-  - name: external_id
+    alias:
+    - arn
+  - name: aws.credentials.role.external_id
     field_type: String
-    comments: This field was added for kinesis. Not sure if it's useful for other connectors.  Please ignore it in the documentation for now.
+    comments: external ID in IAM role trust policy
     required: false
-  - name: profile
+    alias:
+    - external_id
+  - name: aws.profile
     field_type: String
     required: false
+    alias:
+    - profile
   - name: properties.batch.size
     field_type: u32
     required: false
@@ -578,6 +751,33 @@ SnowflakeConfig:
     field_type: String
     comments: The s3 region, e.g., us-east-2
     required: true
+SqlServerConfig:
+  fields:
+  - name: sqlserver.host
+    field_type: String
+    required: true
+  - name: sqlserver.port
+    field_type: u16
+    required: true
+  - name: sqlserver.user
+    field_type: String
+    required: true
+  - name: sqlserver.password
+    field_type: String
+    required: true
+  - name: sqlserver.database
+    field_type: String
+    required: true
+  - name: sqlserver.table
+    field_type: String
+    required: true
+  - name: sqlserver.max_batch_rows
+    field_type: usize
+    required: false
+    default: '1024'
+  - name: r#type
+    field_type: String
+    required: true
 StarrocksConfig:
   fields:
   - name: starrocks.host
@@ -588,12 +788,14 @@ StarrocksConfig:
     field_type: String
     comments: The port to the MySQL server of `StarRocks` FE.
     required: true
-    alias: starrocks.query_port
+    alias:
+    - starrocks.query_port
   - name: starrocks.httpport
     field_type: String
     comments: The port to the HTTP server of `StarRocks` FE.
     required: true
-    alias: starrocks.http_port
+    alias:
+    - starrocks.http_port
   - name: starrocks.user
     field_type: String
     comments: The user name used to access the `StarRocks` database.
@@ -610,8 +812,19 @@ StarrocksConfig:
     field_type: String
     comments: The `StarRocks` table you want to sink data to.
     required: true
+  - name: starrocks.stream_load.http.timeout.ms
+    field_type: u64
+    comments: The timeout in milliseconds for stream load http request, defaults to 10 seconds.
+    required: false
+    default: 30 * 1000
+  - name: commit_checkpoint_interval
+    field_type: u64
+    comments: Set this option to a positive integer n, RisingWave will try to commit data  to Starrocks at every n checkpoints by leveraging the  [StreamLoad Transaction API](https://docs.starrocks.io/docs/loading/Stream_Load_transaction_interface/),  also, in this time, the `sink_decouple` option should be enabled as well.  Defaults to 1 if commit_checkpoint_interval <= 0
+    required: false
+    default: Default::default
   - name: starrocks.partial_update
     field_type: String
+    comments: Enable partial update
     required: false
   - name: r#type
     field_type: String
diff --git a/src/connector/with_options_source.yaml b/src/connector/with_options_source.yaml
index 2ea0e5f3488ee..822ab25ea3ef7 100644
--- a/src/connector/with_options_source.yaml
+++ b/src/connector/with_options_source.yaml
@@ -80,20 +80,25 @@ KafkaProperties:
     field_type: String
     comments: This parameter is not intended to be exposed to users.  This parameter specifies only for one parallelism. The parallelism of kafka source  is equal to the parallelism passed into compute nodes. So users need to calculate  how many bytes will be consumed in total across all the parallelism by themselves.
     required: false
-    alias: kafka.bytes.per.second
+    alias:
+    - kafka.bytes.per.second
   - name: max.num.messages
     field_type: String
     comments: This parameter is not intended to be exposed to users.  This parameter specifies only for one parallelism. The parallelism of kafka source  is equal to the parallelism passed into compute nodes. So users need to calculate  how many messages will be consumed in total across all the parallelism by themselves.
     required: false
-    alias: kafka.max.num.messages
+    alias:
+    - kafka.max.num.messages
   - name: scan.startup.mode
     field_type: String
     required: false
-    alias: kafka.scan.startup.mode
+    alias:
+    - kafka.scan.startup.mode
   - name: scan.startup.timestamp.millis
     field_type: String
     required: false
-    alias: scan.startup.timestamp_millis
+    alias:
+    - kafka.time.offset
+    - scan.startup.timestamp_millis
   - name: upsert
     field_type: String
     comments: 'This parameter is used to tell `KafkaSplitReader` to produce `UpsertMessage`s, which  combine both key and value fields of the Kafka message.  TODO: Currently, `Option<bool>` can not be parsed here.'
@@ -101,11 +106,13 @@ KafkaProperties:
   - name: properties.bootstrap.server
     field_type: String
     required: true
-    alias: kafka.brokers
+    alias:
+    - kafka.brokers
   - name: topic
     field_type: String
     required: true
-    alias: kafka.topic
+    alias:
+    - kafka.topic
   - name: properties.sync.call.timeout
     field_type: Duration
     required: false
@@ -135,7 +142,7 @@ KafkaProperties:
     required: false
   - name: properties.sasl.mechanism
     field_type: String
-    comments: SASL mechanism if SASL is enabled. Currently support PLAIN, SCRAM and GSSAPI.
+    comments: SASL mechanism if SASL is enabled. Currently support PLAIN, SCRAM, GSSAPI, and AWS_MSK_IAM.
     required: false
   - name: properties.sasl.username
     field_type: String
@@ -211,50 +218,102 @@ KafkaProperties:
     comments: 'Whether to automatically and periodically commit offsets in the background.   Note that RisingWave does NOT rely on committed offsets. Committing offset is only for exposing the  progress for monitoring. Setting this to false can avoid creating consumer groups.   default: true'
     required: false
   - name: broker.rewrite.endpoints
-    field_type: HashMap<String,String>
+    field_type: BTreeMap<String,String>
     comments: This is generated from `private_link_targets` and `private_link_endpoint` in frontend, instead of given by users.
     required: false
+  - name: aws.region
+    field_type: String
+    required: false
+    alias:
+    - region
+  - name: aws.endpoint_url
+    field_type: String
+    required: false
+    alias:
+    - endpoint_url
+    - endpoint
+  - name: aws.credentials.access_key_id
+    field_type: String
+    required: false
+    alias:
+    - access_key
+  - name: aws.credentials.secret_access_key
+    field_type: String
+    required: false
+    alias:
+    - secret_key
+  - name: aws.credentials.session_token
+    field_type: String
+    required: false
+    alias:
+    - session_token
+  - name: aws.credentials.role.arn
+    field_type: String
+    comments: IAM role
+    required: false
+    alias:
+    - arn
+  - name: aws.credentials.role.external_id
+    field_type: String
+    comments: external ID in IAM role trust policy
+    required: false
+    alias:
+    - external_id
+  - name: aws.profile
+    field_type: String
+    required: false
+    alias:
+    - profile
 KinesisProperties:
   fields:
   - name: scan.startup.mode
     field_type: String
     required: false
-    alias: kinesis.scan.startup.mode
+    alias:
+    - kinesis.scan.startup.mode
   - name: scan.startup.timestamp.millis
     field_type: i64
     required: false
   - name: stream
     field_type: String
     required: true
-    alias: kinesis.stream.name
+    alias:
+    - kinesis.stream.name
   - name: aws.region
     field_type: String
     required: true
-    alias: kinesis.stream.region
+    alias:
+    - kinesis.stream.region
   - name: endpoint
     field_type: String
     required: false
-    alias: kinesis.endpoint
+    alias:
+    - kinesis.endpoint
   - name: aws.credentials.access_key_id
     field_type: String
     required: false
-    alias: kinesis.credentials.access
+    alias:
+    - kinesis.credentials.access
   - name: aws.credentials.secret_access_key
     field_type: String
     required: false
-    alias: kinesis.credentials.secret
+    alias:
+    - kinesis.credentials.secret
   - name: aws.credentials.session_token
     field_type: String
     required: false
-    alias: kinesis.credentials.session_token
+    alias:
+    - kinesis.credentials.session_token
   - name: aws.credentials.role.arn
     field_type: String
     required: false
-    alias: kinesis.assumerole.arn
+    alias:
+    - kinesis.assumerole.arn
   - name: aws.credentials.role.external_id
     field_type: String
     required: false
-    alias: kinesis.assumerole.external_id
+    alias:
+    - kinesis.assumerole.external_id
 MqttProperties:
   fields:
   - name: url
@@ -350,7 +409,8 @@ NatsProperties:
   - name: scan.startup.timestamp.millis
     field_type: String
     required: false
-    alias: scan.startup.timestamp_millis
+    alias:
+    - scan.startup.timestamp_millis
   - name: stream
     field_type: String
     required: true
@@ -569,7 +629,7 @@ PubsubProperties:
   fields:
   - name: pubsub.subscription
     field_type: String
-    comments: pubsub subscription to consume messages from  The subscription should be configured with the `retain-on-ack` property to enable  message recovery within risingwave.
+    comments: 'Pub/Sub subscription to consume messages from.   Note that we rely on Pub/Sub to load-balance messages between all Readers pulling from  the same subscription. So one `subscription` (i.e., one `Source`) can only used for one MV  (shared between the actors of its fragment).  Otherwise, different MVs on the same Source will both receive part of the messages.  TODO: check and enforce this on Meta.'
     required: true
   - name: pubsub.emulator_host
     field_type: String
@@ -585,26 +645,35 @@ PubsubProperties:
     required: false
   - name: pubsub.start_snapshot
     field_type: String
-    comments: '`start_snapshot` is a named pub/sub snapshot. If present, the connector will first seek  to the snapshot before starting consumption. Snapshots are the preferred seeking mechanism  in pub/sub because they guarantee retention of:  - All unacknowledged messages at the time of their creation.  - All messages created after their creation.  Besides retention guarantees, timestamps are also more precise than timestamp-based seeks.  See [Seeking to a snapshot](https://cloud.google.com/pubsub/docs/replay-overview#seeking_to_a_timestamp) for  more details.'
+    comments: '`start_snapshot` is a named pub/sub snapshot. If present, the connector will first seek  to the snapshot before starting consumption. Snapshots are the preferred seeking mechanism  in pub/sub because they guarantee retention of:  - All unacknowledged messages at the time of their creation.  - All messages created after their creation.  Besides retention guarantees, snapshots are also more precise than timestamp-based seeks.  See [Seeking to a snapshot](https://cloud.google.com/pubsub/docs/replay-overview#seeking_to_a_timestamp) for  more details.'
+    required: false
+  - name: pubsub.parallelism
+    field_type: u32
+    comments: '`parallelism` is the number of parallel consumers to run for the subscription.  TODO: use system parallelism if not set'
     required: false
 PulsarProperties:
   fields:
   - name: scan.startup.mode
     field_type: String
     required: false
-    alias: pulsar.scan.startup.mode
+    alias:
+    - pulsar.scan.startup.mode
   - name: scan.startup.timestamp.millis
     field_type: String
     required: false
-    alias: scan.startup.timestamp_millis
+    alias:
+    - pulsar.time.offset
+    - scan.startup.timestamp_millis
   - name: topic
     field_type: String
     required: true
-    alias: pulsar.topic
+    alias:
+    - pulsar.topic
   - name: service.url
     field_type: String
     required: true
-    alias: pulsar.service.url
+    alias:
+    - pulsar.service.url
   - name: auth.token
     field_type: String
     required: false
@@ -620,32 +689,49 @@ PulsarProperties:
   - name: oauth.scope
     field_type: String
     required: false
-  - name: region
+  - name: aws.region
     field_type: String
     required: false
-  - name: endpoint
+    alias:
+    - region
+  - name: aws.endpoint_url
     field_type: String
     required: false
-    alias: endpoint_url
-  - name: access_key
+    alias:
+    - endpoint_url
+    - endpoint
+  - name: aws.credentials.access_key_id
     field_type: String
     required: false
-  - name: secret_key
+    alias:
+    - access_key
+  - name: aws.credentials.secret_access_key
     field_type: String
     required: false
-  - name: session_token
+    alias:
+    - secret_key
+  - name: aws.credentials.session_token
     field_type: String
     required: false
-  - name: arn
+    alias:
+    - session_token
+  - name: aws.credentials.role.arn
     field_type: String
+    comments: IAM role
     required: false
-  - name: external_id
+    alias:
+    - arn
+  - name: aws.credentials.role.external_id
     field_type: String
-    comments: This field was added for kinesis. Not sure if it's useful for other connectors.  Please ignore it in the documentation for now.
+    comments: external ID in IAM role trust policy
     required: false
-  - name: profile
+    alias:
+    - external_id
+  - name: aws.profile
     field_type: String
     required: false
+    alias:
+    - profile
   - name: iceberg.enabled
     field_type: bool
     required: false
@@ -679,5 +765,5 @@ S3Properties:
 TestSourceProperties:
   fields:
   - name: properties
-    field_type: HashMap<String,String>
-    required: false
+    field_type: BTreeMap<String,String>
+    required: true
diff --git a/src/ctl/Cargo.toml b/src/ctl/Cargo.toml
index 109d412ba1e89..0b9f7bb3f1e4c 100644
--- a/src/ctl/Cargo.toml
+++ b/src/ctl/Cargo.toml
@@ -20,6 +20,7 @@ chrono = "0.4"
 clap = { workspace = true }
 comfy-table = "7"
 etcd-client = { workspace = true }
+foyer ={ workspace = true }
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 hex = "0.4"
 inquire = "0.7.0"
diff --git a/src/ctl/src/cmd_impl/await_tree.rs b/src/ctl/src/cmd_impl/await_tree.rs
index 09c7c36119262..1c4ff98562791 100644
--- a/src/ctl/src/cmd_impl/await_tree.rs
+++ b/src/ctl/src/cmd_impl/await_tree.rs
@@ -13,21 +13,15 @@
 // limitations under the License.
 
 use risingwave_common::util::addr::HostAddr;
+use risingwave_common::util::StackTraceResponseExt;
 use risingwave_pb::common::WorkerType;
 use risingwave_pb::monitor_service::StackTraceResponse;
 use risingwave_rpc_client::{CompactorClient, ComputeClientPool};
 
 use crate::CtlContext;
 
-fn merge(a: &mut StackTraceResponse, b: StackTraceResponse) {
-    a.actor_traces.extend(b.actor_traces);
-    a.rpc_traces.extend(b.rpc_traces);
-    a.compaction_task_traces.extend(b.compaction_task_traces);
-    a.inflight_barrier_traces.extend(b.inflight_barrier_traces);
-}
-
 pub async fn dump(context: &CtlContext) -> anyhow::Result<()> {
-    let mut all = Default::default();
+    let mut all = StackTraceResponse::default();
 
     let meta_client = context.meta_client().await?;
 
@@ -41,7 +35,7 @@ pub async fn dump(context: &CtlContext) -> anyhow::Result<()> {
     for cn in compute_nodes {
         let client = clients.get(&cn).await?;
         let response = client.stack_trace().await?;
-        merge(&mut all, response);
+        all.merge_other(response);
     }
 
     let compactor_nodes = meta_client
@@ -52,7 +46,7 @@ pub async fn dump(context: &CtlContext) -> anyhow::Result<()> {
         let addr: HostAddr = compactor.get_host().unwrap().into();
         let client = CompactorClient::new(addr).await?;
         let response = client.stack_trace().await?;
-        merge(&mut all, response);
+        all.merge_other(response);
     }
 
     if all.actor_traces.is_empty()
@@ -61,32 +55,8 @@ pub async fn dump(context: &CtlContext) -> anyhow::Result<()> {
         && all.inflight_barrier_traces.is_empty()
     {
         println!("No traces found. No actors are running, or `--async-stack-trace` not set?");
-    } else {
-        if !all.actor_traces.is_empty() {
-            println!("--- Actor Traces ---");
-            for (key, trace) in all.actor_traces {
-                println!(">> Actor {key}\n{trace}");
-            }
-        }
-        if !all.rpc_traces.is_empty() {
-            println!("\n\n--- RPC Traces ---");
-            for (key, trace) in all.rpc_traces {
-                println!(">> RPC {key}\n{trace}");
-            }
-        }
-        if !all.compaction_task_traces.is_empty() {
-            println!("\n\n--- Compactor Traces ---");
-            for (key, trace) in all.compaction_task_traces {
-                println!(">> Compaction Task {key}\n{trace}");
-            }
-        }
-        if !all.inflight_barrier_traces.is_empty() {
-            println!("\n\n--- Inflight Barrier Traces ---");
-            for (name, trace) in &all.inflight_barrier_traces {
-                println!(">> Barrier {name}\n{trace}");
-            }
-        }
     }
+    println!("{}", all.output());
 
     Ok(())
 }
diff --git a/src/ctl/src/cmd_impl/compute.rs b/src/ctl/src/cmd_impl/compute.rs
index 0150c00058773..ff519efcfffa7 100644
--- a/src/ctl/src/cmd_impl/compute.rs
+++ b/src/ctl/src/cmd_impl/compute.rs
@@ -11,12 +11,10 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-use std::convert::TryFrom;
 
 use risingwave_common::config::{BatchConfig, StreamingConfig};
 use risingwave_common::util::addr::HostAddr;
 use risingwave_rpc_client::ComputeClient;
-use serde_json;
 
 pub async fn show_config(host: &str) -> anyhow::Result<()> {
     let listen_addr = HostAddr::try_from(host)?;
diff --git a/src/ctl/src/cmd_impl/debug/fix_table_fragments.rs b/src/ctl/src/cmd_impl/debug/fix_table_fragments.rs
index 627545ac178e2..e164fde3e7845 100644
--- a/src/ctl/src/cmd_impl/debug/fix_table_fragments.rs
+++ b/src/ctl/src/cmd_impl/debug/fix_table_fragments.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use etcd_client::ConnectOptions;
-use risingwave_common::util::stream_graph_visitor::visit_stream_node_cont;
+use risingwave_common::util::stream_graph_visitor::visit_stream_node_cont_mut;
 use risingwave_meta::model::{MetadataModel, TableFragments};
 use risingwave_meta::storage::{EtcdMetaStore, WrappedEtcdClient};
 use risingwave_pb::stream_plan::stream_node::NodeBody;
@@ -54,7 +54,7 @@ pub async fn fix_table_fragments(
             .upstream_fragment_ids
             .retain(|id| !dirty_fragment_ids.contains(id));
         for actor in &mut fragment.actors {
-            visit_stream_node_cont(actor.nodes.as_mut().unwrap(), |node| {
+            visit_stream_node_cont_mut(actor.nodes.as_mut().unwrap(), |node| {
                 if let Some(NodeBody::Union(_)) = node.node_body {
                     node.input.retain_mut(|input| {
                         if let Some(NodeBody::Merge(merge_node)) = &mut input.node_body
diff --git a/src/ctl/src/cmd_impl/hummock/compaction_group.rs b/src/ctl/src/cmd_impl/hummock/compaction_group.rs
index 068472dd1ce4a..d58aeb7bffe79 100644
--- a/src/ctl/src/cmd_impl/hummock/compaction_group.rs
+++ b/src/ctl/src/cmd_impl/hummock/compaction_group.rs
@@ -20,6 +20,7 @@ use risingwave_hummock_sdk::compaction_group::StateTableId;
 use risingwave_hummock_sdk::{CompactionGroupId, HummockContextId};
 use risingwave_pb::hummock::compact_task::TaskStatus;
 use risingwave_pb::hummock::rise_ctl_update_compaction_config_request::mutable_config::MutableConfig;
+use risingwave_pb::hummock::rise_ctl_update_compaction_config_request::CompressionAlgorithm;
 
 use crate::CtlContext;
 
@@ -63,6 +64,8 @@ pub fn build_compaction_config_vec(
     level0_overlapping_sub_level_compact_level_count: Option<u32>,
     enable_emergency_picker: Option<bool>,
     tombstone_reclaim_ratio: Option<u32>,
+    compress_algorithm: Option<CompressionAlgorithm>,
+    max_l0_compact_level: Option<u32>,
 ) -> Vec<MutableConfig> {
     let mut configs = vec![];
     if let Some(c) = max_bytes_for_level_base {
@@ -110,6 +113,12 @@ pub fn build_compaction_config_vec(
     if let Some(c) = tombstone_reclaim_ratio {
         configs.push(MutableConfig::TombstoneReclaimRatio(c))
     }
+    if let Some(c) = compress_algorithm {
+        configs.push(MutableConfig::CompressionAlgorithm(c))
+    }
+    if let Some(c) = max_l0_compact_level {
+        configs.push(MutableConfig::MaxL0CompactLevelCount(c))
+    }
 
     configs
 }
diff --git a/src/ctl/src/cmd_impl/hummock/pause_resume.rs b/src/ctl/src/cmd_impl/hummock/pause_resume.rs
index e8574ce60af7e..85791267bfd8a 100644
--- a/src/ctl/src/cmd_impl/hummock/pause_resume.rs
+++ b/src/ctl/src/cmd_impl/hummock/pause_resume.rs
@@ -61,7 +61,7 @@ pub async fn replay_version(context: &CtlContext) -> anyhow::Result<()> {
     println!("replay starts");
     println!("base version {}", base_version.id);
     let delta_fetch_size = 100;
-    let mut current_delta_id = base_version.id + 1;
+    let mut current_delta_id = base_version.next_version_id();
     loop {
         let deltas = meta_client
             .list_version_deltas(current_delta_id, delta_fetch_size, HummockEpoch::MAX)
@@ -78,7 +78,7 @@ pub async fn replay_version(context: &CtlContext) -> anyhow::Result<()> {
             base_version.apply_version_delta(&delta);
             println!("replayed version {}", base_version.id);
         }
-        current_delta_id = base_version.id + 1;
+        current_delta_id = base_version.next_version_id();
     }
     println!("replay ends");
     Ok(())
diff --git a/src/ctl/src/cmd_impl/meta/migration.rs b/src/ctl/src/cmd_impl/meta/migration.rs
index 8a77ca5650f9d..b9aa81b677519 100644
--- a/src/ctl/src/cmd_impl/meta/migration.rs
+++ b/src/ctl/src/cmd_impl/meta/migration.rs
@@ -45,20 +45,20 @@ use risingwave_meta_model_v2::hummock_version_stats::TableStats;
 use risingwave_meta_model_v2::object::ObjectType;
 use risingwave_meta_model_v2::prelude::{
     Actor, ActorDispatcher, CatalogVersion, Cluster, Connection, Database, Fragment, Function,
-    Index, Object, ObjectDependency, Schema, Sink, Source, StreamingJob, Subscription,
+    Index, Object, ObjectDependency, Schema, Secret, Sink, Source, StreamingJob, Subscription,
     SystemParameter, Table, User, UserPrivilege, View, Worker, WorkerProperty,
 };
 use risingwave_meta_model_v2::{
     catalog_version, cluster, compaction_config, compaction_status, compaction_task, connection,
     database, function, hummock_pinned_snapshot, hummock_pinned_version, hummock_sequence,
-    hummock_version_delta, hummock_version_stats, index, object, object_dependency, schema, sink,
-    source, streaming_job, subscription, table, user, user_privilege, view, worker,
+    hummock_version_delta, hummock_version_stats, index, object, object_dependency, schema, secret,
+    sink, source, streaming_job, subscription, table, user, user_privilege, view, worker,
     worker_property, CreateType, JobStatus, ObjectId, StreamingParallelism,
 };
 use risingwave_pb::catalog::table::PbTableType;
 use risingwave_pb::catalog::{
-    PbConnection, PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbSubscription,
-    PbTable, PbView,
+    PbConnection, PbDatabase, PbFunction, PbIndex, PbSchema, PbSecret, PbSink, PbSource,
+    PbSubscription, PbTable, PbView,
 };
 use risingwave_pb::common::WorkerType;
 use risingwave_pb::hummock::{
@@ -188,6 +188,7 @@ pub async fn migrate(from: EtcdBackend, target: String, force_clean: bool) -> an
     let functions = PbFunction::list(&meta_store).await?;
     let connections = PbConnection::list(&meta_store).await?;
     let subscriptions = PbSubscription::list(&meta_store).await?;
+    let secrets = PbSecret::list(&meta_store).await?;
 
     // inuse object ids.
     let mut inuse_obj_ids = tables
@@ -319,6 +320,28 @@ pub async fn migrate(from: EtcdBackend, target: String, force_clean: bool) -> an
     }
     println!("connections migrated");
 
+    // secret mapping
+    let mut secret_rewrite = HashMap::new();
+    for mut secret in secrets {
+        let id = next_available_id();
+        secret_rewrite.insert(secret.id, id);
+        secret.id = id as _;
+
+        let obj = object::ActiveModel {
+            oid: Set(id as _),
+            obj_type: Set(ObjectType::Secret),
+            owner_id: Set(secret.owner as _),
+            database_id: Set(Some(*db_rewrite.get(&secret.database_id).unwrap() as _)),
+            schema_id: Set(Some(*schema_rewrite.get(&secret.schema_id).unwrap() as _)),
+            ..Default::default()
+        };
+        Object::insert(obj).exec(&meta_store_sql.conn).await?;
+        Secret::insert(secret::ActiveModel::from(secret))
+            .exec(&meta_store_sql.conn)
+            .await?;
+    }
+    println!("secrets migrated");
+
     // add object: table, source, sink, index, view, subscription.
     macro_rules! insert_objects {
         ($objects:expr, $object_type:expr) => {
@@ -525,6 +548,16 @@ pub async fn migrate(from: EtcdBackend, target: String, force_clean: bool) -> an
                 if let Some(id) = s.connection_id.as_mut() {
                     *id = *connection_rewrite.get(id).unwrap();
                 }
+                for secret_id in s.secret_ref.values_mut() {
+                    *secret_id = *secret_rewrite.get(secret_id).unwrap();
+                }
+                object_dependencies.extend(s.secret_ref.values().map(|id| {
+                    object_dependency::ActiveModel {
+                        id: NotSet,
+                        oid: Set(*id as _),
+                        used_by: Set(s.id as _),
+                    }
+                }));
                 s.into()
             })
             .collect();
@@ -560,13 +593,11 @@ pub async fn migrate(from: EtcdBackend, target: String, force_clean: bool) -> an
         let subscription_models: Vec<subscription::ActiveModel> = subscriptions
             .into_iter()
             .map(|s| {
-                object_dependencies.extend(s.dependent_relations.iter().map(|id| {
-                    object_dependency::ActiveModel {
-                        id: NotSet,
-                        oid: Set(*id as _),
-                        used_by: Set(s.id as _),
-                    }
-                }));
+                object_dependencies.push(object_dependency::ActiveModel {
+                    id: NotSet,
+                    oid: Set(s.dependent_table_id as _),
+                    used_by: Set(s.id as _),
+                });
                 s.into()
             })
             .collect();
diff --git a/src/ctl/src/cmd_impl/meta/reschedule.rs b/src/ctl/src/cmd_impl/meta/reschedule.rs
index 64d3e229c9112..8d0a45be842fb 100644
--- a/src/ctl/src/cmd_impl/meta/reschedule.rs
+++ b/src/ctl/src/cmd_impl/meta/reschedule.rs
@@ -24,7 +24,6 @@ use risingwave_pb::meta::get_reschedule_plan_request::PbPolicy;
 use risingwave_pb::meta::table_fragments::ActorStatus;
 use risingwave_pb::meta::{GetClusterInfoResponse, GetReschedulePlanResponse, Reschedule};
 use serde::{Deserialize, Serialize};
-use serde_yaml;
 use thiserror_ext::AsReport;
 
 use crate::CtlContext;
diff --git a/src/ctl/src/cmd_impl/meta/serving.rs b/src/ctl/src/cmd_impl/meta/serving.rs
index c6c5d3cf81985..cd8b1be50e206 100644
--- a/src/ctl/src/cmd_impl/meta/serving.rs
+++ b/src/ctl/src/cmd_impl/meta/serving.rs
@@ -16,30 +16,26 @@ use std::collections::HashMap;
 
 use comfy_table::{Row, Table};
 use itertools::Itertools;
-use risingwave_common::hash::{ParallelUnitId, VirtualNode};
-use risingwave_pb::common::{WorkerNode, WorkerType};
+use risingwave_common::hash::VirtualNode;
+use risingwave_pb::common::WorkerType;
 
 use crate::CtlContext;
 
 pub async fn list_serving_fragment_mappings(context: &CtlContext) -> anyhow::Result<()> {
     let meta_client = context.meta_client().await?;
     let mappings = meta_client.list_serving_vnode_mappings().await?;
-    let workers = meta_client
+    let workers: HashMap<_, _> = meta_client
         .list_worker_nodes(Some(WorkerType::ComputeNode))
-        .await?;
-    let mut pu_to_worker: HashMap<ParallelUnitId, &WorkerNode> = HashMap::new();
-    for w in &workers {
-        for pu in &w.parallel_units {
-            pu_to_worker.insert(pu.id, w);
-        }
-    }
+        .await?
+        .into_iter()
+        .map(|worker| (worker.id, worker))
+        .collect();
 
     let mut table = Table::new();
     table.set_header({
         let mut row = Row::new();
         row.add_cell("Table Id".into());
         row.add_cell("Fragment Id".into());
-        row.add_cell("Parallel Unit Id".into());
         row.add_cell("Virtual Node".into());
         row.add_cell("Worker".into());
         row
@@ -48,28 +44,25 @@ pub async fn list_serving_fragment_mappings(context: &CtlContext) -> anyhow::Res
     let rows = mappings
         .iter()
         .flat_map(|(fragment_id, (table_id, mapping))| {
-            let mut pu_vnodes: HashMap<ParallelUnitId, Vec<VirtualNode>> = HashMap::new();
-            for (vnode, pu) in mapping.iter_with_vnode() {
-                pu_vnodes.entry(pu).or_default().push(vnode);
+            let mut worker_nodes: HashMap<u32, Vec<VirtualNode>> = HashMap::new();
+            for (vnode, worker_slot_id) in mapping.iter_with_vnode() {
+                worker_nodes
+                    .entry(worker_slot_id.worker_id())
+                    .or_default()
+                    .push(vnode);
             }
-            pu_vnodes.into_iter().map(|(pu_id, vnodes)| {
-                (
-                    *table_id,
-                    *fragment_id,
-                    pu_id,
-                    vnodes,
-                    pu_to_worker.get(&pu_id),
-                )
+            worker_nodes.into_iter().map(|(worker_id, vnodes)| {
+                (*table_id, *fragment_id, vnodes, workers.get(&worker_id))
             })
         })
         .collect_vec();
-    for (table_id, fragment_id, pu_id, vnodes, worker) in
-        rows.into_iter().sorted_by_key(|(t, f, p, ..)| (*t, *f, *p))
+
+    for (table_id, fragment_id, vnodes, worker) in
+        rows.into_iter().sorted_by_key(|(t, f, ..)| (*t, *f))
     {
         let mut row = Row::new();
         row.add_cell(table_id.into());
         row.add_cell(fragment_id.into());
-        row.add_cell(pu_id.into());
         row.add_cell(
             format!(
                 "{} in total: {}",
diff --git a/src/ctl/src/cmd_impl/scale/resize.rs b/src/ctl/src/cmd_impl/scale/resize.rs
index 6ef0b3c15d7e1..6b990c7865519 100644
--- a/src/ctl/src/cmd_impl/scale/resize.rs
+++ b/src/ctl/src/cmd_impl/scale/resize.rs
@@ -24,7 +24,6 @@ use risingwave_pb::meta::get_reschedule_plan_request::{
 use risingwave_pb::meta::update_worker_node_schedulability_request::Schedulability;
 use risingwave_pb::meta::{GetClusterInfoResponse, GetReschedulePlanResponse};
 use risingwave_stream::task::FragmentId;
-use serde_yaml;
 use thiserror_ext::AsReport;
 
 use crate::cmd_impl::meta::ReschedulePayload;
diff --git a/src/ctl/src/common/hummock_service.rs b/src/ctl/src/common/hummock_service.rs
index e10d3669af601..59d272c3a27ff 100644
--- a/src/ctl/src/common/hummock_service.rs
+++ b/src/ctl/src/common/hummock_service.rs
@@ -17,11 +17,12 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::{anyhow, bail, Result};
-use risingwave_common::config::{EvictionConfig, MetricLevel, ObjectStoreConfig};
+use foyer::HybridCacheBuilder;
+use risingwave_common::config::{MetricLevel, ObjectStoreConfig};
 use risingwave_object_store::object::build_remote_object_store;
 use risingwave_rpc_client::MetaClient;
 use risingwave_storage::hummock::hummock_meta_client::MonitoredHummockMetaClient;
-use risingwave_storage::hummock::{FileCache, HummockStorage, SstableStore, SstableStoreConfig};
+use risingwave_storage::hummock::{HummockStorage, SstableStore, SstableStoreConfig};
 use risingwave_storage::monitor::{
     global_hummock_state_store_metrics, CompactorMetrics, HummockMetrics, HummockStateStoreMetrics,
     MonitoredStateStore, MonitoredStorageMetrics, ObjectStoreMetrics,
@@ -161,29 +162,36 @@ impl HummockServiceOpts {
             self.hummock_url.strip_prefix("hummock+").unwrap(),
             Arc::new(ObjectStoreMetrics::unused()),
             "Hummock",
-            ObjectStoreConfig::default(),
+            Arc::new(ObjectStoreConfig::default()),
         )
         .await;
 
         let opts = self.get_storage_opts();
 
+        let meta_cache = HybridCacheBuilder::new()
+            .memory(opts.meta_cache_capacity_mb * (1 << 20))
+            .with_shards(opts.meta_cache_shard_num)
+            .storage()
+            .build()
+            .await?;
+        let block_cache = HybridCacheBuilder::new()
+            .memory(opts.block_cache_capacity_mb * (1 << 20))
+            .with_shards(opts.block_cache_shard_num)
+            .storage()
+            .build()
+            .await?;
+
         Ok(Arc::new(SstableStore::new(SstableStoreConfig {
             store: Arc::new(object_store),
             path: opts.data_directory,
-            block_cache_capacity: opts.block_cache_capacity_mb * (1 << 20),
-            meta_cache_capacity: opts.meta_cache_capacity_mb * (1 << 20),
-            block_cache_shard_num: opts.block_cache_shard_num,
-            meta_cache_shard_num: opts.meta_cache_shard_num,
-            block_cache_eviction: EvictionConfig::for_test(),
-            meta_cache_eviction: EvictionConfig::for_test(),
             prefetch_buffer_capacity: opts.block_cache_capacity_mb * (1 << 20),
             max_prefetch_block_number: opts.max_prefetch_block_number,
-            data_file_cache: FileCache::none(),
-            meta_file_cache: FileCache::none(),
             recent_filter: None,
             state_store_metrics: Arc::new(global_hummock_state_store_metrics(
                 MetricLevel::Disabled,
             )),
+            meta_cache,
+            block_cache,
         })))
     }
 }
diff --git a/src/ctl/src/lib.rs b/src/ctl/src/lib.rs
index a1aaa8f48c5f5..1f50250276d6e 100644
--- a/src/ctl/src/lib.rs
+++ b/src/ctl/src/lib.rs
@@ -22,6 +22,7 @@ use cmd_impl::hummock::SstDumpArgs;
 use itertools::Itertools;
 use risingwave_hummock_sdk::HummockEpoch;
 use risingwave_meta::backup_restore::RestoreOpts;
+use risingwave_pb::hummock::rise_ctl_update_compaction_config_request::CompressionAlgorithm;
 use risingwave_pb::meta::update_worker_node_schedulability_request::Schedulability;
 use thiserror_ext::AsReport;
 
@@ -254,6 +255,12 @@ enum HummockCommands {
         enable_emergency_picker: Option<bool>,
         #[clap(long)]
         tombstone_reclaim_ratio: Option<u32>,
+        #[clap(long)]
+        compression_level: Option<u32>,
+        #[clap(long)]
+        compression_algorithm: Option<String>,
+        #[clap(long)]
+        max_l0_compact_level: Option<u32>,
     },
     /// Split given compaction group into two. Moves the given tables to the new group.
     SplitCompactionGroup {
@@ -683,6 +690,9 @@ async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> {
             level0_overlapping_sub_level_compact_level_count,
             enable_emergency_picker,
             tombstone_reclaim_ratio,
+            compression_level,
+            compression_algorithm,
+            max_l0_compact_level,
         }) => {
             cmd_impl::hummock::update_compaction_config(
                 context,
@@ -703,6 +713,16 @@ async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> {
                     level0_overlapping_sub_level_compact_level_count,
                     enable_emergency_picker,
                     tombstone_reclaim_ratio,
+                    if let Some(level) = compression_level {
+                        assert!(compression_algorithm.is_some());
+                        Some(CompressionAlgorithm {
+                            level,
+                            compression_algorithm: compression_algorithm.unwrap(),
+                        })
+                    } else {
+                        None
+                    },
+                    max_l0_compact_level,
                 ),
             )
             .await?
diff --git a/src/dml/src/dml_manager.rs b/src/dml/src/dml_manager.rs
index f6611b6c94c21..d3502f1f9d2de 100644
--- a/src/dml/src/dml_manager.rs
+++ b/src/dml/src/dml_manager.rs
@@ -170,7 +170,6 @@ mod tests {
     use risingwave_common::array::StreamChunk;
     use risingwave_common::catalog::INITIAL_TABLE_VERSION_ID;
     use risingwave_common::test_prelude::StreamChunkTestExt;
-    use risingwave_common::transaction::transaction_id::TxnId;
     use risingwave_common::types::DataType;
 
     use super::*;
diff --git a/src/dml/src/table.rs b/src/dml/src/table.rs
index 4981ebce2e8a3..a5d0ecf8c23a4 100644
--- a/src/dml/src/table.rs
+++ b/src/dml/src/table.rs
@@ -283,14 +283,11 @@ impl TableStreamReader {
 
 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
-
     use assert_matches::assert_matches;
     use futures::StreamExt;
     use itertools::Itertools;
-    use risingwave_common::array::{Array, I64Array, Op, StreamChunk};
+    use risingwave_common::array::{Array, I64Array, Op};
     use risingwave_common::catalog::ColumnId;
-    use risingwave_common::transaction::transaction_id::TxnId;
     use risingwave_common::types::DataType;
 
     use super::*;
diff --git a/src/error/src/macros.rs b/src/error/src/macros.rs
index b3079aa281ce9..16a11d2dba4d6 100644
--- a/src/error/src/macros.rs
+++ b/src/error/src/macros.rs
@@ -149,9 +149,6 @@ pub use must_match;
 
 #[cfg(test)]
 mod ensure_tests {
-    use std::convert::Into;
-    use std::result::Result::Err;
-
     use anyhow::anyhow;
     use thiserror::Error;
 
diff --git a/src/expr/core/Cargo.toml b/src/expr/core/Cargo.toml
index 3f5ca590026db..cbff3a5ff2e28 100644
--- a/src/expr/core/Cargo.toml
+++ b/src/expr/core/Cargo.toml
@@ -15,18 +15,10 @@ ignored = ["workspace-hack", "ctor"]
 [package.metadata.cargo-udeps.ignore]
 normal = ["workspace-hack", "ctor"]
 
-[features]
-embedded-deno-udf = ["arrow-udf-js-deno"]
-embedded-python-udf = ["arrow-udf-python"]
-
 [dependencies]
 anyhow = "1"
 arrow-array = { workspace = true }
 arrow-schema = { workspace = true }
-arrow-udf-js = { workspace = true }
-arrow-udf-js-deno = { workspace = true, optional = true }
-arrow-udf-python = { workspace = true, optional = true }
-arrow-udf-wasm = { workspace = true }
 async-trait = "0.1"
 auto_impl = "1"
 await-tree = { workspace = true }
@@ -38,7 +30,7 @@ chrono = { version = "0.4", default-features = false, features = [
 const-currying = "0.0.4"
 downcast-rs = "1.2"
 easy-ext = "1"
-educe = "0.5"
+educe = "0.6"
 either = "1"
 enum-as-inner = "0.6"
 futures = "0.3"
@@ -46,17 +38,14 @@ futures-async-stream = { workspace = true }
 futures-util = "0.3"
 itertools = { workspace = true }
 linkme = { version = "0.3", features = ["used_linker"] }
-md5 = "0.7"
-moka = { version = "0.12", features = ["sync"] }
 num-traits = "0.2"
-openssl = { version = "0.10", features = ["vendored"] }
 parse-display = "0.9"
 paste = "1"
+prometheus = "0.13"
 risingwave_common = { workspace = true }
 risingwave_common_estimate_size = { workspace = true }
 risingwave_expr_macro = { path = "../macro" }
 risingwave_pb = { workspace = true }
-risingwave_udf = { workspace = true }
 smallvec = "1"
 static_assertions = "1"
 thiserror = "1"
@@ -66,7 +55,6 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [
     "macros",
 ] }
 tracing = "0.1"
-zstd = { version = "0.13", default-features = false }
 
 [target.'cfg(not(madsim))'.dependencies]
 workspace-hack = { path = "../../workspace-hack" }
diff --git a/src/expr/core/src/aggregate/def.rs b/src/expr/core/src/aggregate/def.rs
index 80293d98eaf11..5ca07d0153196 100644
--- a/src/expr/core/src/aggregate/def.rs
+++ b/src/expr/core/src/aggregate/def.rs
@@ -24,7 +24,7 @@ use risingwave_common::types::{DataType, Datum};
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use risingwave_common::util::value_encoding::DatumFromProtoExt;
 use risingwave_pb::expr::agg_call::PbType;
-use risingwave_pb::expr::{PbAggCall, PbInputRef};
+use risingwave_pb::expr::{PbAggCall, PbInputRef, PbUserDefinedFunctionMetadata};
 
 use crate::expr::{
     build_from_prost, BoxedExpression, ExpectExt, Expression, LiteralExpression, Token,
@@ -32,6 +32,10 @@ use crate::expr::{
 use crate::Result;
 
 /// Represents an aggregation function.
+// TODO(runji):
+//  remove this struct from the expression module.
+//  this module only cares about aggregate functions themselves.
+//  advanced features like order by, filter, distinct, etc. should be handled by the upper layer.
 #[derive(Debug, Clone)]
 pub struct AggCall {
     /// Aggregation kind for constructing agg state.
@@ -52,6 +56,9 @@ pub struct AggCall {
 
     /// Constant arguments.
     pub direct_args: Vec<LiteralExpression>,
+
+    /// Additional metadata for user defined functions.
+    pub user_defined: Option<PbUserDefinedFunctionMetadata>,
 }
 
 impl AggCall {
@@ -90,6 +97,7 @@ impl AggCall {
             filter,
             distinct: agg_call.distinct,
             direct_args,
+            user_defined: agg_call.udf.clone(),
         })
     }
 
@@ -148,17 +156,16 @@ impl<Iter: Iterator<Item = Token>> Parser<Iter> {
 
         AggCall {
             kind: AggKind::from_protobuf(func).unwrap(),
-            args: match children.as_slice() {
-                [] => AggArgs::None,
-                [(i, t)] => AggArgs::Unary(t.clone(), *i),
-                [(i0, t0), (i1, t1)] => AggArgs::Binary([t0.clone(), t1.clone()], [*i0, *i1]),
-                _ => panic!("too many arguments for agg call"),
+            args: AggArgs {
+                data_types: children.iter().map(|(_, ty)| ty.clone()).collect(),
+                val_indices: children.iter().map(|(idx, _)| *idx).collect(),
             },
             return_type: ty,
             column_orders,
             filter: None,
             distinct,
             direct_args: Vec::new(),
+            user_defined: None,
         }
     }
 
@@ -236,6 +243,7 @@ pub enum AggKind {
 
     /// Return last seen one of the input values.
     InternalLastSeenValue,
+    UserDefined,
 }
 
 impl AggKind {
@@ -268,6 +276,7 @@ impl AggKind {
             PbType::Mode => Ok(AggKind::Mode),
             PbType::Grouping => Ok(AggKind::Grouping),
             PbType::InternalLastSeenValue => Ok(AggKind::InternalLastSeenValue),
+            PbType::UserDefined => Ok(AggKind::UserDefined),
             PbType::Unspecified => bail!("Unrecognized agg."),
         }
     }
@@ -301,6 +310,7 @@ impl AggKind {
             Self::Mode => PbType::Mode,
             Self::Grouping => PbType::Grouping,
             Self::InternalLastSeenValue => PbType::InternalLastSeenValue,
+            Self::UserDefined => PbType::UserDefined,
         }
     }
 }
@@ -409,6 +419,7 @@ pub mod agg_kinds {
                 | AggKind::BoolOr
                 | AggKind::BitAnd
                 | AggKind::BitOr
+                | AggKind::UserDefined
         };
     }
     pub use simply_cannot_two_phase;
@@ -428,6 +439,7 @@ pub mod agg_kinds {
                 | AggKind::BoolOr
                 | AggKind::ApproxCountDistinct
                 | AggKind::InternalLastSeenValue
+                | AggKind::UserDefined
         };
     }
     pub use single_value_state;
@@ -469,52 +481,40 @@ impl AggKind {
 }
 
 /// An aggregation function may accept 0, 1 or 2 arguments.
-#[derive(Clone, Debug)]
-pub enum AggArgs {
-    /// `None` is used for function calls that accept 0 argument, e.g. `count(*)`.
-    None,
-    /// `Unary` is used for function calls that accept 1 argument, e.g. `sum(x)`.
-    Unary(DataType, usize),
-    /// `Binary` is used for function calls that accept 2 arguments, e.g. `string_agg(x, delim)`.
-    Binary([DataType; 2], [usize; 2]),
+#[derive(Clone, Debug, Default)]
+pub struct AggArgs {
+    data_types: Box<[DataType]>,
+    val_indices: Box<[usize]>,
 }
 
 impl AggArgs {
     pub fn from_protobuf(args: &[PbInputRef]) -> Result<Self> {
-        let args = match args {
-            [] => Self::None,
-            [arg] => Self::Unary(DataType::from(arg.get_type()?), arg.get_index() as usize),
-            [arg1, arg2] => Self::Binary(
-                [
-                    DataType::from(arg1.get_type()?),
-                    DataType::from(arg2.get_type()?),
-                ],
-                [arg1.get_index() as usize, arg2.get_index() as usize],
-            ),
-            _ => bail!("too many arguments for agg call"),
-        };
-        Ok(args)
+        Ok(AggArgs {
+            data_types: args
+                .iter()
+                .map(|arg| DataType::from(arg.get_type().unwrap()))
+                .collect(),
+            val_indices: args.iter().map(|arg| arg.get_index() as usize).collect(),
+        })
     }
-}
 
-impl AggArgs {
     /// return the types of arguments.
     pub fn arg_types(&self) -> &[DataType] {
-        use AggArgs::*;
-        match self {
-            None => &[],
-            Unary(typ, _) => std::slice::from_ref(typ),
-            Binary(typs, _) => typs,
-        }
+        &self.data_types
     }
 
     /// return the indices of the arguments in [`risingwave_common::array::StreamChunk`].
     pub fn val_indices(&self) -> &[usize] {
-        use AggArgs::*;
-        match self {
-            None => &[],
-            Unary(_, val_idx) => std::slice::from_ref(val_idx),
-            Binary(_, val_indices) => val_indices,
+        &self.val_indices
+    }
+}
+
+impl FromIterator<(DataType, usize)> for AggArgs {
+    fn from_iter<T: IntoIterator<Item = (DataType, usize)>>(iter: T) -> Self {
+        let (data_types, val_indices): (Vec<_>, Vec<_>) = iter.into_iter().unzip();
+        AggArgs {
+            data_types: data_types.into(),
+            val_indices: val_indices.into(),
         }
     }
 }
diff --git a/src/expr/core/src/aggregate/mod.rs b/src/expr/core/src/aggregate/mod.rs
index 4e8666e8c9dee..2a1119d6fe301 100644
--- a/src/expr/core/src/aggregate/mod.rs
+++ b/src/expr/core/src/aggregate/mod.rs
@@ -15,6 +15,7 @@
 use std::fmt::Debug;
 use std::ops::Range;
 
+use anyhow::anyhow;
 use downcast_rs::{impl_downcast, Downcast};
 use itertools::Itertools;
 use risingwave_common::array::StreamChunk;
@@ -26,6 +27,8 @@ use crate::{ExprError, Result};
 
 // aggregate definition
 mod def;
+// user defined aggregate function
+mod user_defined;
 
 pub use self::def::*;
 
@@ -36,8 +39,8 @@ pub trait AggregateFunction: Send + Sync + 'static {
     fn return_type(&self) -> DataType;
 
     /// Creates an initial state of the aggregate function.
-    fn create_state(&self) -> AggregateState {
-        AggregateState::Datum(None)
+    fn create_state(&self) -> Result<AggregateState> {
+        Ok(AggregateState::Datum(None))
     }
 
     /// Update the state with multiple rows.
@@ -58,7 +61,7 @@ pub trait AggregateFunction: Send + Sync + 'static {
     fn encode_state(&self, state: &AggregateState) -> Result<Datum> {
         match state {
             AggregateState::Datum(d) => Ok(d.clone()),
-            _ => panic!("cannot encode state"),
+            AggregateState::Any(_) => Err(ExprError::Internal(anyhow!("cannot encode state"))),
         }
     }
 
@@ -140,6 +143,10 @@ pub fn build_retractable(agg: &AggCall) -> Result<BoxedAggregateFunction> {
 /// NOTE: This function ignores argument indices, `column_orders`, `filter` and `distinct` in
 /// `AggCall`. Such operations should be done in batch or streaming executors.
 pub fn build(agg: &AggCall, prefer_append_only: bool) -> Result<BoxedAggregateFunction> {
+    if agg.kind == AggKind::UserDefined {
+        return user_defined::new_user_defined(agg);
+    }
+
     let sig = crate::sig::FUNCTION_REGISTRY
         .get(agg.kind, agg.args.arg_types(), &agg.return_type)
         .ok_or_else(|| {
diff --git a/src/expr/core/src/aggregate/user_defined.rs b/src/expr/core/src/aggregate/user_defined.rs
new file mode 100644
index 0000000000000..067063c22f4a2
--- /dev/null
+++ b/src/expr/core/src/aggregate/user_defined.rs
@@ -0,0 +1,163 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use anyhow::Context;
+use arrow_array::ArrayRef;
+use arrow_schema::{Field, Fields, Schema, SchemaRef};
+use risingwave_common::array::arrow::{FromArrow, ToArrow, UdfArrowConvert};
+use risingwave_common::array::Op;
+use risingwave_common::buffer::Bitmap;
+
+use super::*;
+use crate::sig::{UdfImpl, UdfKind, UdfOptions};
+
+#[derive(Debug)]
+pub struct UserDefinedAggregateFunction {
+    arg_schema: SchemaRef,
+    return_type: DataType,
+    return_field: Field,
+    state_field: Field,
+    runtime: Box<dyn UdfImpl>,
+}
+
+#[async_trait::async_trait]
+impl AggregateFunction for UserDefinedAggregateFunction {
+    fn return_type(&self) -> DataType {
+        self.return_type.clone()
+    }
+
+    /// Creates an initial state of the aggregate function.
+    fn create_state(&self) -> Result<AggregateState> {
+        let state = self.runtime.call_agg_create_state()?;
+        Ok(AggregateState::Any(Box::new(State(state))))
+    }
+
+    /// Update the state with multiple rows.
+    async fn update(&self, state: &mut AggregateState, input: &StreamChunk) -> Result<()> {
+        let state = &mut state.downcast_mut::<State>().0;
+        let ops = input
+            .visibility()
+            .iter_ones()
+            .map(|i| Some(matches!(input.ops()[i], Op::Delete | Op::UpdateDelete)))
+            .collect();
+        // this will drop invisible rows
+        let arrow_input = UdfArrowConvert::default()
+            .to_record_batch(self.arg_schema.clone(), input.data_chunk())?;
+        let new_state = self
+            .runtime
+            .call_agg_accumulate_or_retract(state, &ops, &arrow_input)?;
+        *state = new_state;
+        Ok(())
+    }
+
+    /// Update the state with a range of rows.
+    async fn update_range(
+        &self,
+        state: &mut AggregateState,
+        input: &StreamChunk,
+        range: Range<usize>,
+    ) -> Result<()> {
+        // XXX(runji): this may be inefficient
+        let vis = input.visibility() & Bitmap::from_range(input.capacity(), range);
+        let input = input.clone_with_vis(vis);
+        self.update(state, &input).await
+    }
+
+    /// Get aggregate result from the state.
+    async fn get_result(&self, state: &AggregateState) -> Result<Datum> {
+        let state = &state.downcast_ref::<State>().0;
+        let arrow_output = self.runtime.call_agg_finish(state)?;
+        let output = UdfArrowConvert::default().from_array(&self.return_field, &arrow_output)?;
+        Ok(output.datum_at(0))
+    }
+
+    /// Encode the state into a datum that can be stored in state table.
+    fn encode_state(&self, state: &AggregateState) -> Result<Datum> {
+        let state = &state.downcast_ref::<State>().0;
+        let state = UdfArrowConvert::default().from_array(&self.state_field, state)?;
+        Ok(state.datum_at(0))
+    }
+
+    /// Decode the state from a datum in state table.
+    fn decode_state(&self, datum: Datum) -> Result<AggregateState> {
+        let array = {
+            let mut builder = DataType::Bytea.create_array_builder(1);
+            builder.append(datum);
+            builder.finish()
+        };
+        let state = UdfArrowConvert::default().to_array(self.state_field.data_type(), &array)?;
+        Ok(AggregateState::Any(Box::new(State(state))))
+    }
+}
+
+// In arrow-udf, aggregate state is represented as an `ArrayRef`.
+// To avoid unnecessary conversion between `ArrayRef` and `Datum`,
+// we store `ArrayRef` directly in our `AggregateState`.
+#[derive(Debug)]
+struct State(ArrayRef);
+
+impl EstimateSize for State {
+    fn estimated_heap_size(&self) -> usize {
+        self.0.get_array_memory_size()
+    }
+}
+
+impl AggStateDyn for State {}
+
+/// Create a new user-defined aggregate function.
+pub fn new_user_defined(agg: &AggCall) -> Result<BoxedAggregateFunction> {
+    let udf = agg
+        .user_defined
+        .as_ref()
+        .context("missing UDF definition")?;
+
+    let identifier = udf.get_identifier()?;
+    let language = udf.language.as_str();
+    let runtime = udf.runtime.as_deref();
+    let link = udf.link.as_deref();
+
+    let build_fn = crate::sig::find_udf_impl(language, runtime, link)?.build_fn;
+    let runtime = build_fn(UdfOptions {
+        kind: UdfKind::Aggregate,
+        body: udf.body.as_deref(),
+        compressed_binary: udf.compressed_binary.as_deref(),
+        link: udf.link.as_deref(),
+        identifier,
+        arg_names: &udf.arg_names,
+        return_type: &agg.return_type,
+        always_retry_on_network_error: false,
+        function_type: udf.function_type.as_deref(),
+    })
+    .context("failed to build UDF runtime")?;
+
+    // legacy UDF runtimes do not support aggregate functions,
+    // so we can assume that the runtime is not legacy
+    let arrow_convert = UdfArrowConvert::default();
+    let arg_schema = Arc::new(Schema::new(
+        udf.arg_types
+            .iter()
+            .map(|t| arrow_convert.to_arrow_field("", &DataType::from(t)))
+            .try_collect::<_, Fields, _>()?,
+    ));
+
+    Ok(Box::new(UserDefinedAggregateFunction {
+        return_field: arrow_convert.to_arrow_field("", &agg.return_type)?,
+        state_field: Field::new("state", arrow_schema::DataType::Binary, true),
+        return_type: agg.return_type.clone(),
+        arg_schema,
+        runtime,
+    }))
+}
diff --git a/src/expr/core/src/codegen.rs b/src/expr/core/src/codegen.rs
index c9022e40aec91..825889bf6fc77 100644
--- a/src/expr/core/src/codegen.rs
+++ b/src/expr/core/src/codegen.rs
@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//! This module contains imports that are used in the generated code for the `#[function]` macro.
+
 pub use async_trait::async_trait;
 pub use futures_async_stream::try_stream;
 pub use futures_util::stream::BoxStream;
-pub use itertools::multizip;
 pub use linkme;
+pub use thiserror_ext::AsReport;
diff --git a/src/expr/core/src/error.rs b/src/expr/core/src/error.rs
index 6688824093d2d..e02c5f4521cf5 100644
--- a/src/expr/core/src/error.rs
+++ b/src/expr/core/src/error.rs
@@ -15,7 +15,7 @@
 use std::fmt::{Debug, Display};
 
 use risingwave_common::array::{ArrayError, ArrayRef};
-use risingwave_common::types::DataType;
+use risingwave_common::types::{DataType, DatumRef, ToText};
 use risingwave_pb::PbFieldNotFound;
 use thiserror::Error;
 use thiserror_ext::AsReport;
@@ -95,13 +95,6 @@ pub enum ExprError {
         anyhow::Error,
     ),
 
-    #[error("UDF error: {0}")]
-    Udf(
-        #[from]
-        #[backtrace]
-        risingwave_udf::Error,
-    ),
-
     #[error("not a constant")]
     NotConstant,
 
@@ -117,26 +110,72 @@ pub enum ExprError {
     #[error("invalid state: {0}")]
     InvalidState(String),
 
-    #[error("error in cryptography: {0}")]
-    Cryptography(Box<CryptographyError>),
-}
-
-#[derive(Debug)]
-pub enum CryptographyStage {
-    Encrypt,
-    Decrypt,
-}
+    /// Function error message returned by UDF.
+    #[error("{0}")]
+    Custom(String),
 
-#[derive(Debug, Error)]
-#[error("{stage:?} stage, reason: {reason}")]
-pub struct CryptographyError {
-    pub stage: CryptographyStage,
-    #[source]
-    pub reason: openssl::error::ErrorStack,
+    /// Error from a function call.
+    ///
+    /// Use [`ExprError::function`] to create this error.
+    #[error("error while evaluating expression `{display}`")]
+    Function {
+        display: Box<str>,
+        #[backtrace]
+        // We don't use `anyhow::Error` because we don't want to always capture the backtrace.
+        source: Box<dyn std::error::Error + Send + Sync>,
+    },
 }
 
 static_assertions::const_assert_eq!(std::mem::size_of::<ExprError>(), 40);
 
+impl ExprError {
+    /// Constructs a [`ExprError::Function`] error with the given information for display.
+    pub fn function<'a>(
+        fn_name: &str,
+        args: impl IntoIterator<Item = DatumRef<'a>>,
+        source: impl Into<Box<dyn std::error::Error + Send + Sync>>,
+    ) -> Self {
+        use std::fmt::Write;
+
+        let display = {
+            let mut s = String::new();
+            write!(s, "{}(", fn_name).unwrap();
+            for (i, arg) in args.into_iter().enumerate() {
+                if i > 0 {
+                    write!(s, ", ").unwrap();
+                }
+                if let Some(arg) = arg {
+                    // Act like `quote_literal(arg::varchar)`.
+                    // Since this is mainly for debugging, we don't need to be too precise.
+                    let arg = arg.to_text();
+                    if arg.contains('\\') {
+                        // use escape format: E'...'
+                        write!(s, "E").unwrap();
+                    }
+                    write!(s, "'").unwrap();
+                    for c in arg.chars() {
+                        match c {
+                            '\'' => write!(s, "''").unwrap(),
+                            '\\' => write!(s, "\\\\").unwrap(),
+                            _ => write!(s, "{}", c).unwrap(),
+                        }
+                    }
+                    write!(s, "'").unwrap();
+                } else {
+                    write!(s, "NULL").unwrap();
+                }
+            }
+            write!(s, ")").unwrap();
+            s
+        };
+
+        Self::Function {
+            display: display.into(),
+            source: source.into(),
+        }
+    }
+}
+
 impl From<chrono::ParseError> for ExprError {
     fn from(e: chrono::ParseError) -> Self {
         Self::Parse(e.to_report_string().into())
@@ -178,6 +217,12 @@ impl From<Vec<ExprError>> for MultiExprError {
     }
 }
 
+impl FromIterator<ExprError> for MultiExprError {
+    fn from_iter<T: IntoIterator<Item = ExprError>>(iter: T) -> Self {
+        Self(iter.into_iter().collect())
+    }
+}
+
 impl IntoIterator for MultiExprError {
     type IntoIter = std::vec::IntoIter<ExprError>;
     type Item = ExprError;
diff --git a/src/expr/core/src/expr/and_or.rs b/src/expr/core/src/expr/and_or.rs
index fac820a61b5b3..f265a8a208d29 100644
--- a/src/expr/core/src/expr/and_or.rs
+++ b/src/expr/core/src/expr/and_or.rs
@@ -142,9 +142,6 @@ fn or(l: Option<bool>, r: Option<bool>) -> Option<bool> {
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::array::DataChunk;
-    use risingwave_common::test_prelude::DataChunkTestExt;
-
     use super::*;
     use crate::expr::build_from_pretty;
 
diff --git a/src/expr/core/src/expr/expr_literal.rs b/src/expr/core/src/expr/expr_literal.rs
index ff6fdb79197a2..e85e6f03af6d7 100644
--- a/src/expr/core/src/expr/expr_literal.rs
+++ b/src/expr/core/src/expr/expr_literal.rs
@@ -96,7 +96,6 @@ mod tests {
     use risingwave_pb::data::{PbDataType, PbDatum};
     use risingwave_pb::expr::expr_node::RexNode::{self, Constant};
     use risingwave_pb::expr::expr_node::Type;
-    use risingwave_pb::expr::ExprNode;
 
     use super::*;
 
diff --git a/src/expr/core/src/expr/expr_some_all.rs b/src/expr/core/src/expr/expr_some_all.rs
index 9250aa6d877cf..3f7bc7abbc9f5 100644
--- a/src/expr/core/src/expr/expr_some_all.rs
+++ b/src/expr/core/src/expr/expr_some_all.rs
@@ -272,7 +272,6 @@ impl Build for SomeAllExpression {
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::array::DataChunk;
     use risingwave_common::row::Row;
     use risingwave_common::test_prelude::DataChunkTestExt;
     use risingwave_common::types::ToOwnedDatum;
diff --git a/src/expr/core/src/expr/expr_udf.rs b/src/expr/core/src/expr/expr_udf.rs
index ed7d597cce52a..c5bce6a2944df 100644
--- a/src/expr/core/src/expr/expr_udf.rs
+++ b/src/expr/core/src/expr/expr_udf.rs
@@ -12,71 +12,37 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
-use std::convert::TryFrom;
-use std::sync::atomic::{AtomicU8, Ordering};
-use std::sync::{Arc, LazyLock, Weak};
-use std::time::Duration;
-
-use anyhow::{Context, Error};
-use arrow_schema::{Field, Fields, Schema};
-use arrow_udf_js::{CallMode as JsCallMode, Runtime as JsRuntime};
-#[cfg(feature = "embedded-deno-udf")]
-use arrow_udf_js_deno::{CallMode as DenoCallMode, Runtime as DenoRuntime};
-#[cfg(feature = "embedded-python-udf")]
-use arrow_udf_python::{CallMode as PythonCallMode, Runtime as PythonRuntime};
-use arrow_udf_wasm::Runtime as WasmRuntime;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, LazyLock};
+
+use anyhow::Context;
+use arrow_schema::{Fields, Schema, SchemaRef};
 use await_tree::InstrumentAwait;
-use cfg_or_panic::cfg_or_panic;
-use moka::sync::Cache;
-use risingwave_common::array::{ArrayError, ArrayRef, DataChunk};
+use prometheus::{exponential_buckets, Registry};
+use risingwave_common::array::arrow::{FromArrow, ToArrow, UdfArrowConvert};
+use risingwave_common::array::{Array, ArrayRef, DataChunk};
+use risingwave_common::metrics::*;
+use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
 use risingwave_common::row::OwnedRow;
 use risingwave_common::types::{DataType, Datum};
 use risingwave_expr::expr_context::FRAGMENT_ID;
 use risingwave_pb::expr::ExprNode;
-use risingwave_udf::metrics::GLOBAL_METRICS;
-use risingwave_udf::ArrowFlightUdfClient;
-use thiserror_ext::AsReport;
 
 use super::{BoxedExpression, Build};
 use crate::expr::Expression;
-use crate::{bail, Result};
+use crate::sig::{UdfImpl, UdfKind, UdfOptions};
+use crate::{bail, ExprError, Result};
 
 #[derive(Debug)]
 pub struct UserDefinedFunction {
     children: Vec<BoxedExpression>,
     arg_types: Vec<DataType>,
     return_type: DataType,
-    #[expect(dead_code)]
-    arg_schema: Arc<Schema>,
-    imp: UdfImpl,
-    identifier: String,
+    arg_schema: SchemaRef,
+    runtime: Box<dyn UdfImpl>,
+    arrow_convert: UdfArrowConvert,
     span: await_tree::Span,
-    /// Number of remaining successful calls until retry is enabled.
-    /// This parameter is designed to prevent continuous retry on every call, which would increase delay.
-    /// Logic:
-    /// It resets to `INITIAL_RETRY_COUNT` after a single failure and then decrements with each call, enabling retry when it reaches zero.
-    /// If non-zero, we will not retry on connection errors to prevent blocking the stream.
-    /// On each connection error, the count will be reset to `INITIAL_RETRY_COUNT`.
-    /// On each successful call, the count will be decreased by 1.
-    /// Link:
-    /// See <https://github.com/risingwavelabs/risingwave/issues/13791>.
-    disable_retry_count: AtomicU8,
-    /// Always retry. Overrides `disable_retry_count`.
-    always_retry_on_network_error: bool,
-}
-
-const INITIAL_RETRY_COUNT: u8 = 16;
-
-#[derive(Debug)]
-pub enum UdfImpl {
-    External(Arc<ArrowFlightUdfClient>),
-    Wasm(Arc<WasmRuntime>),
-    JavaScript(JsRuntime),
-    #[cfg(feature = "embedded-python-udf")]
-    Python(PythonRuntime),
-    #[cfg(feature = "embedded-deno-udf")]
-    Deno(Arc<DenoRuntime>),
+    metrics: Metrics,
 }
 
 #[async_trait::async_trait]
@@ -117,103 +83,39 @@ impl Expression for UserDefinedFunction {
 impl UserDefinedFunction {
     async fn eval_inner(&self, input: &DataChunk) -> Result<ArrayRef> {
         // this will drop invisible rows
-        let arrow_input = arrow_array::RecordBatch::try_from(input)?;
+        let arrow_input = self
+            .arrow_convert
+            .to_record_batch(self.arg_schema.clone(), input)?;
 
         // metrics
-        let metrics = &*GLOBAL_METRICS;
-        // batch query does not have a fragment_id
-        let fragment_id = FRAGMENT_ID::try_with(ToOwned::to_owned)
-            .unwrap_or(0)
-            .to_string();
-        let addr = match &self.imp {
-            UdfImpl::External(client) => client.get_addr(),
-            _ => "",
-        };
-        let language = match &self.imp {
-            UdfImpl::Wasm(_) => "wasm",
-            UdfImpl::JavaScript(_) => "javascript(quickjs)",
-            #[cfg(feature = "embedded-python-udf")]
-            UdfImpl::Python(_) => "python",
-            #[cfg(feature = "embedded-deno-udf")]
-            UdfImpl::Deno(_) => "javascript(deno)",
-            UdfImpl::External(_) => "external",
-        };
-        let labels: &[&str; 4] = &[addr, language, &self.identifier, fragment_id.as_str()];
-        metrics
-            .udf_input_chunk_rows
-            .with_label_values(labels)
+        self.metrics
+            .input_chunk_rows
             .observe(arrow_input.num_rows() as f64);
-        metrics
-            .udf_input_rows
-            .with_label_values(labels)
+        self.metrics
+            .input_rows
             .inc_by(arrow_input.num_rows() as u64);
-        metrics
-            .udf_input_bytes
-            .with_label_values(labels)
+        self.metrics
+            .input_bytes
             .inc_by(arrow_input.get_array_memory_size() as u64);
-        let timer = metrics.udf_latency.with_label_values(labels).start_timer();
-
-        let arrow_output_result: Result<arrow_array::RecordBatch, Error> = match &self.imp {
-            UdfImpl::Wasm(runtime) => runtime.call(&self.identifier, &arrow_input),
-            UdfImpl::JavaScript(runtime) => runtime.call(&self.identifier, &arrow_input),
-            #[cfg(feature = "embedded-python-udf")]
-            UdfImpl::Python(runtime) => runtime.call(&self.identifier, &arrow_input),
-            #[cfg(feature = "embedded-deno-udf")]
-            UdfImpl::Deno(runtime) => tokio::task::block_in_place(|| {
-                tokio::runtime::Handle::current()
-                    .block_on(runtime.call(&self.identifier, arrow_input))
-            }),
-            UdfImpl::External(client) => {
-                let disable_retry_count = self.disable_retry_count.load(Ordering::Relaxed);
-                let result = if self.always_retry_on_network_error {
-                    client
-                        .call_with_always_retry_on_network_error(
-                            &self.identifier,
-                            arrow_input,
-                            &fragment_id,
-                        )
-                        .instrument_await(self.span.clone())
-                        .await
-                } else {
-                    let result = if disable_retry_count != 0 {
-                        client
-                            .call(&self.identifier, arrow_input)
-                            .instrument_await(self.span.clone())
-                            .await
-                    } else {
-                        client
-                            .call_with_retry(&self.identifier, arrow_input)
-                            .instrument_await(self.span.clone())
-                            .await
-                    };
-                    let disable_retry_count = self.disable_retry_count.load(Ordering::Relaxed);
-                    let connection_error = matches!(&result, Err(e) if e.is_connection_error());
-                    if connection_error && disable_retry_count != INITIAL_RETRY_COUNT {
-                        // reset count on connection error
-                        self.disable_retry_count
-                            .store(INITIAL_RETRY_COUNT, Ordering::Relaxed);
-                    } else if !connection_error && disable_retry_count != 0 {
-                        // decrease count on success, ignore if exchange failed
-                        _ = self.disable_retry_count.compare_exchange(
-                            disable_retry_count,
-                            disable_retry_count - 1,
-                            Ordering::Relaxed,
-                            Ordering::Relaxed,
-                        );
-                    }
-                    result
-                };
-                result.map_err(|e| e.into())
-            }
-        };
+        let timer = self.metrics.latency.start_timer();
+
+        let arrow_output_result = self
+            .runtime
+            .call(&arrow_input)
+            .instrument_await(self.span.clone())
+            .await;
+
         timer.stop_and_record();
         if arrow_output_result.is_ok() {
-            &metrics.udf_success_count
+            &self.metrics.success_count
         } else {
-            &metrics.udf_failure_count
+            &self.metrics.failure_count
         }
-        .with_label_values(labels)
         .inc();
+        // update memory usage
+        self.metrics
+            .memory_usage_bytes
+            .set(self.runtime.memory_usage() as i64);
 
         let arrow_output = arrow_output_result?;
 
@@ -225,7 +127,7 @@ impl UserDefinedFunction {
             );
         }
 
-        let output = DataChunk::try_from(&arrow_output)?;
+        let output = self.arrow_convert.from_record_batch(&arrow_output)?;
         let output = output.uncompact(input.visibility().clone());
 
         let Some(array) = output.columns().first() else {
@@ -239,6 +141,22 @@ impl UserDefinedFunction {
             );
         }
 
+        // handle optional error column
+        if let Some(errors) = output.columns().get(1) {
+            if errors.data_type() != DataType::Varchar {
+                bail!(
+                    "UDF returned errors column with invalid type: {:?}",
+                    errors.data_type()
+                );
+            }
+            let errors = errors
+                .as_utf8()
+                .iter()
+                .filter_map(|msg| msg.map(|s| ExprError::Custom(s.into())))
+                .collect();
+            return Err(crate::ExprError::Multiple(array.clone(), errors));
+        }
+
         Ok(array.clone())
     }
 }
@@ -250,171 +168,206 @@ impl Build for UserDefinedFunction {
     ) -> Result<Self> {
         let return_type = DataType::from(prost.get_return_type().unwrap());
         let udf = prost.get_rex_node().unwrap().as_udf().unwrap();
+        let identifier = udf.get_identifier()?;
 
-        #[cfg(not(feature = "embedded-deno-udf"))]
-        let runtime = "quickjs";
+        let language = udf.language.as_str();
+        let runtime = udf.runtime.as_deref();
+        let link = udf.link.as_deref();
 
-        #[cfg(feature = "embedded-deno-udf")]
-        let runtime = match udf.runtime.as_deref() {
-            Some("deno") => "deno",
-            _ => "quickjs",
-        };
+        // lookup UDF builder
+        let build_fn = crate::sig::find_udf_impl(language, runtime, link)?.build_fn;
+        let runtime = build_fn(UdfOptions {
+            kind: UdfKind::Scalar,
+            body: udf.body.as_deref(),
+            compressed_binary: udf.compressed_binary.as_deref(),
+            link: udf.link.as_deref(),
+            identifier,
+            arg_names: &udf.arg_names,
+            return_type: &return_type,
+            always_retry_on_network_error: udf.always_retry_on_network_error,
+            function_type: udf.function_type.as_deref(),
+        })
+        .context("failed to build UDF runtime")?;
 
-        let identifier = udf.get_identifier()?;
-        let imp = match udf.language.as_str() {
-            #[cfg(not(madsim))]
-            "wasm" | "rust" => {
-                let compressed_wasm_binary = udf.get_compressed_binary()?;
-                let wasm_binary = zstd::stream::decode_all(compressed_wasm_binary.as_slice())
-                    .context("failed to decompress wasm binary")?;
-                let runtime = get_or_create_wasm_runtime(&wasm_binary)?;
-                UdfImpl::Wasm(runtime)
-            }
-            "javascript" if runtime != "deno" => {
-                let mut rt = JsRuntime::new()?;
-                let body = format!(
-                    "export function {}({}) {{ {} }}",
-                    identifier,
-                    udf.arg_names.join(","),
-                    udf.get_body()?
-                );
-                rt.add_function(
-                    identifier,
-                    arrow_schema::DataType::try_from(&return_type)?,
-                    JsCallMode::CalledOnNullInput,
-                    &body,
-                )?;
-                UdfImpl::JavaScript(rt)
-            }
-            #[cfg(feature = "embedded-deno-udf")]
-            "javascript" if runtime == "deno" => {
-                let rt = DenoRuntime::new();
-                let body = match udf.get_body() {
-                    Ok(body) => body.clone(),
-                    Err(_) => match udf.get_compressed_binary() {
-                        Ok(compressed_binary) => {
-                            let binary = zstd::stream::decode_all(compressed_binary.as_slice())
-                                .context("failed to decompress binary")?;
-                            String::from_utf8(binary).context("failed to decode binary")?
-                        }
-                        Err(_) => {
-                            bail!("UDF body or compressed binary is required for deno UDF");
-                        }
-                    },
-                };
-
-                let body = if matches!(udf.function_type.as_deref(), Some("async")) {
-                    format!(
-                        "export async function {}({}) {{ {} }}",
-                        identifier,
-                        udf.arg_names.join(","),
-                        body
-                    )
-                } else {
-                    format!(
-                        "export function {}({}) {{ {} }}",
-                        identifier,
-                        udf.arg_names.join(","),
-                        body
-                    )
-                };
-
-                futures::executor::block_on(rt.add_function(
-                    identifier,
-                    arrow_schema::DataType::try_from(&return_type)?,
-                    DenoCallMode::CalledOnNullInput,
-                    &body,
-                ))?;
-
-                UdfImpl::Deno(rt)
-            }
-            #[cfg(feature = "embedded-python-udf")]
-            "python" if udf.body.is_some() => {
-                let mut rt = PythonRuntime::builder().sandboxed(true).build()?;
-                let body = udf.get_body()?;
-                rt.add_function(
-                    identifier,
-                    arrow_schema::DataType::try_from(&return_type)?,
-                    PythonCallMode::CalledOnNullInput,
-                    body,
-                )?;
-                UdfImpl::Python(rt)
-            }
-            #[cfg(not(madsim))]
-            _ => {
-                let link = udf.get_link()?;
-                UdfImpl::External(get_or_create_flight_client(link)?)
-            }
-            #[cfg(madsim)]
-            l => panic!("UDF language {l:?} is not supported on madsim"),
+        let arrow_convert = UdfArrowConvert {
+            legacy: runtime.is_legacy(),
         };
 
         let arg_schema = Arc::new(Schema::new(
             udf.arg_types
                 .iter()
-                .map::<Result<_>, _>(|t| {
-                    Ok(Field::new(
-                        "",
-                        DataType::from(t).try_into().map_err(|e: ArrayError| {
-                            risingwave_udf::Error::unsupported(e.to_report_string())
-                        })?,
-                        true,
-                    ))
-                })
+                .map(|t| arrow_convert.to_arrow_field("", &DataType::from(t)))
                 .try_collect::<Fields>()?,
         ));
 
+        let metrics = GLOBAL_METRICS.with_label_values(
+            link.unwrap_or(""),
+            language,
+            identifier,
+            // batch query does not have a fragment_id
+            &FRAGMENT_ID::try_with(ToOwned::to_owned)
+                .unwrap_or(0)
+                .to_string(),
+        );
+
         Ok(Self {
             children: udf.children.iter().map(build_child).try_collect()?,
             arg_types: udf.arg_types.iter().map(|t| t.into()).collect(),
             return_type,
             arg_schema,
-            imp,
-            identifier: identifier.clone(),
+            runtime,
+            arrow_convert,
             span: format!("udf_call({})", identifier).into(),
-            disable_retry_count: AtomicU8::new(0),
-            always_retry_on_network_error: udf.always_retry_on_network_error,
+            metrics,
         })
     }
 }
 
-#[cfg(not(madsim))]
-/// Get or create a client for the given UDF service.
-///
-/// There is a global cache for clients, so that we can reuse the same client for the same service.
-pub(crate) fn get_or_create_flight_client(link: &str) -> Result<Arc<ArrowFlightUdfClient>> {
-    static CLIENTS: LazyLock<std::sync::Mutex<HashMap<String, Weak<ArrowFlightUdfClient>>>> =
-        LazyLock::new(Default::default);
-    let mut clients = CLIENTS.lock().unwrap();
-    if let Some(client) = clients.get(link).and_then(|c| c.upgrade()) {
-        // reuse existing client
-        Ok(client)
-    } else {
-        // create new client
-        let client = Arc::new(ArrowFlightUdfClient::connect_lazy(link)?);
-        clients.insert(link.into(), Arc::downgrade(&client));
-        Ok(client)
-    }
+/// Monitor metrics for UDF.
+#[derive(Debug, Clone)]
+struct MetricsVec {
+    /// Number of successful UDF calls.
+    success_count: LabelGuardedIntCounterVec<4>,
+    /// Number of failed UDF calls.
+    failure_count: LabelGuardedIntCounterVec<4>,
+    /// Total number of retried UDF calls.
+    retry_count: LabelGuardedIntCounterVec<4>,
+    /// Input chunk rows of UDF calls.
+    input_chunk_rows: LabelGuardedHistogramVec<4>,
+    /// The latency of UDF calls in seconds.
+    latency: LabelGuardedHistogramVec<4>,
+    /// Total number of input rows of UDF calls.
+    input_rows: LabelGuardedIntCounterVec<4>,
+    /// Total number of input bytes of UDF calls.
+    input_bytes: LabelGuardedIntCounterVec<4>,
+    /// Total memory usage of UDF runtime in bytes.
+    memory_usage_bytes: LabelGuardedIntGaugeVec<5>,
 }
 
-/// Get or create a wasm runtime.
-///
-/// Runtimes returned by this function are cached inside for at least 60 seconds.
-/// Later calls with the same binary will reuse the same runtime.
-#[cfg_or_panic(not(madsim))]
-pub fn get_or_create_wasm_runtime(binary: &[u8]) -> Result<Arc<WasmRuntime>> {
-    static RUNTIMES: LazyLock<Cache<md5::Digest, Arc<WasmRuntime>>> = LazyLock::new(|| {
-        Cache::builder()
-            .time_to_idle(Duration::from_secs(60))
-            .build()
-    });
-
-    let md5 = md5::compute(binary);
-    if let Some(runtime) = RUNTIMES.get(&md5) {
-        return Ok(runtime.clone());
+/// Monitor metrics for UDF.
+#[derive(Debug, Clone)]
+struct Metrics {
+    /// Number of successful UDF calls.
+    success_count: LabelGuardedIntCounter<4>,
+    /// Number of failed UDF calls.
+    failure_count: LabelGuardedIntCounter<4>,
+    /// Total number of retried UDF calls.
+    #[allow(dead_code)]
+    retry_count: LabelGuardedIntCounter<4>,
+    /// Input chunk rows of UDF calls.
+    input_chunk_rows: LabelGuardedHistogram<4>,
+    /// The latency of UDF calls in seconds.
+    latency: LabelGuardedHistogram<4>,
+    /// Total number of input rows of UDF calls.
+    input_rows: LabelGuardedIntCounter<4>,
+    /// Total number of input bytes of UDF calls.
+    input_bytes: LabelGuardedIntCounter<4>,
+    /// Total memory usage of UDF runtime in bytes.
+    memory_usage_bytes: LabelGuardedIntGauge<5>,
+}
+
+/// Global UDF metrics.
+static GLOBAL_METRICS: LazyLock<MetricsVec> =
+    LazyLock::new(|| MetricsVec::new(&GLOBAL_METRICS_REGISTRY));
+
+impl MetricsVec {
+    fn new(registry: &Registry) -> Self {
+        let labels = &["link", "language", "name", "fragment_id"];
+        let labels5 = &["link", "language", "name", "fragment_id", "instance_id"];
+        let success_count = register_guarded_int_counter_vec_with_registry!(
+            "udf_success_count",
+            "Total number of successful UDF calls",
+            labels,
+            registry
+        )
+        .unwrap();
+        let failure_count = register_guarded_int_counter_vec_with_registry!(
+            "udf_failure_count",
+            "Total number of failed UDF calls",
+            labels,
+            registry
+        )
+        .unwrap();
+        let retry_count = register_guarded_int_counter_vec_with_registry!(
+            "udf_retry_count",
+            "Total number of retried UDF calls",
+            labels,
+            registry
+        )
+        .unwrap();
+        let input_chunk_rows = register_guarded_histogram_vec_with_registry!(
+            "udf_input_chunk_rows",
+            "Input chunk rows of UDF calls",
+            labels,
+            exponential_buckets(1.0, 2.0, 10).unwrap(), // 1 to 1024
+            registry
+        )
+        .unwrap();
+        let latency = register_guarded_histogram_vec_with_registry!(
+            "udf_latency",
+            "The latency(s) of UDF calls",
+            labels,
+            exponential_buckets(0.000001, 2.0, 30).unwrap(), // 1us to 1000s
+            registry
+        )
+        .unwrap();
+        let input_rows = register_guarded_int_counter_vec_with_registry!(
+            "udf_input_rows",
+            "Total number of input rows of UDF calls",
+            labels,
+            registry
+        )
+        .unwrap();
+        let input_bytes = register_guarded_int_counter_vec_with_registry!(
+            "udf_input_bytes",
+            "Total number of input bytes of UDF calls",
+            labels,
+            registry
+        )
+        .unwrap();
+        let memory_usage_bytes = register_guarded_int_gauge_vec_with_registry!(
+            "udf_memory_usage",
+            "Total memory usage of UDF runtime in bytes",
+            labels5,
+            registry
+        )
+        .unwrap();
+
+        MetricsVec {
+            success_count,
+            failure_count,
+            retry_count,
+            input_chunk_rows,
+            latency,
+            input_rows,
+            input_bytes,
+            memory_usage_bytes,
+        }
     }
 
-    let runtime = Arc::new(arrow_udf_wasm::Runtime::new(binary)?);
-    RUNTIMES.insert(md5, runtime.clone());
-    Ok(runtime)
+    fn with_label_values(
+        &self,
+        link: &str,
+        language: &str,
+        identifier: &str,
+        fragment_id: &str,
+    ) -> Metrics {
+        // generate an unique id for each instance
+        static NEXT_INSTANCE_ID: AtomicU64 = AtomicU64::new(0);
+        let instance_id = NEXT_INSTANCE_ID.fetch_add(1, Ordering::Relaxed).to_string();
+
+        let labels = &[link, language, identifier, fragment_id];
+        let labels5 = &[link, language, identifier, fragment_id, &instance_id];
+
+        Metrics {
+            success_count: self.success_count.with_guarded_label_values(labels),
+            failure_count: self.failure_count.with_guarded_label_values(labels),
+            retry_count: self.retry_count.with_guarded_label_values(labels),
+            input_chunk_rows: self.input_chunk_rows.with_guarded_label_values(labels),
+            latency: self.latency.with_guarded_label_values(labels),
+            input_rows: self.input_rows.with_guarded_label_values(labels),
+            input_bytes: self.input_bytes.with_guarded_label_values(labels),
+            memory_usage_bytes: self.memory_usage_bytes.with_guarded_label_values(labels5),
+        }
+    }
 }
diff --git a/src/expr/core/src/expr/mod.rs b/src/expr/core/src/expr/mod.rs
index 6dbb3906f5618..3f4b07d86bedf 100644
--- a/src/expr/core/src/expr/mod.rs
+++ b/src/expr/core/src/expr/mod.rs
@@ -51,7 +51,6 @@ use risingwave_common::types::{DataType, Datum};
 pub use self::build::*;
 pub use self::expr_input_ref::InputRefExpression;
 pub use self::expr_literal::LiteralExpression;
-pub use self::expr_udf::get_or_create_wasm_runtime;
 pub use self::value::{ValueImpl, ValueRef};
 pub use self::wrapper::*;
 pub use super::{ExprError, Result};
diff --git a/src/expr/core/src/lib.rs b/src/expr/core/src/lib.rs
index da89a4207cfc5..b250b8ce901f5 100644
--- a/src/expr/core/src/lib.rs
+++ b/src/expr/core/src/lib.rs
@@ -35,6 +35,6 @@ pub mod sig;
 pub mod table_function;
 pub mod window_function;
 
-pub use error::{ContextUnavailable, CryptographyError, CryptographyStage, ExprError, Result};
+pub use error::{ContextUnavailable, ExprError, Result};
 pub use risingwave_common::{bail, ensure};
 pub use risingwave_expr_macro::*;
diff --git a/src/expr/core/src/sig/mod.rs b/src/expr/core/src/sig/mod.rs
index 747779f81e990..124a002f6519e 100644
--- a/src/expr/core/src/sig/mod.rs
+++ b/src/expr/core/src/sig/mod.rs
@@ -30,6 +30,10 @@ use crate::expr::BoxedExpression;
 use crate::table_function::BoxedTableFunction;
 use crate::ExprError;
 
+mod udf;
+
+pub use self::udf::*;
+
 /// The global registry of all function signatures.
 pub static FUNCTION_REGISTRY: LazyLock<FunctionRegistry> = LazyLock::new(|| {
     let mut map = FunctionRegistry::default();
diff --git a/src/expr/core/src/sig/udf.rs b/src/expr/core/src/sig/udf.rs
new file mode 100644
index 0000000000000..9a253a78051e9
--- /dev/null
+++ b/src/expr/core/src/sig/udf.rs
@@ -0,0 +1,165 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! UDF implementation interface.
+//!
+//! To support a new language or runtime for UDF, implement the interface in this module.
+//!
+//! See expr/impl/src/udf for the implementations.
+
+use anyhow::{bail, Context, Result};
+use arrow_array::{ArrayRef, BooleanArray, RecordBatch};
+use enum_as_inner::EnumAsInner;
+use futures::stream::BoxStream;
+use risingwave_common::types::DataType;
+
+/// The global registry of UDF implementations.
+///
+/// To register a new UDF implementation:
+///
+/// ```ignore
+/// #[linkme::distributed_slice(UDF_IMPLS)]
+/// static MY_UDF_LANGUAGE: UdfImplDescriptor = UdfImplDescriptor {...};
+/// ```
+#[linkme::distributed_slice]
+pub static UDF_IMPLS: [UdfImplDescriptor];
+
+/// Find a UDF implementation by language.
+pub fn find_udf_impl(
+    language: &str,
+    runtime: Option<&str>,
+    link: Option<&str>,
+) -> Result<&'static UdfImplDescriptor> {
+    let mut impls = UDF_IMPLS
+        .iter()
+        .filter(|desc| (desc.match_fn)(language, runtime, link));
+    let impl_ = impls.next().context(
+        "language not found.\nHINT: UDF feature flag may not be enabled during compilation",
+    )?;
+    if impls.next().is_some() {
+        bail!("multiple UDF implementations found for language: {language}");
+    }
+    Ok(impl_)
+}
+
+/// UDF implementation descriptor.
+///
+/// Every UDF implementation should provide 3 functions:
+pub struct UdfImplDescriptor {
+    /// Returns if a function matches the implementation.
+    ///
+    /// This function is used to determine which implementation to use for a UDF.
+    pub match_fn: fn(language: &str, runtime: Option<&str>, link: Option<&str>) -> bool,
+
+    /// Creates a function from options.
+    ///
+    /// This function will be called when `create function` statement is executed on the frontend.
+    pub create_fn: fn(opts: CreateFunctionOptions<'_>) -> Result<CreateFunctionOutput>,
+
+    /// Builds UDF runtime from verified options.
+    ///
+    /// This function will be called before the UDF is executed on the backend.
+    pub build_fn: fn(opts: UdfOptions<'_>) -> Result<Box<dyn UdfImpl>>,
+}
+
+/// Options for creating a function.
+///
+/// These information are parsed from `CREATE FUNCTION` statement.
+/// Implementations should verify the options and return a `CreateFunctionOutput` in `create_fn`.
+pub struct CreateFunctionOptions<'a> {
+    pub kind: UdfKind,
+    pub name: &'a str,
+    pub arg_names: &'a [String],
+    pub arg_types: &'a [DataType],
+    pub return_type: &'a DataType,
+    pub as_: Option<&'a str>,
+    pub using_link: Option<&'a str>,
+    pub using_base64_decoded: Option<&'a [u8]>,
+}
+
+/// Output of creating a function.
+pub struct CreateFunctionOutput {
+    pub identifier: String,
+    pub body: Option<String>,
+    pub compressed_binary: Option<Vec<u8>>,
+}
+
+/// Options for building a UDF runtime.
+pub struct UdfOptions<'a> {
+    pub kind: UdfKind,
+    pub body: Option<&'a str>,
+    pub compressed_binary: Option<&'a [u8]>,
+    pub link: Option<&'a str>,
+    pub identifier: &'a str,
+    pub arg_names: &'a [String],
+    pub return_type: &'a DataType,
+    pub always_retry_on_network_error: bool,
+    pub function_type: Option<&'a str>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumAsInner)]
+pub enum UdfKind {
+    Scalar,
+    Table,
+    Aggregate,
+}
+
+/// UDF implementation.
+#[async_trait::async_trait]
+pub trait UdfImpl: std::fmt::Debug + Send + Sync {
+    /// Call the scalar function.
+    async fn call(&self, input: &RecordBatch) -> Result<RecordBatch>;
+
+    /// Call the table function.
+    async fn call_table_function<'a>(
+        &'a self,
+        input: &'a RecordBatch,
+    ) -> Result<BoxStream<'a, Result<RecordBatch>>>;
+
+    /// For aggregate function, create the initial state.
+    fn call_agg_create_state(&self) -> Result<ArrayRef> {
+        bail!("aggregate function is not supported");
+    }
+
+    /// For aggregate function, accumulate or retract the state.
+    fn call_agg_accumulate_or_retract(
+        &self,
+        _state: &ArrayRef,
+        _ops: &BooleanArray,
+        _input: &RecordBatch,
+    ) -> Result<ArrayRef> {
+        bail!("aggregate function is not supported");
+    }
+
+    /// For aggregate function, get aggregate result from the state.
+    fn call_agg_finish(&self, _state: &ArrayRef) -> Result<ArrayRef> {
+        bail!("aggregate function is not supported");
+    }
+
+    /// Whether the UDF talks in legacy mode.
+    ///
+    /// If true, decimal and jsonb types are mapped to Arrow `LargeBinary` and `LargeUtf8` types.
+    /// Otherwise, they are mapped to Arrow extension types.
+    /// See <https://github.com/risingwavelabs/arrow-udf/tree/main#extension-types>.
+    fn is_legacy(&self) -> bool {
+        false
+    }
+
+    /// Return the memory size consumed by UDF runtime in bytes.
+    ///
+    /// If not available, return 0.
+    fn memory_usage(&self) -> usize {
+        0
+    }
+}
diff --git a/src/expr/core/src/table_function/mod.rs b/src/expr/core/src/table_function/mod.rs
index 9d2d747cc4ed5..d2d8e291ee076 100644
--- a/src/expr/core/src/table_function/mod.rs
+++ b/src/expr/core/src/table_function/mod.rs
@@ -12,16 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use either::Either;
 use futures_async_stream::try_stream;
 use futures_util::stream::BoxStream;
 use futures_util::StreamExt;
-use itertools::Itertools;
-use risingwave_common::array::{Array, ArrayBuilder, ArrayImpl, ArrayRef, DataChunk};
+use risingwave_common::array::{Array, ArrayBuilder, ArrayImpl, DataChunk};
 use risingwave_common::types::{DataType, DatumRef};
-use risingwave_pb::expr::project_set_select_item::SelectItem;
 use risingwave_pb::expr::table_function::PbType;
-use risingwave_pb::expr::{PbProjectSetSelectItem, PbTableFunction};
+use risingwave_pb::expr::PbTableFunction;
 
 use super::{ExprError, Result};
 use crate::expr::{build_from_prost as expr_build_from_prost, BoxedExpression};
@@ -34,20 +31,21 @@ pub use self::empty::*;
 pub use self::repeat::*;
 use self::user_defined::*;
 
-/// Instance of a table function.
+/// A table function takes a row as input and returns multiple rows as output.
 ///
-/// A table function takes a row as input and returns a table. It is also known as Set-Returning
-/// Function.
+/// It is also known as Set-Returning Function.
 #[async_trait::async_trait]
 pub trait TableFunction: std::fmt::Debug + Sync + Send {
+    /// The data type of the output.
     fn return_type(&self) -> DataType;
 
     /// # Contract of the output
     ///
-    /// The returned `DataChunk` contains exact two columns:
+    /// The returned `DataChunk` contains two or three columns:
     /// - The first column is an I32Array containing row indices of input chunk. It should be
     ///   monotonically increasing.
     /// - The second column is the output values. The data type of the column is `return_type`.
+    /// - (Optional) If any error occurs, the error message is stored in the third column.
     ///
     /// i.e., for the `i`-th input row, the output rows are `(i, output_1)`, `(i, output_2)`, ...
     ///
@@ -108,9 +106,7 @@ pub trait TableFunction: std::fmt::Debug + Sync + Send {
 pub type BoxedTableFunction = Box<dyn TableFunction>;
 
 pub fn build_from_prost(prost: &PbTableFunction, chunk_size: usize) -> Result<BoxedTableFunction> {
-    use risingwave_pb::expr::table_function::Type::*;
-
-    if prost.get_function_type().unwrap() == Udtf {
+    if prost.get_function_type().unwrap() == PbType::UserDefined {
         return new_user_defined(prost, chunk_size);
     }
 
@@ -129,6 +125,7 @@ pub fn build(
     chunk_size: usize,
     children: Vec<BoxedExpression>,
 ) -> Result<BoxedTableFunction> {
+    use itertools::Itertools;
     let args = children.iter().map(|t| t.return_type()).collect_vec();
     let desc = crate::sig::FUNCTION_REGISTRY
         .get(func, &args, &return_type)
@@ -143,53 +140,6 @@ pub fn build(
     desc.build_table(return_type, chunk_size, children)
 }
 
-/// See also [`PbProjectSetSelectItem`]
-#[derive(Debug)]
-pub enum ProjectSetSelectItem {
-    TableFunction(BoxedTableFunction),
-    Expr(BoxedExpression),
-}
-
-impl From<BoxedTableFunction> for ProjectSetSelectItem {
-    fn from(table_function: BoxedTableFunction) -> Self {
-        ProjectSetSelectItem::TableFunction(table_function)
-    }
-}
-
-impl From<BoxedExpression> for ProjectSetSelectItem {
-    fn from(expr: BoxedExpression) -> Self {
-        ProjectSetSelectItem::Expr(expr)
-    }
-}
-
-impl ProjectSetSelectItem {
-    pub fn from_prost(prost: &PbProjectSetSelectItem, chunk_size: usize) -> Result<Self> {
-        match prost.select_item.as_ref().unwrap() {
-            SelectItem::Expr(expr) => expr_build_from_prost(expr).map(Into::into),
-            SelectItem::TableFunction(tf) => build_from_prost(tf, chunk_size).map(Into::into),
-        }
-    }
-
-    pub fn return_type(&self) -> DataType {
-        match self {
-            ProjectSetSelectItem::TableFunction(tf) => tf.return_type(),
-            ProjectSetSelectItem::Expr(expr) => expr.return_type(),
-        }
-    }
-
-    pub async fn eval<'a>(
-        &'a self,
-        input: &'a DataChunk,
-    ) -> Result<Either<TableFunctionOutputIter<'a>, ArrayRef>> {
-        match self {
-            Self::TableFunction(tf) => Ok(Either::Left(
-                TableFunctionOutputIter::new(tf.eval(input).await).await?,
-            )),
-            Self::Expr(expr) => expr.eval(input).await.map(Either::Right),
-        }
-    }
-}
-
 /// A wrapper over the output of table function that allows iteration by rows.
 ///
 /// If the table function returns multiple columns, the output will be struct values.
@@ -224,7 +174,7 @@ impl ProjectSetSelectItem {
 /// for i in 0..4 {
 ///     let (index, value) = iter.peek().unwrap();
 ///     assert_eq!(index, i);
-///     assert_eq!(value, Some((i as i64).into()));
+///     assert_eq!(value.unwrap(), Some((i as i64).into()));
 ///     iter.next().await.unwrap();
 /// }
 /// assert!(iter.peek().is_none());
@@ -250,11 +200,19 @@ impl<'a> TableFunctionOutputIter<'a> {
     }
 
     /// Gets the current row.
-    pub fn peek(&'a self) -> Option<(usize, DatumRef<'a>)> {
+    pub fn peek(&'a self) -> Option<(usize, Result<DatumRef<'a>>)> {
         let chunk = self.chunk.as_ref()?;
         let index = chunk.column_at(0).as_int32().value_at(self.index).unwrap() as usize;
-        let value = chunk.column_at(1).value_at(self.index);
-        Some((index, value))
+        let result = if let Some(msg) = chunk
+            .columns()
+            .get(2)
+            .and_then(|errors| errors.as_utf8().value_at(self.index))
+        {
+            Err(ExprError::Custom(msg.into()))
+        } else {
+            Ok(chunk.column_at(1).value_at(self.index))
+        };
+        Some((index, result))
     }
 
     /// Moves to the next row.
@@ -280,3 +238,20 @@ impl<'a> TableFunctionOutputIter<'a> {
         Ok(())
     }
 }
+
+/// Checks if the output chunk returned by `TableFunction::eval` contains any error.
+pub fn check_error(chunk: &DataChunk) -> Result<()> {
+    if let Some(errors) = chunk.columns().get(2) {
+        if errors.null_bitmap().any() {
+            return Err(ExprError::Custom(
+                errors
+                    .as_utf8()
+                    .iter()
+                    .find_map(|s| s)
+                    .expect("no error message")
+                    .into(),
+            ));
+        }
+    }
+    Ok(())
+}
diff --git a/src/expr/core/src/table_function/user_defined.rs b/src/expr/core/src/table_function/user_defined.rs
index b65ee5e77758b..919c258299cde 100644
--- a/src/expr/core/src/table_function/user_defined.rs
+++ b/src/expr/core/src/table_function/user_defined.rs
@@ -15,30 +15,21 @@
 use std::sync::Arc;
 
 use anyhow::Context;
-use arrow_array::RecordBatch;
-use arrow_schema::{Field, Fields, Schema, SchemaRef};
-use arrow_udf_js::{CallMode as JsCallMode, Runtime as JsRuntime};
-#[cfg(feature = "embedded-deno-udf")]
-use arrow_udf_js_deno::{CallMode as DenoCallMode, Runtime as DenoRuntime};
-#[cfg(feature = "embedded-python-udf")]
-use arrow_udf_python::{CallMode as PythonCallMode, Runtime as PythonRuntime};
-use cfg_or_panic::cfg_or_panic;
-use futures_util::stream;
-use risingwave_common::array::{ArrayError, DataChunk, I32Array};
+use arrow_schema::{Fields, Schema, SchemaRef};
+use risingwave_common::array::arrow::{FromArrow, ToArrow, UdfArrowConvert};
+use risingwave_common::array::I32Array;
 use risingwave_common::bail;
-use thiserror_ext::AsReport;
 
 use super::*;
-use crate::expr::expr_udf::UdfImpl;
+use crate::sig::{UdfImpl, UdfKind, UdfOptions};
 
 #[derive(Debug)]
 pub struct UserDefinedTableFunction {
     children: Vec<BoxedExpression>,
-    #[allow(dead_code)]
     arg_schema: SchemaRef,
     return_type: DataType,
-    client: UdfImpl,
-    identifier: String,
+    runtime: Box<dyn UdfImpl>,
+    arrow_convert: UdfArrowConvert,
     #[allow(dead_code)]
     chunk_size: usize,
 }
@@ -49,54 +40,11 @@ impl TableFunction for UserDefinedTableFunction {
         self.return_type.clone()
     }
 
-    #[cfg_or_panic(not(madsim))]
     async fn eval<'a>(&'a self, input: &'a DataChunk) -> BoxStream<'a, Result<DataChunk>> {
         self.eval_inner(input)
     }
 }
 
-#[cfg(not(madsim))]
-impl UdfImpl {
-    #[try_stream(ok = RecordBatch, error = ExprError)]
-    async fn call_table_function<'a>(&'a self, identifier: &'a str, input: RecordBatch) {
-        match self {
-            UdfImpl::External(client) => {
-                #[for_await]
-                for res in client
-                    .call_stream(identifier, stream::once(async { input }))
-                    .await?
-                {
-                    yield res?;
-                }
-            }
-            UdfImpl::JavaScript(runtime) => {
-                for res in runtime.call_table_function(identifier, &input, 1024)? {
-                    yield res?;
-                }
-            }
-            #[cfg(feature = "embedded-python-udf")]
-            UdfImpl::Python(runtime) => {
-                for res in runtime.call_table_function(identifier, &input, 1024)? {
-                    yield res?;
-                }
-            }
-            #[cfg(feature = "embedded-deno-udf")]
-            UdfImpl::Deno(runtime) => {
-                let mut iter = runtime.call_table_function(identifier, input, 1024).await?;
-                while let Some(res) = iter.next().await {
-                    yield res?;
-                }
-            }
-            UdfImpl::Wasm(runtime) => {
-                for res in runtime.call_table_function(identifier, &input)? {
-                    yield res?;
-                }
-            }
-        }
-    }
-}
-
-#[cfg(not(madsim))]
 impl UserDefinedTableFunction {
     #[try_stream(boxed, ok = DataChunk, error = ExprError)]
     async fn eval_inner<'a>(&'a self, input: &'a DataChunk) {
@@ -109,17 +57,16 @@ impl UserDefinedTableFunction {
         let direct_input = DataChunk::new(columns, input.visibility().clone());
 
         // compact the input chunk and record the row mapping
-        let visible_rows = direct_input.visibility().iter_ones().collect_vec();
-        let compacted_input = direct_input.compact_cow();
-        let arrow_input = RecordBatch::try_from(compacted_input.as_ref())?;
+        let visible_rows = direct_input.visibility().iter_ones().collect::<Vec<_>>();
+        // this will drop invisible rows
+        let arrow_input = self
+            .arrow_convert
+            .to_record_batch(self.arg_schema.clone(), &direct_input)?;
 
         // call UDTF
         #[for_await]
-        for res in self
-            .client
-            .call_table_function(&self.identifier, arrow_input)
-        {
-            let output = DataChunk::try_from(&res?)?;
+        for res in self.runtime.call_table_function(&arrow_input).await? {
+            let output = self.arrow_convert.from_record_batch(&res?)?;
             self.check_output(&output)?;
 
             // we send the compacted input to UDF, so we need to map the row indices back to the
@@ -173,126 +120,46 @@ impl UserDefinedTableFunction {
     }
 }
 
-#[cfg_or_panic(not(madsim))]
 pub fn new_user_defined(prost: &PbTableFunction, chunk_size: usize) -> Result<BoxedTableFunction> {
-    let Some(udtf) = &prost.udtf else {
-        bail!("expect UDTF");
-    };
+    let udf = prost.get_udf()?;
 
-    let arg_schema = Arc::new(Schema::new(
-        udtf.arg_types
-            .iter()
-            .map::<Result<_>, _>(|t| {
-                Ok(Field::new(
-                    "",
-                    DataType::from(t).try_into().map_err(|e: ArrayError| {
-                        risingwave_udf::Error::unsupported(e.to_report_string())
-                    })?,
-                    true,
-                ))
-            })
-            .try_collect::<_, Fields, _>()?,
-    ));
-
-    let identifier = udtf.get_identifier()?;
+    let identifier = udf.get_identifier()?;
     let return_type = DataType::from(prost.get_return_type()?);
 
-    #[cfg(not(feature = "embedded-deno-udf"))]
-    let runtime = "quickjs";
-
-    #[cfg(feature = "embedded-deno-udf")]
-    let runtime = match udtf.runtime.as_deref() {
-        Some("deno") => "deno",
-        _ => "quickjs",
-    };
-
-    let client = match udtf.language.as_str() {
-        "wasm" | "rust" => {
-            let compressed_wasm_binary = udtf.get_compressed_binary()?;
-            let wasm_binary = zstd::stream::decode_all(compressed_wasm_binary.as_slice())
-                .context("failed to decompress wasm binary")?;
-            let runtime = crate::expr::expr_udf::get_or_create_wasm_runtime(&wasm_binary)?;
-            UdfImpl::Wasm(runtime)
-        }
-        "javascript" if runtime != "deno" => {
-            let mut rt = JsRuntime::new()?;
-            let body = format!(
-                "export function* {}({}) {{ {} }}",
-                identifier,
-                udtf.arg_names.join(","),
-                udtf.get_body()?
-            );
-            rt.add_function(
-                identifier,
-                arrow_schema::DataType::try_from(&return_type)?,
-                JsCallMode::CalledOnNullInput,
-                &body,
-            )?;
-            UdfImpl::JavaScript(rt)
-        }
-        #[cfg(feature = "embedded-deno-udf")]
-        "javascript" if runtime == "deno" => {
-            let rt = DenoRuntime::new();
-            let body = match udtf.get_body() {
-                Ok(body) => body.clone(),
-                Err(_) => match udtf.get_compressed_binary() {
-                    Ok(compressed_binary) => {
-                        let binary = zstd::stream::decode_all(compressed_binary.as_slice())
-                            .context("failed to decompress binary")?;
-                        String::from_utf8(binary).context("failed to decode binary")?
-                    }
-                    Err(_) => {
-                        bail!("UDF body or compressed binary is required for deno UDF");
-                    }
-                },
-            };
-
-            let body = format!(
-                "export {} {}({}) {{ {} }}",
-                match udtf.function_type.as_deref() {
-                    Some("async") => "async function",
-                    Some("async_generator") => "async function*",
-                    Some("sync") => "function",
-                    _ => "function*",
-                },
-                identifier,
-                udtf.arg_names.join(","),
-                body
-            );
-
-            futures::executor::block_on(rt.add_function(
-                identifier,
-                arrow_schema::DataType::try_from(&return_type)?,
-                DenoCallMode::CalledOnNullInput,
-                &body,
-            ))?;
-            UdfImpl::Deno(rt)
-        }
-        #[cfg(feature = "embedded-python-udf")]
-        "python" if udtf.body.is_some() => {
-            let mut rt = PythonRuntime::builder().sandboxed(true).build()?;
-            let body = udtf.get_body()?;
-            rt.add_function(
-                identifier,
-                arrow_schema::DataType::try_from(&return_type)?,
-                PythonCallMode::CalledOnNullInput,
-                body,
-            )?;
-            UdfImpl::Python(rt)
-        }
-        // connect to UDF service
-        _ => {
-            let link = udtf.get_link()?;
-            UdfImpl::External(crate::expr::expr_udf::get_or_create_flight_client(link)?)
-        }
+    let language = udf.language.as_str();
+    let runtime = udf.runtime.as_deref();
+    let link = udf.link.as_deref();
+
+    let build_fn = crate::sig::find_udf_impl(language, runtime, link)?.build_fn;
+    let runtime = build_fn(UdfOptions {
+        kind: UdfKind::Table,
+        body: udf.body.as_deref(),
+        compressed_binary: udf.compressed_binary.as_deref(),
+        link: udf.link.as_deref(),
+        identifier,
+        arg_names: &udf.arg_names,
+        return_type: &return_type,
+        always_retry_on_network_error: false,
+        function_type: udf.function_type.as_deref(),
+    })
+    .context("failed to build UDF runtime")?;
+
+    let arrow_convert = UdfArrowConvert {
+        legacy: runtime.is_legacy(),
     };
+    let arg_schema = Arc::new(Schema::new(
+        udf.arg_types
+            .iter()
+            .map(|t| arrow_convert.to_arrow_field("", &DataType::from(t)))
+            .try_collect::<Fields>()?,
+    ));
 
     Ok(UserDefinedTableFunction {
         children: prost.args.iter().map(expr_build_from_prost).try_collect()?,
         return_type,
         arg_schema,
-        client,
-        identifier: identifier.clone(),
+        runtime,
+        arrow_convert,
         chunk_size,
     }
     .boxed())
diff --git a/src/expr/core/src/window_function/call.rs b/src/expr/core/src/window_function/call.rs
index f5714c64345a1..5c6f40ff25bc0 100644
--- a/src/expr/core/src/window_function/call.rs
+++ b/src/expr/core/src/window_function/call.rs
@@ -13,34 +13,19 @@
 // limitations under the License.
 
 use std::fmt::Display;
-use std::ops::Deref;
-use std::sync::Arc;
 
-use anyhow::Context;
-use educe::Educe;
 use enum_as_inner::EnumAsInner;
-use futures_util::FutureExt;
 use parse_display::Display;
-use risingwave_common::row::OwnedRow;
-use risingwave_common::types::{
-    DataType, Datum, IsNegative, ScalarImpl, ScalarRefImpl, Sentinelled, ToOwnedDatum, ToText,
-};
-use risingwave_common::util::sort_util::{Direction, OrderType};
-use risingwave_common::util::value_encoding::{DatumFromProtoExt, DatumToProtoExt};
+use risingwave_common::types::DataType;
 use risingwave_common::{bail, must_match};
-use risingwave_pb::expr::window_frame::{
-    PbBound, PbBoundType, PbBounds, PbExclusion, PbRangeFrameBound, PbRangeFrameBounds,
-    PbRowsFrameBound, PbRowsFrameBounds,
-};
+use risingwave_pb::expr::window_frame::{PbBounds, PbExclusion};
 use risingwave_pb::expr::{PbWindowFrame, PbWindowFunction};
 use FrameBound::{CurrentRow, Following, Preceding, UnboundedFollowing, UnboundedPreceding};
 
-use super::WindowFuncKind;
-use crate::aggregate::AggArgs;
-use crate::expr::{
-    build_func, BoxedExpression, Expression, ExpressionBoxExt, InputRefExpression,
-    LiteralExpression,
+use super::{
+    RangeFrameBounds, RowsFrameBound, RowsFrameBounds, SessionFrameBounds, WindowFuncKind,
 };
+use crate::aggregate::AggArgs;
 use crate::Result;
 
 #[derive(Debug, Clone)]
@@ -117,6 +102,10 @@ impl Frame {
                 let bounds = must_match!(frame.get_bounds()?, PbBounds::Range(bounds) => bounds);
                 FrameBounds::Range(RangeFrameBounds::from_protobuf(bounds)?)
             }
+            PbType::Session => {
+                let bounds = must_match!(frame.get_bounds()?, PbBounds::Session(bounds) => bounds);
+                FrameBounds::Session(SessionFrameBounds::from_protobuf(bounds)?)
+            }
         };
         let exclusion = FrameExclusion::from_protobuf(frame.get_exclusion()?)?;
         Ok(Self { bounds, exclusion })
@@ -125,8 +114,8 @@ impl Frame {
     pub fn to_protobuf(&self) -> PbWindowFrame {
         use risingwave_pb::expr::window_frame::PbType;
         let exclusion = self.exclusion.to_protobuf() as _;
+        #[expect(deprecated)] // because of `start` and `end` fields
         match &self.bounds {
-            #[expect(deprecated)]
             FrameBounds::Rows(bounds) => PbWindowFrame {
                 r#type: PbType::Rows as _,
                 start: None, // deprecated
@@ -134,7 +123,6 @@ impl Frame {
                 exclusion,
                 bounds: Some(PbBounds::Rows(bounds.to_protobuf())),
             },
-            #[expect(deprecated)]
             FrameBounds::Range(bounds) => PbWindowFrame {
                 r#type: PbType::Range as _,
                 start: None, // deprecated
@@ -142,6 +130,13 @@ impl Frame {
                 exclusion,
                 bounds: Some(PbBounds::Range(bounds.to_protobuf())),
             },
+            FrameBounds::Session(bounds) => PbWindowFrame {
+                r#type: PbType::Session as _,
+                start: None, // deprecated
+                end: None,   // deprecated
+                exclusion,
+                bounds: Some(PbBounds::Session(bounds.to_protobuf())),
+            },
         }
     }
 }
@@ -152,6 +147,7 @@ pub enum FrameBounds {
     Rows(RowsFrameBounds),
     // Groups(GroupsFrameBounds),
     Range(RangeFrameBounds),
+    Session(SessionFrameBounds),
 }
 
 impl FrameBounds {
@@ -159,6 +155,7 @@ impl FrameBounds {
         match self {
             Self::Rows(bounds) => bounds.validate(),
             Self::Range(bounds) => bounds.validate(),
+            Self::Session(bounds) => bounds.validate(),
         }
     }
 
@@ -166,6 +163,7 @@ impl FrameBounds {
         match self {
             Self::Rows(RowsFrameBounds { start, .. }) => start.is_unbounded_preceding(),
             Self::Range(RangeFrameBounds { start, .. }) => start.is_unbounded_preceding(),
+            Self::Session(_) => false,
         }
     }
 
@@ -173,6 +171,7 @@ impl FrameBounds {
         match self {
             Self::Rows(RowsFrameBounds { end, .. }) => end.is_unbounded_following(),
             Self::Range(RangeFrameBounds { end, .. }) => end.is_unbounded_following(),
+            Self::Session(_) => false,
         }
     }
 
@@ -185,324 +184,6 @@ pub trait FrameBoundsImpl {
     fn validate(&self) -> Result<()>;
 }
 
-#[derive(Display, Debug, Clone, Eq, PartialEq, Hash)]
-#[display("ROWS BETWEEN {start} AND {end}")]
-pub struct RowsFrameBounds {
-    pub start: RowsFrameBound,
-    pub end: RowsFrameBound,
-}
-
-impl RowsFrameBounds {
-    fn from_protobuf(bounds: &PbRowsFrameBounds) -> Result<Self> {
-        let start = FrameBound::<usize>::from_protobuf(bounds.get_start()?)?;
-        let end = FrameBound::<usize>::from_protobuf(bounds.get_end()?)?;
-        Ok(Self { start, end })
-    }
-
-    fn to_protobuf(&self) -> PbRowsFrameBounds {
-        PbRowsFrameBounds {
-            start: Some(self.start.to_protobuf()),
-            end: Some(self.end.to_protobuf()),
-        }
-    }
-}
-
-impl RowsFrameBounds {
-    /// Check if the `ROWS` frame is canonical.
-    ///
-    /// A canonical `ROWS` frame is defined as:
-    ///
-    /// - Its bounds are valid (see [`Self::validate`]).
-    /// - It contains the current row.
-    pub fn is_canonical(&self) -> bool {
-        self.validate().is_ok() && {
-            let start = self.start.to_offset();
-            let end = self.end.to_offset();
-            start.unwrap_or(0) <= 0 && end.unwrap_or(0) >= 0
-        }
-    }
-
-    /// Get the number of preceding rows.
-    pub fn n_preceding_rows(&self) -> Option<usize> {
-        match (&self.start, &self.end) {
-            (UnboundedPreceding, _) => None,
-            (Preceding(n1), Preceding(n2)) => Some(*n1.max(n2)),
-            (Preceding(n), _) => Some(*n),
-            (CurrentRow | Following(_) | UnboundedFollowing, _) => Some(0),
-        }
-    }
-
-    /// Get the number of following rows.
-    pub fn n_following_rows(&self) -> Option<usize> {
-        match (&self.start, &self.end) {
-            (_, UnboundedFollowing) => None,
-            (Following(n1), Following(n2)) => Some(*n1.max(n2)),
-            (_, Following(n)) => Some(*n),
-            (_, CurrentRow | Preceding(_) | UnboundedPreceding) => Some(0),
-        }
-    }
-}
-
-impl FrameBoundsImpl for RowsFrameBounds {
-    fn validate(&self) -> Result<()> {
-        FrameBound::validate_bounds(&self.start, &self.end, |_| Ok(()))
-    }
-}
-
-#[derive(Debug, Clone, Eq, PartialEq, Hash)]
-pub struct RangeFrameBounds {
-    pub order_data_type: DataType,
-    pub order_type: OrderType,
-    pub offset_data_type: DataType,
-    pub start: RangeFrameBound,
-    pub end: RangeFrameBound,
-}
-
-impl RangeFrameBounds {
-    fn from_protobuf(bounds: &PbRangeFrameBounds) -> Result<Self> {
-        let order_data_type = DataType::from(bounds.get_order_data_type()?);
-        let order_type = OrderType::from_protobuf(bounds.get_order_type()?);
-        let offset_data_type = DataType::from(bounds.get_offset_data_type()?);
-        let start = FrameBound::<RangeFrameOffset>::from_protobuf(
-            bounds.get_start()?,
-            &order_data_type,
-            &offset_data_type,
-        )?;
-        let end = FrameBound::<RangeFrameOffset>::from_protobuf(
-            bounds.get_end()?,
-            &order_data_type,
-            &offset_data_type,
-        )?;
-        Ok(Self {
-            order_data_type,
-            order_type,
-            offset_data_type,
-            start,
-            end,
-        })
-    }
-
-    fn to_protobuf(&self) -> PbRangeFrameBounds {
-        PbRangeFrameBounds {
-            start: Some(self.start.to_protobuf()),
-            end: Some(self.end.to_protobuf()),
-            order_data_type: Some(self.order_data_type.to_protobuf()),
-            order_type: Some(self.order_type.to_protobuf()),
-            offset_data_type: Some(self.offset_data_type.to_protobuf()),
-        }
-    }
-}
-
-/// The wrapper type for [`ScalarImpl`] range frame offset, containing
-/// two expressions to help adding and subtracting the offset.
-#[derive(Debug, Clone, Educe)]
-#[educe(PartialEq, Eq, Hash)]
-pub struct RangeFrameOffset {
-    /// The original offset value.
-    offset: ScalarImpl,
-    /// Built expression for `$0 + offset`.
-    #[educe(PartialEq(ignore), Hash(ignore))]
-    add_expr: Option<Arc<BoxedExpression>>,
-    /// Built expression for `$0 - offset`.
-    #[educe(PartialEq(ignore), Hash(ignore))]
-    sub_expr: Option<Arc<BoxedExpression>>,
-}
-
-impl RangeFrameOffset {
-    pub fn new(offset: ScalarImpl) -> Self {
-        Self {
-            offset,
-            add_expr: None,
-            sub_expr: None,
-        }
-    }
-
-    fn build_exprs(
-        &mut self,
-        order_data_type: &DataType,
-        offset_data_type: &DataType,
-    ) -> Result<()> {
-        use risingwave_pb::expr::expr_node::PbType as PbExprType;
-
-        let input_expr = InputRefExpression::new(order_data_type.clone(), 0);
-        let offset_expr =
-            LiteralExpression::new(offset_data_type.clone(), Some(self.offset.clone()));
-        self.add_expr = Some(Arc::new(build_func(
-            PbExprType::Add,
-            order_data_type.clone(),
-            vec![input_expr.clone().boxed(), offset_expr.clone().boxed()],
-        )?));
-        self.sub_expr = Some(Arc::new(build_func(
-            PbExprType::Subtract,
-            order_data_type.clone(),
-            vec![input_expr.boxed(), offset_expr.boxed()],
-        )?));
-        Ok(())
-    }
-
-    pub fn new_for_test(
-        offset: ScalarImpl,
-        order_data_type: &DataType,
-        offset_data_type: &DataType,
-    ) -> Self {
-        let mut offset = Self::new(offset);
-        offset
-            .build_exprs(order_data_type, offset_data_type)
-            .unwrap();
-        offset
-    }
-}
-
-impl Deref for RangeFrameOffset {
-    type Target = ScalarImpl;
-
-    fn deref(&self) -> &Self::Target {
-        &self.offset
-    }
-}
-
-impl Display for RangeFrameBounds {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "RANGE BETWEEN {} AND {}",
-            self.start.for_display(),
-            self.end.for_display()
-        )?;
-        Ok(())
-    }
-}
-
-impl FrameBoundsImpl for RangeFrameBounds {
-    fn validate(&self) -> Result<()> {
-        fn validate_non_negative(val: impl IsNegative + Display) -> Result<()> {
-            if val.is_negative() {
-                bail!(
-                    "frame bound offset should be non-negative, but {} is given",
-                    val
-                );
-            }
-            Ok(())
-        }
-
-        FrameBound::validate_bounds(&self.start, &self.end, |offset| {
-            match offset.as_scalar_ref_impl() {
-                // TODO(rc): use decl macro?
-                ScalarRefImpl::Int16(val) => validate_non_negative(val)?,
-                ScalarRefImpl::Int32(val) => validate_non_negative(val)?,
-                ScalarRefImpl::Int64(val) => validate_non_negative(val)?,
-                ScalarRefImpl::Float32(val) => validate_non_negative(val)?,
-                ScalarRefImpl::Float64(val) => validate_non_negative(val)?,
-                ScalarRefImpl::Decimal(val) => validate_non_negative(val)?,
-                ScalarRefImpl::Interval(val) => {
-                    if !val.is_never_negative() {
-                        bail!(
-                            "for frame bound offset of type `interval`, each field should be non-negative, but {} is given",
-                            val
-                        );
-                    }
-                    if matches!(self.order_data_type, DataType::Timestamptz) {
-                        // for `timestamptz`, we only support offset without `month` and `day` fields
-                        if val.months() != 0 || val.days() != 0 {
-                            bail!(
-                                "for frame order column of type `timestamptz`, offset should not have non-zero `month` and `day`",
-                            );
-                        }
-                    }
-                },
-                _ => unreachable!("other order column data types are not supported and should be banned in frontend"),
-            }
-            Ok(())
-        })
-    }
-}
-
-impl RangeFrameBounds {
-    /// Get the frame start for a given order column value.
-    ///
-    /// ## Examples
-    ///
-    /// For the following frames:
-    ///
-    /// ```sql
-    /// ORDER BY x ASC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
-    /// ORDER BY x DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
-    /// ```
-    ///
-    /// For any CURRENT ROW with any order value, the frame start is always the first-most row, which is
-    /// represented by [`Sentinelled::Smallest`].
-    ///
-    /// For the following frame:
-    ///
-    /// ```sql
-    /// ORDER BY x ASC RANGE BETWEEN 10 PRECEDING AND CURRENT ROW
-    /// ```
-    ///
-    /// For CURRENT ROW with order value `100`, the frame start is the **FIRST** row with order value `90`.
-    ///
-    /// For the following frame:
-    ///
-    /// ```sql
-    /// ORDER BY x DESC RANGE BETWEEN 10 PRECEDING AND CURRENT ROW
-    /// ```
-    ///
-    /// For CURRENT ROW with order value `100`, the frame start is the **FIRST** row with order value `110`.
-    pub fn frame_start_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled<Datum> {
-        self.start.for_calc().bound_of(order_value, self.order_type)
-    }
-
-    /// Get the frame end for a given order column value. It's very similar to `frame_start_of`, just with
-    /// everything on the other direction.
-    pub fn frame_end_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled<Datum> {
-        self.end.for_calc().bound_of(order_value, self.order_type)
-    }
-
-    /// Get the order value of the CURRENT ROW of the first frame that includes the given order value.
-    ///
-    /// ## Examples
-    ///
-    /// For the following frames:
-    ///
-    /// ```sql
-    /// ORDER BY x ASC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
-    /// ORDER BY x DESC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
-    /// ```
-    ///
-    /// For any given order value, the first CURRENT ROW is always the first-most row, which is
-    /// represented by [`Sentinelled::Smallest`].
-    ///
-    /// For the following frame:
-    ///
-    /// ```sql
-    /// ORDER BY x ASC RANGE BETWEEN CURRENT ROW AND 10 FOLLOWING
-    /// ```
-    ///
-    /// For a given order value `100`, the first CURRENT ROW should have order value `90`.
-    ///
-    /// For the following frame:
-    ///
-    /// ```sql
-    /// ORDER BY x DESC RANGE BETWEEN CURRENT ROW AND 10 FOLLOWING
-    /// ```
-    ///
-    /// For a given order value `100`, the first CURRENT ROW should have order value `110`.
-    pub fn first_curr_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled<Datum> {
-        self.end
-            .for_calc()
-            .reverse()
-            .bound_of(order_value, self.order_type)
-    }
-
-    /// Get the order value of the CURRENT ROW of the last frame that includes the given order value.
-    /// It's very similar to `first_curr_of`, just with everything on the other direction.
-    pub fn last_curr_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled<Datum> {
-        self.start
-            .for_calc()
-            .reverse()
-            .bound_of(order_value, self.order_type)
-    }
-}
-
 #[derive(Display, Debug, Clone, Eq, PartialEq, Hash, EnumAsInner)]
 #[display(style = "TITLE CASE")]
 pub enum FrameBound<T> {
@@ -515,9 +196,6 @@ pub enum FrameBound<T> {
     UnboundedFollowing,
 }
 
-pub type RowsFrameBound = FrameBound<usize>;
-pub type RangeFrameBound = FrameBound<RangeFrameOffset>;
-
 impl<T> FrameBound<T> {
     fn offset_value(&self) -> Option<&T> {
         match self {
@@ -526,7 +204,7 @@ impl<T> FrameBound<T> {
         }
     }
 
-    fn validate_bounds(
+    pub(super) fn validate_bounds(
         start: &Self,
         end: &Self,
         offset_checker: impl Fn(&T) -> Result<()>,
@@ -569,7 +247,7 @@ impl<T> FrameBound<T>
 where
     T: Copy,
 {
-    fn reverse(self) -> FrameBound<T> {
+    pub(super) fn reverse(self) -> FrameBound<T> {
         match self {
             UnboundedPreceding => UnboundedFollowing,
             Preceding(offset) => Following(offset),
@@ -580,172 +258,6 @@ where
     }
 }
 
-impl RowsFrameBound {
-    fn from_protobuf_legacy(bound: &PbBound) -> Result<Self> {
-        use risingwave_pb::expr::window_frame::bound::PbOffset;
-
-        let offset = bound.get_offset()?;
-        let bound = match offset {
-            PbOffset::Integer(offset) => Self::from_protobuf(&PbRowsFrameBound {
-                r#type: bound.get_type()? as _,
-                offset: Some(*offset),
-            })?,
-            PbOffset::Datum(_) => bail!("offset of `RowsFrameBound` must be `Integer`"),
-        };
-        Ok(bound)
-    }
-
-    fn from_protobuf(bound: &PbRowsFrameBound) -> Result<Self> {
-        let bound = match bound.get_type()? {
-            PbBoundType::Unspecified => bail!("unspecified type of `RowsFrameBound`"),
-            PbBoundType::UnboundedPreceding => Self::UnboundedPreceding,
-            PbBoundType::Preceding => Self::Preceding(*bound.get_offset()? as usize),
-            PbBoundType::CurrentRow => Self::CurrentRow,
-            PbBoundType::Following => Self::Following(*bound.get_offset()? as usize),
-            PbBoundType::UnboundedFollowing => Self::UnboundedFollowing,
-        };
-        Ok(bound)
-    }
-
-    fn to_protobuf(&self) -> PbRowsFrameBound {
-        let (r#type, offset) = match self {
-            Self::UnboundedPreceding => (PbBoundType::UnboundedPreceding, None),
-            Self::Preceding(offset) => (PbBoundType::Preceding, Some(*offset as _)),
-            Self::CurrentRow => (PbBoundType::CurrentRow, None),
-            Self::Following(offset) => (PbBoundType::Following, Some(*offset as _)),
-            Self::UnboundedFollowing => (PbBoundType::UnboundedFollowing, None),
-        };
-        PbRowsFrameBound {
-            r#type: r#type as _,
-            offset,
-        }
-    }
-}
-
-impl RowsFrameBound {
-    /// Convert the bound to sized offset from current row. `None` if the bound is unbounded.
-    pub fn to_offset(&self) -> Option<isize> {
-        match self {
-            UnboundedPreceding | UnboundedFollowing => None,
-            CurrentRow => Some(0),
-            Preceding(n) => Some(-(*n as isize)),
-            Following(n) => Some(*n as isize),
-        }
-    }
-}
-
-impl RangeFrameBound {
-    fn from_protobuf(
-        bound: &PbRangeFrameBound,
-        order_data_type: &DataType,
-        offset_data_type: &DataType,
-    ) -> Result<Self> {
-        let bound = match bound.get_type()? {
-            PbBoundType::Unspecified => bail!("unspecified type of `RangeFrameBound`"),
-            PbBoundType::UnboundedPreceding => Self::UnboundedPreceding,
-            PbBoundType::CurrentRow => Self::CurrentRow,
-            PbBoundType::UnboundedFollowing => Self::UnboundedFollowing,
-            bound_type @ (PbBoundType::Preceding | PbBoundType::Following) => {
-                let offset_value = Datum::from_protobuf(bound.get_offset()?, offset_data_type)
-                    .context("offset `Datum` is not decodable")?
-                    .context("offset of `RangeFrameBound` must be non-NULL")?;
-                let mut offset = RangeFrameOffset::new(offset_value);
-                offset.build_exprs(order_data_type, offset_data_type)?;
-                if bound_type == PbBoundType::Preceding {
-                    Self::Preceding(offset)
-                } else {
-                    Self::Following(offset)
-                }
-            }
-        };
-        Ok(bound)
-    }
-
-    fn to_protobuf(&self) -> PbRangeFrameBound {
-        let (r#type, offset) = match self {
-            Self::UnboundedPreceding => (PbBoundType::UnboundedPreceding, None),
-            Self::Preceding(offset) => (
-                PbBoundType::Preceding,
-                Some(Some(offset.as_scalar_ref_impl()).to_protobuf()),
-            ),
-            Self::CurrentRow => (PbBoundType::CurrentRow, None),
-            Self::Following(offset) => (
-                PbBoundType::Following,
-                Some(Some(offset.as_scalar_ref_impl()).to_protobuf()),
-            ),
-            Self::UnboundedFollowing => (PbBoundType::UnboundedFollowing, None),
-        };
-        PbRangeFrameBound {
-            r#type: r#type as _,
-            offset,
-        }
-    }
-}
-
-impl RangeFrameBound {
-    fn for_display(&self) -> FrameBound<String> {
-        match self {
-            UnboundedPreceding => UnboundedPreceding,
-            Preceding(offset) => Preceding(offset.as_scalar_ref_impl().to_text()),
-            CurrentRow => CurrentRow,
-            Following(offset) => Following(offset.as_scalar_ref_impl().to_text()),
-            UnboundedFollowing => UnboundedFollowing,
-        }
-    }
-
-    fn for_calc(&self) -> FrameBound<RangeFrameOffsetRef<'_>> {
-        match self {
-            UnboundedPreceding => UnboundedPreceding,
-            Preceding(offset) => Preceding(RangeFrameOffsetRef {
-                add_expr: offset.add_expr.as_ref().unwrap().as_ref(),
-                sub_expr: offset.sub_expr.as_ref().unwrap().as_ref(),
-            }),
-            CurrentRow => CurrentRow,
-            Following(offset) => Following(RangeFrameOffsetRef {
-                add_expr: offset.add_expr.as_ref().unwrap().as_ref(),
-                sub_expr: offset.sub_expr.as_ref().unwrap().as_ref(),
-            }),
-            UnboundedFollowing => UnboundedFollowing,
-        }
-    }
-}
-
-#[derive(Debug, Educe)]
-#[educe(Clone, Copy)]
-pub struct RangeFrameOffsetRef<'a> {
-    /// Built expression for `$0 + offset`.
-    add_expr: &'a dyn Expression,
-    /// Built expression for `$0 - offset`.
-    sub_expr: &'a dyn Expression,
-}
-
-impl FrameBound<RangeFrameOffsetRef<'_>> {
-    fn bound_of(self, order_value: impl ToOwnedDatum, order_type: OrderType) -> Sentinelled<Datum> {
-        let expr = match (self, order_type.direction()) {
-            (UnboundedPreceding, _) => return Sentinelled::Smallest,
-            (UnboundedFollowing, _) => return Sentinelled::Largest,
-            (CurrentRow, _) => return Sentinelled::Normal(order_value.to_owned_datum()),
-            (Preceding(offset), Direction::Ascending)
-            | (Following(offset), Direction::Descending) => {
-                // should SUBTRACT the offset
-                offset.sub_expr
-            }
-            (Following(offset), Direction::Ascending)
-            | (Preceding(offset), Direction::Descending) => {
-                // should ADD the offset
-                offset.add_expr
-            }
-        };
-        let row = OwnedRow::new(vec![order_value.to_owned_datum()]);
-        Sentinelled::Normal(
-            expr.eval_row(&row)
-                .now_or_never()
-                .expect("frame bound calculation should finish immediately")
-                .expect("just simple calculation, should succeed"), // TODO(rc): handle overflow
-        )
-    }
-}
-
 #[derive(Display, Debug, Copy, Clone, Eq, PartialEq, Hash, Default, EnumAsInner)]
 #[display("EXCLUDE {}", style = "TITLE CASE")]
 pub enum FrameExclusion {
@@ -773,92 +285,3 @@ impl FrameExclusion {
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-
-    #[test]
-    fn test_rows_frame_bounds() {
-        let bounds = RowsFrameBounds {
-            start: Preceding(1),
-            end: CurrentRow,
-        };
-        assert!(bounds.validate().is_ok());
-        assert!(bounds.is_canonical());
-        assert_eq!(bounds.start.to_offset(), Some(-1));
-        assert_eq!(bounds.end.to_offset(), Some(0));
-        assert_eq!(bounds.n_preceding_rows(), Some(1));
-        assert_eq!(bounds.n_following_rows(), Some(0));
-
-        let bounds = RowsFrameBounds {
-            start: CurrentRow,
-            end: Following(1),
-        };
-        assert!(bounds.validate().is_ok());
-        assert!(bounds.is_canonical());
-        assert_eq!(bounds.start.to_offset(), Some(0));
-        assert_eq!(bounds.end.to_offset(), Some(1));
-        assert_eq!(bounds.n_preceding_rows(), Some(0));
-        assert_eq!(bounds.n_following_rows(), Some(1));
-
-        let bounds = RowsFrameBounds {
-            start: UnboundedPreceding,
-            end: Following(10),
-        };
-        assert!(bounds.validate().is_ok());
-        assert!(bounds.is_canonical());
-        assert_eq!(bounds.start.to_offset(), None);
-        assert_eq!(bounds.end.to_offset(), Some(10));
-        assert_eq!(bounds.n_preceding_rows(), None);
-        assert_eq!(bounds.n_following_rows(), Some(10));
-
-        let bounds = RowsFrameBounds {
-            start: Preceding(10),
-            end: UnboundedFollowing,
-        };
-        assert!(bounds.validate().is_ok());
-        assert!(bounds.is_canonical());
-        assert_eq!(bounds.start.to_offset(), Some(-10));
-        assert_eq!(bounds.end.to_offset(), None);
-        assert_eq!(bounds.n_preceding_rows(), Some(10));
-        assert_eq!(bounds.n_following_rows(), None);
-
-        let bounds = RowsFrameBounds {
-            start: Preceding(1),
-            end: Preceding(10),
-        };
-        assert!(bounds.validate().is_ok());
-        assert!(!bounds.is_canonical());
-        assert_eq!(bounds.start.to_offset(), Some(-1));
-        assert_eq!(bounds.end.to_offset(), Some(-10));
-        assert_eq!(bounds.n_preceding_rows(), Some(10));
-        assert_eq!(bounds.n_following_rows(), Some(0));
-
-        let bounds = RowsFrameBounds {
-            start: Following(10),
-            end: Following(1),
-        };
-        assert!(bounds.validate().is_ok());
-        assert!(!bounds.is_canonical());
-        assert_eq!(bounds.start.to_offset(), Some(10));
-        assert_eq!(bounds.end.to_offset(), Some(1));
-        assert_eq!(bounds.n_preceding_rows(), Some(0));
-        assert_eq!(bounds.n_following_rows(), Some(10));
-
-        let bounds = RowsFrameBounds {
-            start: UnboundedFollowing,
-            end: Following(10),
-        };
-        assert!(bounds.validate().is_err());
-        assert!(!bounds.is_canonical());
-
-        let bounds = RowsFrameBounds {
-            start: Preceding(10),
-            end: UnboundedPreceding,
-        };
-        assert!(bounds.validate().is_err());
-        assert!(!bounds.is_canonical());
-    }
-}
diff --git a/src/expr/core/src/window_function/mod.rs b/src/expr/core/src/window_function/mod.rs
index dd735d086fb12..6bbde8c8755e6 100644
--- a/src/expr/core/src/window_function/mod.rs
+++ b/src/expr/core/src/window_function/mod.rs
@@ -17,6 +17,12 @@ pub use kind::*;
 
 mod call;
 pub use call::*;
+mod rows;
+pub use rows::*;
+mod range;
+pub use range::*;
+mod session;
+pub use session::*;
 
 mod state;
 pub use state::*;
diff --git a/src/expr/core/src/window_function/range.rs b/src/expr/core/src/window_function/range.rs
new file mode 100644
index 0000000000000..922e247dd8369
--- /dev/null
+++ b/src/expr/core/src/window_function/range.rs
@@ -0,0 +1,401 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Display;
+use std::ops::Deref;
+use std::sync::Arc;
+
+use anyhow::Context;
+use educe::Educe;
+use futures_util::FutureExt;
+use risingwave_common::bail;
+use risingwave_common::row::OwnedRow;
+use risingwave_common::types::{
+    DataType, Datum, IsNegative, ScalarImpl, ScalarRefImpl, Sentinelled, ToOwnedDatum, ToText,
+};
+use risingwave_common::util::sort_util::{Direction, OrderType};
+use risingwave_common::util::value_encoding::{DatumFromProtoExt, DatumToProtoExt};
+use risingwave_pb::expr::window_frame::{PbBoundType, PbRangeFrameBound, PbRangeFrameBounds};
+
+use super::FrameBound::{
+    self, CurrentRow, Following, Preceding, UnboundedFollowing, UnboundedPreceding,
+};
+use super::FrameBoundsImpl;
+use crate::expr::{
+    build_func, BoxedExpression, Expression, ExpressionBoxExt, InputRefExpression,
+    LiteralExpression,
+};
+use crate::Result;
+
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct RangeFrameBounds {
+    pub order_data_type: DataType,
+    pub order_type: OrderType,
+    pub offset_data_type: DataType,
+    pub start: RangeFrameBound,
+    pub end: RangeFrameBound,
+}
+
+impl RangeFrameBounds {
+    pub(super) fn from_protobuf(bounds: &PbRangeFrameBounds) -> Result<Self> {
+        let order_data_type = DataType::from(bounds.get_order_data_type()?);
+        let order_type = OrderType::from_protobuf(bounds.get_order_type()?);
+        let offset_data_type = DataType::from(bounds.get_offset_data_type()?);
+        let start = FrameBound::<RangeFrameOffset>::from_protobuf(
+            bounds.get_start()?,
+            &order_data_type,
+            &offset_data_type,
+        )?;
+        let end = FrameBound::<RangeFrameOffset>::from_protobuf(
+            bounds.get_end()?,
+            &order_data_type,
+            &offset_data_type,
+        )?;
+        Ok(Self {
+            order_data_type,
+            order_type,
+            offset_data_type,
+            start,
+            end,
+        })
+    }
+
+    pub(super) fn to_protobuf(&self) -> PbRangeFrameBounds {
+        PbRangeFrameBounds {
+            start: Some(self.start.to_protobuf()),
+            end: Some(self.end.to_protobuf()),
+            order_data_type: Some(self.order_data_type.to_protobuf()),
+            order_type: Some(self.order_type.to_protobuf()),
+            offset_data_type: Some(self.offset_data_type.to_protobuf()),
+        }
+    }
+}
+
+impl Display for RangeFrameBounds {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "RANGE BETWEEN {} AND {}",
+            self.start.for_display(),
+            self.end.for_display()
+        )?;
+        Ok(())
+    }
+}
+
+impl FrameBoundsImpl for RangeFrameBounds {
+    fn validate(&self) -> Result<()> {
+        fn validate_non_negative(val: impl IsNegative + Display) -> Result<()> {
+            if val.is_negative() {
+                bail!(
+                    "frame bound offset should be non-negative, but {} is given",
+                    val
+                );
+            }
+            Ok(())
+        }
+
+        FrameBound::validate_bounds(&self.start, &self.end, |offset| {
+            match offset.as_scalar_ref_impl() {
+                // TODO(rc): use decl macro?
+                ScalarRefImpl::Int16(val) => validate_non_negative(val)?,
+                ScalarRefImpl::Int32(val) => validate_non_negative(val)?,
+                ScalarRefImpl::Int64(val) => validate_non_negative(val)?,
+                ScalarRefImpl::Float32(val) => validate_non_negative(val)?,
+                ScalarRefImpl::Float64(val) => validate_non_negative(val)?,
+                ScalarRefImpl::Decimal(val) => validate_non_negative(val)?,
+                ScalarRefImpl::Interval(val) => {
+                    if !val.is_never_negative() {
+                        bail!(
+                            "for frame bound offset of type `interval`, each field should be non-negative, but {} is given",
+                            val
+                        );
+                    }
+                    if matches!(self.order_data_type, DataType::Timestamptz) {
+                        // for `timestamptz`, we only support offset without `month` and `day` fields
+                        if val.months() != 0 || val.days() != 0 {
+                            bail!(
+                                "for frame order column of type `timestamptz`, offset should not have non-zero `month` and `day`",
+                            );
+                        }
+                    }
+                },
+                _ => unreachable!("other order column data types are not supported and should be banned in frontend"),
+            }
+            Ok(())
+        })
+    }
+}
+
+impl RangeFrameBounds {
+    /// Get the frame start for a given order column value.
+    ///
+    /// ## Examples
+    ///
+    /// For the following frames:
+    ///
+    /// ```sql
+    /// ORDER BY x ASC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+    /// ORDER BY x DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+    /// ```
+    ///
+    /// For any CURRENT ROW with any order value, the frame start is always the first-most row, which is
+    /// represented by [`Sentinelled::Smallest`].
+    ///
+    /// For the following frame:
+    ///
+    /// ```sql
+    /// ORDER BY x ASC RANGE BETWEEN 10 PRECEDING AND CURRENT ROW
+    /// ```
+    ///
+    /// For CURRENT ROW with order value `100`, the frame start is the **FIRST** row with order value `90`.
+    ///
+    /// For the following frame:
+    ///
+    /// ```sql
+    /// ORDER BY x DESC RANGE BETWEEN 10 PRECEDING AND CURRENT ROW
+    /// ```
+    ///
+    /// For CURRENT ROW with order value `100`, the frame start is the **FIRST** row with order value `110`.
+    pub fn frame_start_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled<Datum> {
+        self.start.for_calc().bound_of(order_value, self.order_type)
+    }
+
+    /// Get the frame end for a given order column value. It's very similar to `frame_start_of`, just with
+    /// everything on the other direction.
+    pub fn frame_end_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled<Datum> {
+        self.end.for_calc().bound_of(order_value, self.order_type)
+    }
+
+    /// Get the order value of the CURRENT ROW of the first frame that includes the given order value.
+    ///
+    /// ## Examples
+    ///
+    /// For the following frames:
+    ///
+    /// ```sql
+    /// ORDER BY x ASC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+    /// ORDER BY x DESC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
+    /// ```
+    ///
+    /// For any given order value, the first CURRENT ROW is always the first-most row, which is
+    /// represented by [`Sentinelled::Smallest`].
+    ///
+    /// For the following frame:
+    ///
+    /// ```sql
+    /// ORDER BY x ASC RANGE BETWEEN CURRENT ROW AND 10 FOLLOWING
+    /// ```
+    ///
+    /// For a given order value `100`, the first CURRENT ROW should have order value `90`.
+    ///
+    /// For the following frame:
+    ///
+    /// ```sql
+    /// ORDER BY x DESC RANGE BETWEEN CURRENT ROW AND 10 FOLLOWING
+    /// ```
+    ///
+    /// For a given order value `100`, the first CURRENT ROW should have order value `110`.
+    pub fn first_curr_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled<Datum> {
+        self.end
+            .for_calc()
+            .reverse()
+            .bound_of(order_value, self.order_type)
+    }
+
+    /// Get the order value of the CURRENT ROW of the last frame that includes the given order value.
+    /// It's very similar to `first_curr_of`, just with everything on the other direction.
+    pub fn last_curr_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled<Datum> {
+        self.start
+            .for_calc()
+            .reverse()
+            .bound_of(order_value, self.order_type)
+    }
+}
+
+pub type RangeFrameBound = FrameBound<RangeFrameOffset>;
+
+impl RangeFrameBound {
+    fn from_protobuf(
+        bound: &PbRangeFrameBound,
+        order_data_type: &DataType,
+        offset_data_type: &DataType,
+    ) -> Result<Self> {
+        let bound = match bound.get_type()? {
+            PbBoundType::Unspecified => bail!("unspecified type of `RangeFrameBound`"),
+            PbBoundType::UnboundedPreceding => Self::UnboundedPreceding,
+            PbBoundType::CurrentRow => Self::CurrentRow,
+            PbBoundType::UnboundedFollowing => Self::UnboundedFollowing,
+            bound_type @ (PbBoundType::Preceding | PbBoundType::Following) => {
+                let offset_value = Datum::from_protobuf(bound.get_offset()?, offset_data_type)
+                    .context("offset `Datum` is not decodable")?
+                    .context("offset of `RangeFrameBound` must be non-NULL")?;
+                let mut offset = RangeFrameOffset::new(offset_value);
+                offset.prepare(order_data_type, offset_data_type)?;
+                if bound_type == PbBoundType::Preceding {
+                    Self::Preceding(offset)
+                } else {
+                    Self::Following(offset)
+                }
+            }
+        };
+        Ok(bound)
+    }
+
+    fn to_protobuf(&self) -> PbRangeFrameBound {
+        let (r#type, offset) = match self {
+            Self::UnboundedPreceding => (PbBoundType::UnboundedPreceding, None),
+            Self::Preceding(offset) => (
+                PbBoundType::Preceding,
+                Some(Some(offset.as_scalar_ref_impl()).to_protobuf()),
+            ),
+            Self::CurrentRow => (PbBoundType::CurrentRow, None),
+            Self::Following(offset) => (
+                PbBoundType::Following,
+                Some(Some(offset.as_scalar_ref_impl()).to_protobuf()),
+            ),
+            Self::UnboundedFollowing => (PbBoundType::UnboundedFollowing, None),
+        };
+        PbRangeFrameBound {
+            r#type: r#type as _,
+            offset,
+        }
+    }
+}
+
+impl RangeFrameBound {
+    fn for_display(&self) -> FrameBound<String> {
+        match self {
+            UnboundedPreceding => UnboundedPreceding,
+            Preceding(offset) => Preceding(offset.as_scalar_ref_impl().to_text()),
+            CurrentRow => CurrentRow,
+            Following(offset) => Following(offset.as_scalar_ref_impl().to_text()),
+            UnboundedFollowing => UnboundedFollowing,
+        }
+    }
+
+    fn for_calc(&self) -> FrameBound<RangeFrameOffsetRef<'_>> {
+        match self {
+            UnboundedPreceding => UnboundedPreceding,
+            Preceding(offset) => Preceding(RangeFrameOffsetRef {
+                add_expr: offset.add_expr.as_ref().unwrap().as_ref(),
+                sub_expr: offset.sub_expr.as_ref().unwrap().as_ref(),
+            }),
+            CurrentRow => CurrentRow,
+            Following(offset) => Following(RangeFrameOffsetRef {
+                add_expr: offset.add_expr.as_ref().unwrap().as_ref(),
+                sub_expr: offset.sub_expr.as_ref().unwrap().as_ref(),
+            }),
+            UnboundedFollowing => UnboundedFollowing,
+        }
+    }
+}
+
+/// The wrapper type for [`ScalarImpl`] range frame offset, containing
+/// two expressions to help adding and subtracting the offset.
+#[derive(Debug, Clone, Educe)]
+#[educe(PartialEq, Eq, Hash)]
+pub struct RangeFrameOffset {
+    /// The original offset value.
+    offset: ScalarImpl,
+    /// Built expression for `$0 + offset`.
+    #[educe(PartialEq(ignore), Hash(ignore))]
+    add_expr: Option<Arc<BoxedExpression>>,
+    /// Built expression for `$0 - offset`.
+    #[educe(PartialEq(ignore), Hash(ignore))]
+    sub_expr: Option<Arc<BoxedExpression>>,
+}
+
+impl RangeFrameOffset {
+    pub fn new(offset: ScalarImpl) -> Self {
+        Self {
+            offset,
+            add_expr: None,
+            sub_expr: None,
+        }
+    }
+
+    fn prepare(&mut self, order_data_type: &DataType, offset_data_type: &DataType) -> Result<()> {
+        use risingwave_pb::expr::expr_node::PbType as PbExprType;
+
+        let input_expr = InputRefExpression::new(order_data_type.clone(), 0);
+        let offset_expr =
+            LiteralExpression::new(offset_data_type.clone(), Some(self.offset.clone()));
+        self.add_expr = Some(Arc::new(build_func(
+            PbExprType::Add,
+            order_data_type.clone(),
+            vec![input_expr.clone().boxed(), offset_expr.clone().boxed()],
+        )?));
+        self.sub_expr = Some(Arc::new(build_func(
+            PbExprType::Subtract,
+            order_data_type.clone(),
+            vec![input_expr.boxed(), offset_expr.boxed()],
+        )?));
+        Ok(())
+    }
+
+    pub fn new_for_test(
+        offset: ScalarImpl,
+        order_data_type: &DataType,
+        offset_data_type: &DataType,
+    ) -> Self {
+        let mut offset = Self::new(offset);
+        offset.prepare(order_data_type, offset_data_type).unwrap();
+        offset
+    }
+}
+
+impl Deref for RangeFrameOffset {
+    type Target = ScalarImpl;
+
+    fn deref(&self) -> &Self::Target {
+        &self.offset
+    }
+}
+
+#[derive(Debug, Educe)]
+#[educe(Clone, Copy)]
+struct RangeFrameOffsetRef<'a> {
+    /// Built expression for `$0 + offset`.
+    add_expr: &'a dyn Expression,
+    /// Built expression for `$0 - offset`.
+    sub_expr: &'a dyn Expression,
+}
+
+impl FrameBound<RangeFrameOffsetRef<'_>> {
+    fn bound_of(self, order_value: impl ToOwnedDatum, order_type: OrderType) -> Sentinelled<Datum> {
+        let expr = match (self, order_type.direction()) {
+            (UnboundedPreceding, _) => return Sentinelled::Smallest,
+            (UnboundedFollowing, _) => return Sentinelled::Largest,
+            (CurrentRow, _) => return Sentinelled::Normal(order_value.to_owned_datum()),
+            (Preceding(offset), Direction::Ascending)
+            | (Following(offset), Direction::Descending) => {
+                // should SUBTRACT the offset
+                offset.sub_expr
+            }
+            (Following(offset), Direction::Ascending)
+            | (Preceding(offset), Direction::Descending) => {
+                // should ADD the offset
+                offset.add_expr
+            }
+        };
+        let row = OwnedRow::new(vec![order_value.to_owned_datum()]);
+        Sentinelled::Normal(
+            expr.eval_row(&row)
+                .now_or_never()
+                .expect("frame bound calculation should finish immediately")
+                .expect("just simple calculation, should succeed"), // TODO(rc): handle overflow
+        )
+    }
+}
diff --git a/src/expr/core/src/window_function/rows.rs b/src/expr/core/src/window_function/rows.rs
new file mode 100644
index 0000000000000..61eda77b65ef5
--- /dev/null
+++ b/src/expr/core/src/window_function/rows.rs
@@ -0,0 +1,234 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use parse_display::Display;
+use risingwave_common::bail;
+use risingwave_pb::expr::window_frame::{
+    PbBound, PbBoundType, PbRowsFrameBound, PbRowsFrameBounds,
+};
+
+use super::FrameBound::{
+    self, CurrentRow, Following, Preceding, UnboundedFollowing, UnboundedPreceding,
+};
+use super::FrameBoundsImpl;
+use crate::Result;
+
+#[derive(Display, Debug, Clone, Eq, PartialEq, Hash)]
+#[display("ROWS BETWEEN {start} AND {end}")]
+pub struct RowsFrameBounds {
+    pub start: RowsFrameBound,
+    pub end: RowsFrameBound,
+}
+
+impl RowsFrameBounds {
+    pub(super) fn from_protobuf(bounds: &PbRowsFrameBounds) -> Result<Self> {
+        let start = FrameBound::<usize>::from_protobuf(bounds.get_start()?)?;
+        let end = FrameBound::<usize>::from_protobuf(bounds.get_end()?)?;
+        Ok(Self { start, end })
+    }
+
+    pub(super) fn to_protobuf(&self) -> PbRowsFrameBounds {
+        PbRowsFrameBounds {
+            start: Some(self.start.to_protobuf()),
+            end: Some(self.end.to_protobuf()),
+        }
+    }
+}
+
+impl RowsFrameBounds {
+    /// Check if the `ROWS` frame is canonical.
+    ///
+    /// A canonical `ROWS` frame is defined as:
+    ///
+    /// - Its bounds are valid (see [`Self::validate`]).
+    /// - It contains the current row.
+    pub fn is_canonical(&self) -> bool {
+        self.validate().is_ok() && {
+            let start = self.start.to_offset();
+            let end = self.end.to_offset();
+            start.unwrap_or(0) <= 0 && end.unwrap_or(0) >= 0
+        }
+    }
+
+    /// Get the number of preceding rows.
+    pub fn n_preceding_rows(&self) -> Option<usize> {
+        match (&self.start, &self.end) {
+            (UnboundedPreceding, _) => None,
+            (Preceding(n1), Preceding(n2)) => Some(*n1.max(n2)),
+            (Preceding(n), _) => Some(*n),
+            (CurrentRow | Following(_) | UnboundedFollowing, _) => Some(0),
+        }
+    }
+
+    /// Get the number of following rows.
+    pub fn n_following_rows(&self) -> Option<usize> {
+        match (&self.start, &self.end) {
+            (_, UnboundedFollowing) => None,
+            (Following(n1), Following(n2)) => Some(*n1.max(n2)),
+            (_, Following(n)) => Some(*n),
+            (_, CurrentRow | Preceding(_) | UnboundedPreceding) => Some(0),
+        }
+    }
+}
+
+impl FrameBoundsImpl for RowsFrameBounds {
+    fn validate(&self) -> Result<()> {
+        FrameBound::validate_bounds(&self.start, &self.end, |_| Ok(()))
+    }
+}
+
+pub type RowsFrameBound = FrameBound<usize>;
+
+impl RowsFrameBound {
+    pub(super) fn from_protobuf_legacy(bound: &PbBound) -> Result<Self> {
+        use risingwave_pb::expr::window_frame::bound::PbOffset;
+
+        let offset = bound.get_offset()?;
+        let bound = match offset {
+            PbOffset::Integer(offset) => Self::from_protobuf(&PbRowsFrameBound {
+                r#type: bound.get_type()? as _,
+                offset: Some(*offset),
+            })?,
+            PbOffset::Datum(_) => bail!("offset of `RowsFrameBound` must be `Integer`"),
+        };
+        Ok(bound)
+    }
+
+    fn from_protobuf(bound: &PbRowsFrameBound) -> Result<Self> {
+        let bound = match bound.get_type()? {
+            PbBoundType::Unspecified => bail!("unspecified type of `RowsFrameBound`"),
+            PbBoundType::UnboundedPreceding => Self::UnboundedPreceding,
+            PbBoundType::Preceding => Self::Preceding(*bound.get_offset()? as usize),
+            PbBoundType::CurrentRow => Self::CurrentRow,
+            PbBoundType::Following => Self::Following(*bound.get_offset()? as usize),
+            PbBoundType::UnboundedFollowing => Self::UnboundedFollowing,
+        };
+        Ok(bound)
+    }
+
+    fn to_protobuf(&self) -> PbRowsFrameBound {
+        let (r#type, offset) = match self {
+            Self::UnboundedPreceding => (PbBoundType::UnboundedPreceding, None),
+            Self::Preceding(offset) => (PbBoundType::Preceding, Some(*offset as _)),
+            Self::CurrentRow => (PbBoundType::CurrentRow, None),
+            Self::Following(offset) => (PbBoundType::Following, Some(*offset as _)),
+            Self::UnboundedFollowing => (PbBoundType::UnboundedFollowing, None),
+        };
+        PbRowsFrameBound {
+            r#type: r#type as _,
+            offset,
+        }
+    }
+}
+
+impl RowsFrameBound {
+    /// Convert the bound to sized offset from current row. `None` if the bound is unbounded.
+    pub fn to_offset(&self) -> Option<isize> {
+        match self {
+            UnboundedPreceding | UnboundedFollowing => None,
+            CurrentRow => Some(0),
+            Preceding(n) => Some(-(*n as isize)),
+            Following(n) => Some(*n as isize),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+
+    #[test]
+    fn test_rows_frame_bounds() {
+        let bounds = RowsFrameBounds {
+            start: Preceding(1),
+            end: CurrentRow,
+        };
+        assert!(bounds.validate().is_ok());
+        assert!(bounds.is_canonical());
+        assert_eq!(bounds.start.to_offset(), Some(-1));
+        assert_eq!(bounds.end.to_offset(), Some(0));
+        assert_eq!(bounds.n_preceding_rows(), Some(1));
+        assert_eq!(bounds.n_following_rows(), Some(0));
+
+        let bounds = RowsFrameBounds {
+            start: CurrentRow,
+            end: Following(1),
+        };
+        assert!(bounds.validate().is_ok());
+        assert!(bounds.is_canonical());
+        assert_eq!(bounds.start.to_offset(), Some(0));
+        assert_eq!(bounds.end.to_offset(), Some(1));
+        assert_eq!(bounds.n_preceding_rows(), Some(0));
+        assert_eq!(bounds.n_following_rows(), Some(1));
+
+        let bounds = RowsFrameBounds {
+            start: UnboundedPreceding,
+            end: Following(10),
+        };
+        assert!(bounds.validate().is_ok());
+        assert!(bounds.is_canonical());
+        assert_eq!(bounds.start.to_offset(), None);
+        assert_eq!(bounds.end.to_offset(), Some(10));
+        assert_eq!(bounds.n_preceding_rows(), None);
+        assert_eq!(bounds.n_following_rows(), Some(10));
+
+        let bounds = RowsFrameBounds {
+            start: Preceding(10),
+            end: UnboundedFollowing,
+        };
+        assert!(bounds.validate().is_ok());
+        assert!(bounds.is_canonical());
+        assert_eq!(bounds.start.to_offset(), Some(-10));
+        assert_eq!(bounds.end.to_offset(), None);
+        assert_eq!(bounds.n_preceding_rows(), Some(10));
+        assert_eq!(bounds.n_following_rows(), None);
+
+        let bounds = RowsFrameBounds {
+            start: Preceding(1),
+            end: Preceding(10),
+        };
+        assert!(bounds.validate().is_ok());
+        assert!(!bounds.is_canonical());
+        assert_eq!(bounds.start.to_offset(), Some(-1));
+        assert_eq!(bounds.end.to_offset(), Some(-10));
+        assert_eq!(bounds.n_preceding_rows(), Some(10));
+        assert_eq!(bounds.n_following_rows(), Some(0));
+
+        let bounds = RowsFrameBounds {
+            start: Following(10),
+            end: Following(1),
+        };
+        assert!(bounds.validate().is_ok());
+        assert!(!bounds.is_canonical());
+        assert_eq!(bounds.start.to_offset(), Some(10));
+        assert_eq!(bounds.end.to_offset(), Some(1));
+        assert_eq!(bounds.n_preceding_rows(), Some(0));
+        assert_eq!(bounds.n_following_rows(), Some(10));
+
+        let bounds = RowsFrameBounds {
+            start: UnboundedFollowing,
+            end: Following(10),
+        };
+        assert!(bounds.validate().is_err());
+        assert!(!bounds.is_canonical());
+
+        let bounds = RowsFrameBounds {
+            start: Preceding(10),
+            end: UnboundedPreceding,
+        };
+        assert!(bounds.validate().is_err());
+        assert!(!bounds.is_canonical());
+    }
+}
diff --git a/src/expr/core/src/window_function/session.rs b/src/expr/core/src/window_function/session.rs
new file mode 100644
index 0000000000000..81a77058759bf
--- /dev/null
+++ b/src/expr/core/src/window_function/session.rs
@@ -0,0 +1,208 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Display;
+use std::ops::Deref;
+use std::sync::Arc;
+
+use anyhow::Context;
+use educe::Educe;
+use futures::FutureExt;
+use risingwave_common::bail;
+use risingwave_common::row::OwnedRow;
+use risingwave_common::types::{
+    DataType, Datum, IsNegative, ScalarImpl, ScalarRefImpl, ToOwnedDatum, ToText,
+};
+use risingwave_common::util::sort_util::OrderType;
+use risingwave_common::util::value_encoding::{DatumFromProtoExt, DatumToProtoExt};
+use risingwave_pb::expr::window_frame::PbSessionFrameBounds;
+
+use super::FrameBoundsImpl;
+use crate::expr::{
+    build_func, BoxedExpression, Expression, ExpressionBoxExt, InputRefExpression,
+    LiteralExpression,
+};
+use crate::Result;
+
+/// To implement Session Window in a similar way to Range Frame, we define a similar frame bounds
+/// structure here. It's very like [`RangeFrameBounds`](super::RangeFrameBounds), but with a gap
+/// instead of start & end offset.
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct SessionFrameBounds {
+    pub order_data_type: DataType,
+    pub order_type: OrderType,
+    pub gap_data_type: DataType,
+    pub gap: SessionFrameGap,
+}
+
+impl SessionFrameBounds {
+    pub(super) fn from_protobuf(bounds: &PbSessionFrameBounds) -> Result<Self> {
+        let order_data_type = DataType::from(bounds.get_order_data_type()?);
+        let order_type = OrderType::from_protobuf(bounds.get_order_type()?);
+        let gap_data_type = DataType::from(bounds.get_gap_data_type()?);
+        let gap_value = Datum::from_protobuf(bounds.get_gap()?, &gap_data_type)
+            .context("gap `Datum` is not decodable")?
+            .context("gap of session frame must be non-NULL")?;
+        let mut gap = SessionFrameGap::new(gap_value);
+        gap.prepare(&order_data_type, &gap_data_type)?;
+        Ok(Self {
+            order_data_type,
+            order_type,
+            gap_data_type,
+            gap,
+        })
+    }
+
+    pub(super) fn to_protobuf(&self) -> PbSessionFrameBounds {
+        PbSessionFrameBounds {
+            gap: Some(Some(self.gap.as_scalar_ref_impl()).to_protobuf()),
+            order_data_type: Some(self.order_data_type.to_protobuf()),
+            order_type: Some(self.order_type.to_protobuf()),
+            gap_data_type: Some(self.gap_data_type.to_protobuf()),
+        }
+    }
+}
+
+impl Display for SessionFrameBounds {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "SESSION WITH GAP {}",
+            self.gap.as_scalar_ref_impl().to_text()
+        )
+    }
+}
+
+impl FrameBoundsImpl for SessionFrameBounds {
+    fn validate(&self) -> Result<()> {
+        // TODO(rc): maybe can merge with `RangeFrameBounds::validate`
+
+        fn validate_non_negative(val: impl IsNegative + Display) -> Result<()> {
+            if val.is_negative() {
+                bail!("session gap should be non-negative, but {} is given", val);
+            }
+            Ok(())
+        }
+
+        match self.gap.as_scalar_ref_impl() {
+            ScalarRefImpl::Int16(val) => validate_non_negative(val)?,
+            ScalarRefImpl::Int32(val) => validate_non_negative(val)?,
+            ScalarRefImpl::Int64(val) => validate_non_negative(val)?,
+            ScalarRefImpl::Float32(val) => validate_non_negative(val)?,
+            ScalarRefImpl::Float64(val) => validate_non_negative(val)?,
+            ScalarRefImpl::Decimal(val) => validate_non_negative(val)?,
+            ScalarRefImpl::Interval(val) => {
+                if !val.is_never_negative() {
+                    bail!(
+                        "for session gap of type `interval`, each field should be non-negative, but {} is given",
+                        val
+                    );
+                }
+                if matches!(self.order_data_type, DataType::Timestamptz) {
+                    // for `timestamptz`, we only support gap without `month` and `day` fields
+                    if val.months() != 0 || val.days() != 0 {
+                        bail!(
+                            "for session order column of type `timestamptz`, gap should not have non-zero `month` and `day`",
+                        );
+                    }
+                }
+            }
+            _ => unreachable!(
+                "other order column data types are not supported and should be banned in frontend"
+            ),
+        }
+        Ok(())
+    }
+}
+
+impl SessionFrameBounds {
+    pub fn minimal_next_start_of(&self, end_order_value: impl ToOwnedDatum) -> Datum {
+        self.gap.for_calc().minimal_next_start_of(end_order_value)
+    }
+}
+
+/// The wrapper type for [`ScalarImpl`] session gap, containing an expression to help adding the gap
+/// to a given value.
+#[derive(Debug, Clone, Educe)]
+#[educe(PartialEq, Eq, Hash)]
+pub struct SessionFrameGap {
+    /// The original gap value.
+    gap: ScalarImpl,
+    /// Built expression for `$0 + gap`.
+    #[educe(PartialEq(ignore), Hash(ignore))]
+    add_expr: Option<Arc<BoxedExpression>>,
+}
+
+impl Deref for SessionFrameGap {
+    type Target = ScalarImpl;
+
+    fn deref(&self) -> &Self::Target {
+        &self.gap
+    }
+}
+
+impl SessionFrameGap {
+    pub fn new(gap: ScalarImpl) -> Self {
+        Self {
+            gap,
+            add_expr: None,
+        }
+    }
+
+    fn prepare(&mut self, order_data_type: &DataType, gap_data_type: &DataType) -> Result<()> {
+        use risingwave_pb::expr::expr_node::PbType as PbExprType;
+
+        let input_expr = InputRefExpression::new(order_data_type.clone(), 0);
+        let gap_expr = LiteralExpression::new(gap_data_type.clone(), Some(self.gap.clone()));
+        self.add_expr = Some(Arc::new(build_func(
+            PbExprType::Add,
+            order_data_type.clone(),
+            vec![input_expr.clone().boxed(), gap_expr.clone().boxed()],
+        )?));
+        Ok(())
+    }
+
+    pub fn new_for_test(
+        gap: ScalarImpl,
+        order_data_type: &DataType,
+        gap_data_type: &DataType,
+    ) -> Self {
+        let mut gap = Self::new(gap);
+        gap.prepare(order_data_type, gap_data_type).unwrap();
+        gap
+    }
+
+    fn for_calc(&self) -> SessionFrameGapRef<'_> {
+        SessionFrameGapRef {
+            add_expr: self.add_expr.as_ref().unwrap().as_ref(),
+        }
+    }
+}
+
+#[derive(Debug, Educe)]
+#[educe(Clone, Copy)]
+struct SessionFrameGapRef<'a> {
+    add_expr: &'a dyn Expression,
+}
+
+impl<'a> SessionFrameGapRef<'a> {
+    fn minimal_next_start_of(&self, end_order_value: impl ToOwnedDatum) -> Datum {
+        let row = OwnedRow::new(vec![end_order_value.to_owned_datum()]);
+        self.add_expr
+            .eval_row(&row)
+            .now_or_never()
+            .expect("frame bound calculation should finish immediately")
+            .expect("just simple calculation, should succeed") // TODO(rc): handle overflow
+    }
+}
diff --git a/src/expr/core/src/window_function/state/mod.rs b/src/expr/core/src/window_function/state.rs
similarity index 87%
rename from src/expr/core/src/window_function/state/mod.rs
rename to src/expr/core/src/window_function/state.rs
index 6a86667bfcafc..ec5bfc70d8591 100644
--- a/src/expr/core/src/window_function/state/mod.rs
+++ b/src/expr/core/src/window_function/state.rs
@@ -21,14 +21,9 @@ use risingwave_common::util::memcmp_encoding::MemcmpEncoded;
 use risingwave_common_estimate_size::EstimateSize;
 use smallvec::SmallVec;
 
-use super::{WindowFuncCall, WindowFuncKind};
+use super::WindowFuncCall;
 use crate::{ExprError, Result};
 
-mod aggregate;
-mod buffer;
-mod range_utils;
-mod rank;
-
 /// Unique and ordered identifier for a row in internal states.
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, EstimateSize)]
 pub struct StateKey {
@@ -110,22 +105,21 @@ pub trait WindowState: EstimateSize {
 
 pub type BoxedWindowState = Box<dyn WindowState + Send + Sync>;
 
-pub fn create_window_state(call: &WindowFuncCall) -> Result<BoxedWindowState> {
-    assert!(call.frame.bounds.validate().is_ok());
+#[linkme::distributed_slice]
+pub static WINDOW_STATE_BUILDERS: [fn(&WindowFuncCall) -> Result<BoxedWindowState>];
 
-    use WindowFuncKind::*;
-    Ok(match call.kind {
-        RowNumber => Box::new(rank::RankState::<rank::RowNumber>::new(call)),
-        Rank => Box::new(rank::RankState::<rank::Rank>::new(call)),
-        DenseRank => Box::new(rank::RankState::<rank::DenseRank>::new(call)),
-        Aggregate(_) => aggregate::new(call)?,
-        kind => {
-            return Err(ExprError::UnsupportedFunction(format!(
+pub fn create_window_state(call: &WindowFuncCall) -> Result<BoxedWindowState> {
+    // we expect only one builder function in `expr_impl/window_function/mod.rs`
+    let builder = WINDOW_STATE_BUILDERS.iter().next();
+    builder.map_or_else(
+        || {
+            Err(ExprError::UnsupportedFunction(format!(
                 "{}({}) -> {}",
-                kind,
+                call.kind,
                 call.args.arg_types().iter().format(", "),
                 &call.return_type,
-            )));
-        }
-    })
+            )))
+        },
+        |f| f(call),
+    )
 }
diff --git a/src/expr/impl/Cargo.toml b/src/expr/impl/Cargo.toml
index 082f33de2da4e..6dfbcc5905750 100644
--- a/src/expr/impl/Cargo.toml
+++ b/src/expr/impl/Cargo.toml
@@ -15,44 +15,65 @@ ignored = ["workspace-hack", "ctor"]
 [package.metadata.cargo-udeps.ignore]
 normal = ["workspace-hack", "ctor"]
 
+[features]
+external-udf = ["arrow-udf-flight", "arrow-flight", "tonic"]
+js-udf = ["arrow-udf-js"]
+deno-udf = ["arrow-udf-js-deno", "zstd"]
+python-udf = ["arrow-udf-python"]
+wasm-udf = ["arrow-udf-wasm", "zstd"]
+
 [dependencies]
 aho-corasick = "1"
 anyhow = "1"
+arrow-array = { workspace = true }
+arrow-flight = { workspace = true, optional = true }
 arrow-schema = { workspace = true }
+arrow-udf-flight = { workspace = true, optional = true }
+arrow-udf-js = { workspace = true, optional = true }
+arrow-udf-js-deno = { workspace = true, optional = true }
+arrow-udf-python = { workspace = true, optional = true }
+arrow-udf-wasm = { workspace = true, optional = true }
 async-trait = "0.1"
 auto_enums = { workspace = true }
 chrono = { version = "0.4", default-features = false, features = [
     "clock",
     "std",
 ] }
-chrono-tz = { version = "0.8", features = ["case-insensitive"] }
+chrono-tz = { version = "0.9", features = ["case-insensitive"] }
+educe = "0.6"
 fancy-regex = "0.13"
 futures-async-stream = { workspace = true }
 futures-util = "0.3"
+ginepro = "0.7"
 hex = "0.4"
 icelake = { workspace = true }
 itertools = { workspace = true }
-jsonbb = "0.1.2"
+jsonbb = { workspace = true }
 linkme = { version = "0.3", features = ["used_linker"] }
 md5 = "0.7"
+moka = { version = "0.12", features = ["sync"] }
 num-traits = "0.2"
-openssl = { version = "0.10", features = ["vendored"] }
+openssl = "0.10"
 regex = "1"
 risingwave_common = { workspace = true }
 risingwave_common_estimate_size = { workspace = true }
 risingwave_expr = { workspace = true }
 risingwave_pb = { workspace = true }
+risingwave_sqlparser = { workspace = true }
 rust_decimal = { version = "1", features = ["db-postgres", "maths"] }
 self_cell = "1.0.1"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 sha1 = "0.10"
 sha2 = "0.10"
+smallvec = "1"
 sql-json-path = { version = "0.1", features = ["jsonbb"] }
 thiserror = "1"
 thiserror-ext = { workspace = true }
 tokio = { version = "0.2", package = "madsim-tokio", features = ["time"] }
+tonic = { version = "0.10", optional = true }
 tracing = "0.1"
+zstd = { version = "0.13", default-features = false, optional = true }
 
 [target.'cfg(not(madsim))'.dependencies]
 workspace-hack = { path = "../../workspace-hack" }
diff --git a/src/expr/impl/benches/expr.rs b/src/expr/impl/benches/expr.rs
index 8468ae86e241b..21036d3649acb 100644
--- a/src/expr/impl/benches/expr.rs
+++ b/src/expr/impl/benches/expr.rs
@@ -26,10 +26,9 @@ use criterion::{criterion_group, criterion_main, Criterion};
 use risingwave_common::array::*;
 use risingwave_common::types::test_utils::IntervalTestExt;
 use risingwave_common::types::*;
-use risingwave_expr::aggregate::{build_append_only, AggArgs, AggCall, AggKind};
+use risingwave_expr::aggregate::{build_append_only, AggCall, AggKind};
 use risingwave_expr::expr::*;
 use risingwave_expr::sig::FUNCTION_REGISTRY;
-use risingwave_expr::ExprError;
 use risingwave_pb::expr::expr_node::PbType;
 use thiserror_ext::AsReport;
 
@@ -397,26 +396,17 @@ fn bench_expr(c: &mut Criterion) {
         }
         let agg = match build_append_only(&AggCall {
             kind: sig.name.as_aggregate(),
-            args: match sig.inputs_type.as_slice() {
-                [] => AggArgs::None,
-                [t] => AggArgs::Unary(t.as_exact().clone(), input_index_for_type(t.as_exact())),
-                [t1, t2] => AggArgs::Binary(
-                    [t1.as_exact().clone(), t2.as_exact().clone()],
-                    [
-                        input_index_for_type(t1.as_exact()),
-                        input_index_for_type(t2.as_exact()),
-                    ],
-                ),
-                _ => {
-                    println!("todo: {sig:?}");
-                    continue;
-                }
-            },
+            args: sig
+                .inputs_type
+                .iter()
+                .map(|t| (t.as_exact().clone(), input_index_for_type(t.as_exact())))
+                .collect(),
             return_type: sig.ret_type.as_exact().clone(),
             column_orders: vec![],
             filter: None,
             distinct: false,
             direct_args: vec![],
+            user_defined: None,
         }) {
             Ok(agg) => agg,
             Err(e) => {
@@ -434,9 +424,10 @@ fn bench_expr(c: &mut Criterion) {
             _ => unreachable!(),
         };
         c.bench_function(&format!("{sig:?}"), |bencher| {
-            bencher
-                .to_async(FuturesExecutor)
-                .iter(|| async { agg.update(&mut agg.create_state(), &input).await.unwrap() })
+            bencher.to_async(FuturesExecutor).iter(|| async {
+                let mut state = agg.create_state().unwrap();
+                agg.update(&mut state, &input).await.unwrap()
+            })
         });
     }
 }
diff --git a/src/expr/impl/src/aggregate/approx_count_distinct/mod.rs b/src/expr/impl/src/aggregate/approx_count_distinct/mod.rs
index bfd1548963d56..a9cea212d3492 100644
--- a/src/expr/impl/src/aggregate/approx_count_distinct/mod.rs
+++ b/src/expr/impl/src/aggregate/approx_count_distinct/mod.rs
@@ -60,8 +60,8 @@ impl AggregateFunction for UpdatableApproxCountDistinct {
         DataType::Int64
     }
 
-    fn create_state(&self) -> AggregateState {
-        AggregateState::Any(Box::<UpdatableRegisters>::default())
+    fn create_state(&self) -> Result<AggregateState> {
+        Ok(AggregateState::Any(Box::<UpdatableRegisters>::default()))
     }
 
     async fn update(&self, state: &mut AggregateState, input: &StreamChunk) -> Result<()> {
@@ -122,8 +122,8 @@ impl AggregateFunction for AppendOnlyApproxCountDistinct {
         DataType::Int64
     }
 
-    fn create_state(&self) -> AggregateState {
-        AggregateState::Any(Box::<AppendOnlyRegisters>::default())
+    fn create_state(&self) -> Result<AggregateState> {
+        Ok(AggregateState::Any(Box::<AppendOnlyRegisters>::default()))
     }
 
     async fn update(&self, state: &mut AggregateState, input: &StreamChunk) -> Result<()> {
@@ -344,7 +344,7 @@ mod tests {
         for range in [0..20000, 20000..30000, 30000..35000] {
             let col = I32Array::from_iter(range.clone()).into_ref();
             let input = StreamChunk::from(DataChunk::new(vec![col], range.len()));
-            let mut state = approx_count_distinct.create_state();
+            let mut state = approx_count_distinct.create_state().unwrap();
             approx_count_distinct
                 .update(&mut state, &input)
                 .now_or_never()
diff --git a/src/expr/impl/src/aggregate/array_agg.rs b/src/expr/impl/src/aggregate/array_agg.rs
index f656ea9b3e614..5ef268877b511 100644
--- a/src/expr/impl/src/aggregate/array_agg.rs
+++ b/src/expr/impl/src/aggregate/array_agg.rs
@@ -64,7 +64,7 @@ mod tests {
             + 789",
         );
         let array_agg = build_append_only(&AggCall::from_pretty("(array_agg:int4[] $0:int4)"))?;
-        let mut state = array_agg.create_state();
+        let mut state = array_agg.create_state()?;
         array_agg.update(&mut state, &chunk).await?;
         let actual = array_agg.get_result(&state).await?;
         assert_eq!(actual, Some(ListValue::from_iter([123, 456, 789]).into()));
@@ -74,7 +74,7 @@ mod tests {
     #[tokio::test]
     async fn test_array_agg_empty() -> Result<()> {
         let array_agg = build_append_only(&AggCall::from_pretty("(array_agg:int4[] $0:int4)"))?;
-        let mut state = array_agg.create_state();
+        let mut state = array_agg.create_state()?;
 
         assert_eq!(array_agg.get_result(&state).await?, None);
 
diff --git a/src/expr/impl/src/aggregate/general.rs b/src/expr/impl/src/aggregate/general.rs
index 2ee55f0a16d52..0c94312335b4b 100644
--- a/src/expr/impl/src/aggregate/general.rs
+++ b/src/expr/impl/src/aggregate/general.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::convert::From;
-
 use num_traits::{CheckedAdd, CheckedSub};
 use risingwave_expr::{aggregate, ExprError, Result};
 
@@ -205,7 +203,7 @@ mod tests {
 
     fn test_agg(pretty: &str, input: StreamChunk, expected: Datum) {
         let agg = build_append_only(&AggCall::from_pretty(pretty)).unwrap();
-        let mut state = agg.create_state();
+        let mut state = agg.create_state().unwrap();
         agg.update(&mut state, &input)
             .now_or_never()
             .unwrap()
@@ -473,7 +471,7 @@ mod tests {
         let chunk = StreamChunk::from_parts(ops, DataChunk::new(vec![Arc::new(data)], vis));
         let pretty = format!("({agg_desc}:int8 $0:int8)");
         let agg = build_append_only(&AggCall::from_pretty(pretty)).unwrap();
-        let mut state = agg.create_state();
+        let mut state = agg.create_state().unwrap();
         b.iter(|| {
             agg.update(&mut state, &chunk)
                 .now_or_never()
diff --git a/src/expr/impl/src/aggregate/mode.rs b/src/expr/impl/src/aggregate/mode.rs
index e5ebdf5a485ad..6e7954553a3e1 100644
--- a/src/expr/impl/src/aggregate/mode.rs
+++ b/src/expr/impl/src/aggregate/mode.rs
@@ -101,8 +101,8 @@ impl AggregateFunction for Mode {
         self.return_type.clone()
     }
 
-    fn create_state(&self) -> AggregateState {
-        AggregateState::Any(Box::<State>::default())
+    fn create_state(&self) -> Result<AggregateState> {
+        Ok(AggregateState::Any(Box::<State>::default()))
     }
 
     async fn update(&self, state: &mut AggregateState, input: &StreamChunk) -> Result<()> {
diff --git a/src/expr/impl/src/aggregate/percentile_cont.rs b/src/expr/impl/src/aggregate/percentile_cont.rs
index 728495edb1122..a60f21a3099fd 100644
--- a/src/expr/impl/src/aggregate/percentile_cont.rs
+++ b/src/expr/impl/src/aggregate/percentile_cont.rs
@@ -91,8 +91,8 @@ impl AggregateFunction for PercentileCont {
         DataType::Float64
     }
 
-    fn create_state(&self) -> AggregateState {
-        AggregateState::Any(Box::<State>::default())
+    fn create_state(&self) -> Result<AggregateState> {
+        Ok(AggregateState::Any(Box::<State>::default()))
     }
 
     async fn update(&self, state: &mut AggregateState, input: &StreamChunk) -> Result<()> {
diff --git a/src/expr/impl/src/aggregate/percentile_disc.rs b/src/expr/impl/src/aggregate/percentile_disc.rs
index 8b0114be1b8d5..23285ab5ef293 100644
--- a/src/expr/impl/src/aggregate/percentile_disc.rs
+++ b/src/expr/impl/src/aggregate/percentile_disc.rs
@@ -116,8 +116,8 @@ impl AggregateFunction for PercentileDisc {
         self.return_type.clone()
     }
 
-    fn create_state(&self) -> AggregateState {
-        AggregateState::Any(Box::<State>::default())
+    fn create_state(&self) -> Result<AggregateState> {
+        Ok(AggregateState::Any(Box::<State>::default()))
     }
 
     async fn update(&self, state: &mut AggregateState, input: &StreamChunk) -> Result<()> {
diff --git a/src/expr/impl/src/aggregate/string_agg.rs b/src/expr/impl/src/aggregate/string_agg.rs
index 32e1888cdfc2b..0195f2b47b6b2 100644
--- a/src/expr/impl/src/aggregate/string_agg.rs
+++ b/src/expr/impl/src/aggregate/string_agg.rs
@@ -48,7 +48,7 @@ mod tests {
         let string_agg = build_append_only(&AggCall::from_pretty(
             "(string_agg:varchar $0:varchar $1:varchar)",
         ))?;
-        let mut state = string_agg.create_state();
+        let mut state = string_agg.create_state()?;
         string_agg.update(&mut state, &chunk).await?;
         assert_eq!(
             string_agg.get_result(&state).await?,
@@ -69,7 +69,7 @@ mod tests {
         let string_agg = build_append_only(&AggCall::from_pretty(
             "(string_agg:varchar $0:varchar $1:varchar)",
         ))?;
-        let mut state = string_agg.create_state();
+        let mut state = string_agg.create_state()?;
         string_agg.update(&mut state, &chunk).await?;
         assert_eq!(
             string_agg.get_result(&state).await?,
diff --git a/src/expr/impl/src/lib.rs b/src/expr/impl/src/lib.rs
index a47754cb4e821..56bdbe3b81100 100644
--- a/src/expr/impl/src/lib.rs
+++ b/src/expr/impl/src/lib.rs
@@ -27,7 +27,6 @@
 #![feature(iterator_try_collect)]
 #![feature(exclusive_range_pattern)]
 #![feature(lazy_cell)]
-#![feature(round_ties_even)]
 #![feature(coroutines)]
 #![feature(test)]
 #![feature(iter_array_chunks)]
@@ -37,6 +36,8 @@
 mod aggregate;
 mod scalar;
 mod table_function;
+mod udf;
+mod window_function;
 
 /// Enable functions in this crate.
 #[macro_export]
diff --git a/src/expr/impl/src/scalar/arithmetic_op.rs b/src/expr/impl/src/scalar/arithmetic_op.rs
index ad5fe8497d8bb..e48b0b5e94b59 100644
--- a/src/expr/impl/src/scalar/arithmetic_op.rs
+++ b/src/expr/impl/src/scalar/arithmetic_op.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::convert::TryInto;
 use std::fmt::Debug;
 
 use chrono::{Duration, NaiveDateTime};
@@ -443,9 +442,7 @@ mod tests {
 
     use num_traits::Float;
     use risingwave_common::types::test_utils::IntervalTestExt;
-    use risingwave_common::types::{
-        Date, Decimal, Int256, Int256Ref, Interval, Scalar, Timestamp, F32, F64,
-    };
+    use risingwave_common::types::{Int256, Int256Ref, Scalar, F32};
 
     use super::*;
 
diff --git a/src/expr/impl/src/scalar/bitwise_op.rs b/src/expr/impl/src/scalar/bitwise_op.rs
index 38aa4d194ea60..0d7f14bb6965a 100644
--- a/src/expr/impl/src/scalar/bitwise_op.rs
+++ b/src/expr/impl/src/scalar/bitwise_op.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 use std::any::type_name;
-use std::convert::TryInto;
 use std::fmt::Debug;
 use std::ops::{BitAnd, BitOr, BitXor, Not};
 
diff --git a/src/expr/impl/src/scalar/case.rs b/src/expr/impl/src/scalar/case.rs
index 99ec8cb880ddc..f7fb9d89ef41b 100644
--- a/src/expr/impl/src/scalar/case.rs
+++ b/src/expr/impl/src/scalar/case.rs
@@ -285,7 +285,6 @@ fn build_case_expr(
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::row::Row;
     use risingwave_common::test_prelude::DataChunkTestExt;
     use risingwave_common::types::ToOwnedDatum;
     use risingwave_common::util::iter_util::ZipEqDebug;
diff --git a/src/expr/impl/src/scalar/cast.rs b/src/expr/impl/src/scalar/cast.rs
index cb990bdcec6a7..48218d7200809 100644
--- a/src/expr/impl/src/scalar/cast.rs
+++ b/src/expr/impl/src/scalar/cast.rs
@@ -244,11 +244,9 @@ fn struct_cast(input: StructRef<'_>, ctx: &Context) -> Result<StructValue> {
 #[cfg(test)]
 mod tests {
     use chrono::NaiveDateTime;
-    use itertools::Itertools;
     use risingwave_common::array::*;
     use risingwave_common::types::*;
     use risingwave_expr::expr::build_from_pretty;
-    use risingwave_pb::expr::expr_node::PbType;
 
     use super::*;
 
diff --git a/src/expr/impl/src/scalar/cmp.rs b/src/expr/impl/src/scalar/cmp.rs
index 69dfcb5f57ef3..02c1662f18025 100644
--- a/src/expr/impl/src/scalar/cmp.rs
+++ b/src/expr/impl/src/scalar/cmp.rs
@@ -519,7 +519,7 @@ mod tests {
     use risingwave_common::array::*;
     use risingwave_common::row::OwnedRow;
     use risingwave_common::types::test_utils::IntervalTestExt;
-    use risingwave_common::types::{Date, Interval, Scalar};
+    use risingwave_common::types::{Date, Interval};
     use risingwave_pb::expr::expr_node::Type;
 
     use crate::scalar::arithmetic_op::{date_interval_add, date_interval_sub};
diff --git a/src/expr/impl/src/scalar/encrypt.rs b/src/expr/impl/src/scalar/encrypt.rs
index 2c7dd89ed980e..5aa2aa7dabd4e 100644
--- a/src/expr/impl/src/scalar/encrypt.rs
+++ b/src/expr/impl/src/scalar/encrypt.rs
@@ -18,7 +18,7 @@ use std::sync::LazyLock;
 use openssl::error::ErrorStack;
 use openssl::symm::{Cipher, Crypter, Mode as CipherMode};
 use regex::Regex;
-use risingwave_expr::{function, CryptographyError, CryptographyStage, ExprError, Result};
+use risingwave_expr::{function, ExprError, Result};
 
 #[derive(Debug, Clone, PartialEq)]
 enum Algorithm {
@@ -129,14 +129,13 @@ impl CipherConfig {
         })
     }
 
-    fn eval(&self, input: &[u8], stage: CryptographyStage) -> Result<Box<[u8]>> {
+    fn eval(&self, input: &[u8], stage: CryptographyStage) -> Result<Box<[u8]>, CryptographyError> {
         let operation = match stage {
             CryptographyStage::Encrypt => CipherMode::Encrypt,
             CryptographyStage::Decrypt => CipherMode::Decrypt,
         };
-        self.eval_inner(input, operation).map_err(|reason| {
-            ExprError::Cryptography(Box::new(CryptographyError { stage, reason }))
-        })
+        self.eval_inner(input, operation)
+            .map_err(|reason| CryptographyError { stage, reason })
     }
 
     fn eval_inner(
@@ -163,7 +162,7 @@ impl CipherConfig {
     "decrypt(bytea, bytea, varchar) -> bytea",
     prebuild = "CipherConfig::parse_cipher_config($1, $2)?"
 )]
-pub fn decrypt(data: &[u8], config: &CipherConfig) -> Result<Box<[u8]>> {
+fn decrypt(data: &[u8], config: &CipherConfig) -> Result<Box<[u8]>, CryptographyError> {
     config.eval(data, CryptographyStage::Decrypt)
 }
 
@@ -171,10 +170,24 @@ pub fn decrypt(data: &[u8], config: &CipherConfig) -> Result<Box<[u8]>> {
     "encrypt(bytea, bytea, varchar) -> bytea",
     prebuild = "CipherConfig::parse_cipher_config($1, $2)?"
 )]
-pub fn encrypt(data: &[u8], config: &CipherConfig) -> Result<Box<[u8]>> {
+fn encrypt(data: &[u8], config: &CipherConfig) -> Result<Box<[u8]>, CryptographyError> {
     config.eval(data, CryptographyStage::Encrypt)
 }
 
+#[derive(Debug)]
+enum CryptographyStage {
+    Encrypt,
+    Decrypt,
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("{stage:?} stage, reason: {reason}")]
+struct CryptographyError {
+    pub stage: CryptographyStage,
+    #[source]
+    pub reason: openssl::error::ErrorStack,
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -197,13 +210,13 @@ mod test {
 
     #[test]
     fn encrypt_testcase() {
-        let encrypt_wrapper = |data: &[u8], key: &[u8], mode: &str| -> Result<Box<[u8]>> {
-            let config = CipherConfig::parse_cipher_config(key, mode)?;
-            encrypt(data, &config)
+        let encrypt_wrapper = |data: &[u8], key: &[u8], mode: &str| -> Box<[u8]> {
+            let config = CipherConfig::parse_cipher_config(key, mode).unwrap();
+            encrypt(data, &config).unwrap()
         };
-        let decrypt_wrapper = |data: &[u8], key: &[u8], mode: &str| -> Result<Box<[u8]>> {
-            let config = CipherConfig::parse_cipher_config(key, mode)?;
-            decrypt(data, &config)
+        let decrypt_wrapper = |data: &[u8], key: &[u8], mode: &str| -> Box<[u8]> {
+            let config = CipherConfig::parse_cipher_config(key, mode).unwrap();
+            decrypt(data, &config).unwrap()
         };
         let key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f";
 
@@ -211,10 +224,9 @@ mod test {
             b"\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff",
             key,
             "aes-ecb/pad:none",
-        )
-        .unwrap();
+        );
 
-        let decrypted = decrypt_wrapper(&encrypted, key, "aes-ecb/pad:none").unwrap();
+        let decrypted = decrypt_wrapper(&encrypted, key, "aes-ecb/pad:none");
         assert_eq!(
             decrypted,
             (*b"\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff").into()
diff --git a/src/expr/impl/src/scalar/external/iceberg.rs b/src/expr/impl/src/scalar/external/iceberg.rs
index cd616aa5e475a..ea39ea7ef989d 100644
--- a/src/expr/impl/src/scalar/external/iceberg.rs
+++ b/src/expr/impl/src/scalar/external/iceberg.rs
@@ -19,10 +19,10 @@ use std::str::FromStr;
 use std::sync::Arc;
 
 use anyhow::anyhow;
-use arrow_schema::DataType as ArrowDataType;
 use icelake::types::{
     create_transform_function, Any as IcelakeDataType, BoxedTransformFunction, Transform,
 };
+use risingwave_common::array::arrow::{FromArrow, IcebergArrowConvert, ToArrow};
 use risingwave_common::array::{ArrayRef, DataChunk};
 use risingwave_common::ensure;
 use risingwave_common::row::OwnedRow;
@@ -34,6 +34,8 @@ use thiserror_ext::AsReport;
 pub struct IcebergTransform {
     child: BoxedExpression,
     transform: BoxedTransformFunction,
+    input_arrow_type: arrow_schema::DataType,
+    output_arrow_field: arrow_schema::Field,
     return_type: DataType,
 }
 
@@ -56,11 +58,13 @@ impl risingwave_expr::expr::Expression for IcebergTransform {
         // Get the child array
         let array = self.child.eval(data_chunk).await?;
         // Convert to arrow array
-        let arrow_array = array.as_ref().try_into().unwrap();
+        let arrow_array = IcebergArrowConvert.to_array(&self.input_arrow_type, &array)?;
         // Transform
         let res_array = self.transform.transform(arrow_array).unwrap();
         // Convert back to array ref and return it
-        Ok(Arc::new((&res_array).try_into().unwrap()))
+        Ok(Arc::new(
+            IcebergArrowConvert.from_array(&self.output_arrow_field, &res_array)?,
+        ))
     }
 
     async fn eval_row(&self, _row: &OwnedRow) -> Result<Datum> {
@@ -91,15 +95,21 @@ fn build(return_type: DataType, mut children: Vec<BoxedExpression>) -> Result<Bo
     // Check type:
     // 1. input type can be transform successfully
     // 2. return type is the same as the result type
-    let input_type = IcelakeDataType::try_from(ArrowDataType::try_from(children[1].return_type())?)
-        .map_err(|err| ExprError::InvalidParam {
+    let input_arrow_type = IcebergArrowConvert
+        .to_arrow_field("", &children[1].return_type())?
+        .data_type()
+        .clone();
+    let output_arrow_field = IcebergArrowConvert.to_arrow_field("", &return_type)?;
+    let input_type = IcelakeDataType::try_from(input_arrow_type.clone()).map_err(|err| {
+        ExprError::InvalidParam {
             name: "input type in iceberg_transform",
             reason: format!(
                 "Failed to convert input type to icelake type, got error: {}",
                 err.as_report()
             )
             .into(),
-        })?;
+        }
+    })?;
     let expect_res_type = transform_type.result_type(&input_type).map_err(
         |err| ExprError::InvalidParam {
             name: "input type in iceberg_transform",
@@ -109,15 +119,20 @@ fn build(return_type: DataType, mut children: Vec<BoxedExpression>) -> Result<Bo
             )
             .into()
         })?;
-    let actual_res_type = IcelakeDataType::try_from(ArrowDataType::try_from(return_type.clone())?)
-        .map_err(|err| ExprError::InvalidParam {
-            name: "return type in iceberg_transform",
-            reason: format!(
-                "Failed to convert return type to icelake type, got error: {}",
-                err.as_report()
-            )
-            .into(),
-        })?;
+    let actual_res_type = IcelakeDataType::try_from(
+        IcebergArrowConvert
+            .to_arrow_field("", &return_type)?
+            .data_type()
+            .clone(),
+    )
+    .map_err(|err| ExprError::InvalidParam {
+        name: "return type in iceberg_transform",
+        reason: format!(
+            "Failed to convert return type to icelake type, got error: {}",
+            err.as_report()
+        )
+        .into(),
+    })?;
     ensure!(
         expect_res_type == actual_res_type,
         ExprError::InvalidParam {
@@ -134,6 +149,8 @@ fn build(return_type: DataType, mut children: Vec<BoxedExpression>) -> Result<Bo
         child: children.remove(1),
         transform: create_transform_function(&transform_type)
             .map_err(|err| ExprError::Internal(err.into()))?,
+        input_arrow_type,
+        output_arrow_field,
         return_type,
     }))
 }
diff --git a/src/expr/impl/src/scalar/format.rs b/src/expr/impl/src/scalar/format.rs
index 50195638e4d0e..0df53bcc277cd 100644
--- a/src/expr/impl/src/scalar/format.rs
+++ b/src/expr/impl/src/scalar/format.rs
@@ -18,7 +18,6 @@ use std::str::FromStr;
 use risingwave_common::row::Row;
 use risingwave_common::types::{ScalarRefImpl, ToText};
 use risingwave_expr::{function, ExprError, Result};
-use thiserror_ext::AsReport;
 
 use super::string::quote_ident;
 
diff --git a/src/expr/impl/src/scalar/jsonb_record.rs b/src/expr/impl/src/scalar/jsonb_record.rs
index fcc9606897ab7..b1d399d35a5f9 100644
--- a/src/expr/impl/src/scalar/jsonb_record.rs
+++ b/src/expr/impl/src/scalar/jsonb_record.rs
@@ -41,6 +41,14 @@ use risingwave_expr::{function, ExprError, Result};
 /// )).*;
 /// ----
 /// 1 {2,"a b"} (4,"a b c")
+///
+/// query II
+/// select * from jsonb_populate_record(
+///    null::struct<a int, b int>,
+///    '{"a": 1, "b": 2}'
+/// );
+/// ----
+/// 1 2
 /// ```
 #[function("jsonb_populate_record(struct, jsonb) -> struct")]
 fn jsonb_populate_record(
diff --git a/src/expr/impl/src/scalar/jsonb_set.rs b/src/expr/impl/src/scalar/jsonb_set.rs
new file mode 100644
index 0000000000000..e3efefb05d416
--- /dev/null
+++ b/src/expr/impl/src/scalar/jsonb_set.rs
@@ -0,0 +1,186 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use jsonbb::ValueRef;
+use risingwave_common::types::{JsonbRef, JsonbVal, ListRef};
+use risingwave_expr::{function, ExprError, Result};
+
+/// Returns `target` with the item designated by `path` replaced by `new_value`, or with `new_value`
+/// added if `create_if_missing` is true (which is the default) and the item designated by path does
+/// not exist. All earlier steps in the path must exist, or the `target` is returned unchanged. As
+/// with the path oriented operators, negative integers that appear in the path count from the end
+/// of JSON arrays.
+///
+/// If the last path step is an array index that is out of range, and `create_if_missing` is true,
+/// the new value is added at the beginning of the array if the index is negative, or at the end of
+/// the array if it is positive.
+///
+/// # Examples
+///
+/// ```slt
+/// query T
+/// SELECT jsonb_set('[{"f1":1,"f2":null},2,null,3]', '{0,f1}', '[2,3,4]', false);
+/// ----
+/// [{"f1": [2, 3, 4], "f2": null}, 2, null, 3]
+///
+/// query T
+/// SELECT jsonb_set('[{"f1":1,"f2":null},2]', '{0,f3}', '[2,3,4]');
+/// ----
+/// [{"f1": 1, "f2": null, "f3": [2, 3, 4]}, 2]
+/// ```
+#[function("jsonb_set(jsonb, varchar[], jsonb, boolean) -> jsonb")]
+fn jsonb_set4(
+    target: JsonbRef<'_>,
+    path: ListRef<'_>,
+    new_value: JsonbRef<'_>,
+    create_if_missing: bool,
+) -> Result<JsonbVal> {
+    if target.is_scalar() {
+        return Err(ExprError::InvalidParam {
+            name: "jsonb",
+            reason: "cannot set path in scalar".into(),
+        });
+    }
+    let target: ValueRef<'_> = target.into();
+    let new_value: ValueRef<'_> = new_value.into();
+    let mut builder = jsonbb::Builder::<Vec<u8>>::with_capacity(target.capacity());
+    jsonbb_set_path(target, path, 0, new_value, create_if_missing, &mut builder)?;
+    Ok(JsonbVal::from(builder.finish()))
+}
+
+#[function("jsonb_set(jsonb, varchar[], jsonb) -> jsonb")]
+fn jsonb_set3(
+    target: JsonbRef<'_>,
+    path: ListRef<'_>,
+    new_value: JsonbRef<'_>,
+) -> Result<JsonbVal> {
+    jsonb_set4(target, path, new_value, true)
+}
+
+/// Recursively set `path[i..]` in `target` to `new_value` and write the result to `builder`.
+///
+/// Panics if `i` is out of bounds.
+fn jsonbb_set_path(
+    target: ValueRef<'_>,
+    path: ListRef<'_>,
+    i: usize,
+    new_value: ValueRef<'_>,
+    create_if_missing: bool,
+    builder: &mut jsonbb::Builder,
+) -> Result<()> {
+    let last_step = i == path.len() - 1;
+    match target {
+        ValueRef::Object(obj) => {
+            let key = path
+                .get(i)
+                .unwrap()
+                .ok_or_else(|| ExprError::InvalidParam {
+                    name: "path",
+                    reason: format!("path element at position {} is null", i + 1).into(),
+                })?
+                .into_utf8();
+            builder.begin_object();
+            for (k, v) in obj.iter() {
+                builder.add_string(k);
+                if k != key {
+                    builder.add_value(v);
+                } else if last_step {
+                    builder.add_value(new_value);
+                } else {
+                    // recursively set path[i+1..] in v
+                    jsonbb_set_path(v, path, i + 1, new_value, create_if_missing, builder)?;
+                }
+            }
+            if create_if_missing && last_step && !obj.contains_key(key) {
+                builder.add_string(key);
+                builder.add_value(new_value);
+            }
+            builder.end_object();
+            Ok(())
+        }
+        ValueRef::Array(array) => {
+            let key = path
+                .get(i)
+                .unwrap()
+                .ok_or_else(|| ExprError::InvalidParam {
+                    name: "path",
+                    reason: format!("path element at position {} is null", i + 1).into(),
+                })?
+                .into_utf8();
+            let idx = key.parse::<i32>().map_err(|_| ExprError::InvalidParam {
+                name: "path",
+                reason: format!(
+                    "path element at position {} is not an integer: \"{}\"",
+                    i + 1,
+                    key
+                )
+                .into(),
+            })?;
+            let Some(idx) = normalize_array_index(array.len(), idx) else {
+                // out of bounds index
+                if create_if_missing {
+                    builder.begin_array();
+                    // the new value is added at the beginning of the array if the index is negative
+                    if idx < 0 {
+                        builder.add_value(new_value);
+                    }
+                    for v in array.iter() {
+                        builder.add_value(v);
+                    }
+                    // or at the end of the array if it is positive.
+                    if idx >= 0 {
+                        builder.add_value(new_value);
+                    }
+                    builder.end_array();
+                } else {
+                    builder.add_value(target);
+                }
+                return Ok(());
+            };
+            builder.begin_array();
+            for (j, v) in array.iter().enumerate() {
+                if j != idx {
+                    builder.add_value(v);
+                    continue;
+                }
+                if last_step {
+                    builder.add_value(new_value);
+                } else {
+                    // recursively set path[i+1..] in v
+                    jsonbb_set_path(v, path, i + 1, new_value, create_if_missing, builder)?;
+                }
+            }
+            builder.end_array();
+            Ok(())
+        }
+        _ => {
+            builder.add_value(target);
+            Ok(())
+        }
+    }
+}
+
+/// Normalizes an array index to `0..len`.
+/// Negative indices count from the end. i.e. `-len..0 => 0..len`.
+/// Returns `None` if index is out of bounds.
+fn normalize_array_index(len: usize, index: i32) -> Option<usize> {
+    if index < -(len as i32) || index >= (len as i32) {
+        return None;
+    }
+    if index >= 0 {
+        Some(index as usize)
+    } else {
+        Some((len as i32 + index) as usize)
+    }
+}
diff --git a/src/expr/impl/src/scalar/mod.rs b/src/expr/impl/src/scalar/mod.rs
index edbaaf4de01ab..c4e7990de133b 100644
--- a/src/expr/impl/src/scalar/mod.rs
+++ b/src/expr/impl/src/scalar/mod.rs
@@ -58,6 +58,7 @@ mod jsonb_info;
 mod jsonb_object;
 mod jsonb_path;
 mod jsonb_record;
+mod jsonb_set;
 mod length;
 mod lower;
 mod make_time;
diff --git a/src/expr/impl/src/scalar/proctime.rs b/src/expr/impl/src/scalar/proctime.rs
index 2e96e602633cb..2c3c2c651c3df 100644
--- a/src/expr/impl/src/scalar/proctime.rs
+++ b/src/expr/impl/src/scalar/proctime.rs
@@ -25,7 +25,6 @@ fn proctime() -> Result<Timestamptz> {
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::types::Timestamptz;
     use risingwave_common::util::epoch::{Epoch, EpochPair};
 
     use super::*;
diff --git a/src/expr/impl/src/scalar/round.rs b/src/expr/impl/src/scalar/round.rs
index a43dd425e13c3..90c9512c6c9b2 100644
--- a/src/expr/impl/src/scalar/round.rs
+++ b/src/expr/impl/src/scalar/round.rs
@@ -73,9 +73,6 @@ pub fn round_decimal(input: Decimal) -> Decimal {
 mod tests {
     use std::str::FromStr;
 
-    use risingwave_common::types::{Decimal, F64};
-
-    use super::ceil_f64;
     use crate::scalar::round::*;
 
     fn do_test(input: &str, digits: i32, expected_output: Option<&str>) {
diff --git a/src/expr/impl/src/scalar/string.rs b/src/expr/impl/src/scalar/string.rs
index 700fe8e632b35..32ba0787d75b7 100644
--- a/src/expr/impl/src/scalar/string.rs
+++ b/src/expr/impl/src/scalar/string.rs
@@ -472,6 +472,74 @@ pub fn right(s: &str, n: i32, writer: &mut impl Write) {
         .for_each(|c| writer.write_char(c).unwrap());
 }
 
+/// `quote_literal(string text)`
+/// `quote_literal(value anyelement)`
+///
+/// Return the given string suitably quoted to be used as a string literal in an SQL statement
+/// string. Embedded single-quotes and backslashes are properly doubled.
+/// Note that `quote_literal` returns null on null input; if the argument might be null,
+/// `quote_nullable` is often more suitable.
+///
+/// # Example
+///
+/// Note that the quotes are part of the output string.
+///
+/// ```slt
+/// query T
+/// select quote_literal(E'O\'Reilly')
+/// ----
+/// 'O''Reilly'
+///
+/// query T
+/// select quote_literal(E'C:\\Windows\\')
+/// ----
+/// E'C:\\Windows\\'
+///
+/// query T
+/// select quote_literal(42.5)
+/// ----
+/// '42.5'
+///
+/// query T
+/// select quote_literal('hello'::bytea);
+/// ----
+/// E'\\x68656c6c6f'
+///
+/// query T
+/// select quote_literal('{"hello":"world","foo":233}'::jsonb);
+/// ----
+/// '{"foo": 233, "hello": "world"}'
+/// ```
+#[function("quote_literal(varchar) -> varchar")]
+pub fn quote_literal(s: &str, writer: &mut impl Write) {
+    if s.contains('\\') {
+        // use escape format: E'...'
+        write!(writer, "E").unwrap();
+    }
+    write!(writer, "'").unwrap();
+    for c in s.chars() {
+        match c {
+            '\'' => write!(writer, "''").unwrap(),
+            '\\' => write!(writer, "\\\\").unwrap(),
+            _ => write!(writer, "{}", c).unwrap(),
+        }
+    }
+    write!(writer, "'").unwrap();
+}
+
+/// `quote_nullable(string text)`
+///
+/// Return the given string suitably quoted to be used as a string literal in an SQL statement
+/// string; or, if the argument is null, return NULL.
+/// Embedded single-quotes and backslashes are properly doubled.
+#[function("quote_nullable(varchar) -> varchar")]
+pub fn quote_nullable(s: Option<&str>, writer: &mut impl Write) {
+    match s {
+        Some(s) => quote_literal(s, writer),
+        None => write!(writer, "NULL").unwrap(),
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/src/expr/impl/src/scalar/trigonometric.rs b/src/expr/impl/src/scalar/trigonometric.rs
index 705ee8f3eeba2..20e2b3523716b 100644
--- a/src/expr/impl/src/scalar/trigonometric.rs
+++ b/src/expr/impl/src/scalar/trigonometric.rs
@@ -362,7 +362,7 @@ pub fn radians_f64(input: F64) -> F64 {
 mod tests {
     use std::f64::consts::PI;
 
-    use risingwave_common::types::{FloatExt, F64};
+    use risingwave_common::types::FloatExt;
 
     use crate::scalar::trigonometric::*;
 
diff --git a/src/expr/impl/src/table_function/generate_series.rs b/src/expr/impl/src/table_function/generate_series.rs
index d7d4bc1db37d7..91cd748a47922 100644
--- a/src/expr/impl/src/table_function/generate_series.rs
+++ b/src/expr/impl/src/table_function/generate_series.rs
@@ -18,7 +18,7 @@ use risingwave_expr::{function, ExprError, Result};
 
 #[function("generate_series(int4, int4) -> setof int4")]
 #[function("generate_series(int8, int8) -> setof int8")]
-fn generate_series<T>(start: T, stop: T) -> Result<impl Iterator<Item = Result<T>>>
+fn generate_series<T>(start: T, stop: T) -> Result<impl Iterator<Item = T>>
 where
     T: CheckedAdd<Output = T> + PartialOrd + Copy + One + IsNegative,
 {
@@ -26,10 +26,7 @@ where
 }
 
 #[function("generate_series(decimal, decimal) -> setof decimal")]
-fn generate_series_decimal(
-    start: Decimal,
-    stop: Decimal,
-) -> Result<impl Iterator<Item = Result<Decimal>>>
+fn generate_series_decimal(start: Decimal, stop: Decimal) -> Result<impl Iterator<Item = Decimal>>
 where
 {
     validate_range_parameters(start, stop, Decimal::one())?;
@@ -39,7 +36,7 @@ where
 #[function("generate_series(int4, int4, int4) -> setof int4")]
 #[function("generate_series(int8, int8, int8) -> setof int8")]
 #[function("generate_series(timestamp, timestamp, interval) -> setof timestamp")]
-fn generate_series_step<T, S>(start: T, stop: T, step: S) -> Result<impl Iterator<Item = Result<T>>>
+fn generate_series_step<T, S>(start: T, stop: T, step: S) -> Result<impl Iterator<Item = T>>
 where
     T: CheckedAdd<S, Output = T> + PartialOrd + Copy,
     S: IsNegative + Copy,
@@ -52,14 +49,14 @@ fn generate_series_step_decimal(
     start: Decimal,
     stop: Decimal,
     step: Decimal,
-) -> Result<impl Iterator<Item = Result<Decimal>>> {
+) -> Result<impl Iterator<Item = Decimal>> {
     validate_range_parameters(start, stop, step)?;
     range_generic::<_, _, true>(start, stop, step)
 }
 
 #[function("range(int4, int4) -> setof int4")]
 #[function("range(int8, int8) -> setof int8")]
-fn range<T>(start: T, stop: T) -> Result<impl Iterator<Item = Result<T>>>
+fn range<T>(start: T, stop: T) -> Result<impl Iterator<Item = T>>
 where
     T: CheckedAdd<Output = T> + PartialOrd + Copy + One + IsNegative,
 {
@@ -67,7 +64,7 @@ where
 }
 
 #[function("range(decimal, decimal) -> setof decimal")]
-fn range_decimal(start: Decimal, stop: Decimal) -> Result<impl Iterator<Item = Result<Decimal>>>
+fn range_decimal(start: Decimal, stop: Decimal) -> Result<impl Iterator<Item = Decimal>>
 where
 {
     validate_range_parameters(start, stop, Decimal::one())?;
@@ -77,7 +74,7 @@ where
 #[function("range(int4, int4, int4) -> setof int4")]
 #[function("range(int8, int8, int8) -> setof int8")]
 #[function("range(timestamp, timestamp, interval) -> setof timestamp")]
-fn range_step<T, S>(start: T, stop: T, step: S) -> Result<impl Iterator<Item = Result<T>>>
+fn range_step<T, S>(start: T, stop: T, step: S) -> Result<impl Iterator<Item = T>>
 where
     T: CheckedAdd<S, Output = T> + PartialOrd + Copy,
     S: IsNegative + Copy,
@@ -90,7 +87,7 @@ fn range_step_decimal(
     start: Decimal,
     stop: Decimal,
     step: Decimal,
-) -> Result<impl Iterator<Item = Result<Decimal>>> {
+) -> Result<impl Iterator<Item = Decimal>> {
     validate_range_parameters(start, stop, step)?;
     range_generic::<_, _, false>(start, stop, step)
 }
@@ -100,7 +97,7 @@ fn range_generic<T, S, const INCLUSIVE: bool>(
     start: T,
     stop: T,
     step: S,
-) -> Result<impl Iterator<Item = Result<T>>>
+) -> Result<impl Iterator<Item = T>>
 where
     T: CheckedAdd<S, Output = T> + PartialOrd + Copy,
     S: IsNegative + Copy,
@@ -113,21 +110,22 @@ where
     }
     let mut cur = start;
     let neg = step.is_negative();
-    let mut next = move || {
+    let next = move || {
         match (INCLUSIVE, neg) {
-            (true, true) if cur < stop => return Ok(None),
-            (true, false) if cur > stop => return Ok(None),
-            (false, true) if cur <= stop => return Ok(None),
-            (false, false) if cur >= stop => return Ok(None),
+            (true, true) if cur < stop => return None,
+            (true, false) if cur > stop => return None,
+            (false, true) if cur <= stop => return None,
+            (false, false) if cur >= stop => return None,
             _ => {}
         };
         let ret = cur;
-        cur = cur.checked_add(step).ok_or(ExprError::NumericOutOfRange)?;
-        Ok(Some(ret))
+        cur = cur.checked_add(step)?;
+        Some(ret)
     };
-    Ok(std::iter::from_fn(move || next().transpose()))
+    Ok(std::iter::from_fn(next))
 }
 
+/// Validate decimals can not be `NaN` or `infinity`.
 #[inline]
 fn validate_range_parameters(start: Decimal, stop: Decimal, step: Decimal) -> Result<()> {
     validate_decimal(start, "start")?;
@@ -160,8 +158,7 @@ mod tests {
     use risingwave_common::types::test_utils::IntervalTestExt;
     use risingwave_common::types::{DataType, Decimal, Interval, ScalarImpl, Timestamp};
     use risingwave_expr::expr::{BoxedExpression, ExpressionBoxExt, LiteralExpression};
-    use risingwave_expr::table_function::build;
-    use risingwave_expr::ExprError;
+    use risingwave_expr::table_function::{build, check_error};
     use risingwave_pb::expr::table_function::PbType;
 
     const CHUNK_SIZE: usize = 1024;
@@ -309,40 +306,27 @@ mod tests {
 
     #[tokio::test]
     async fn test_generate_series_decimal() {
-        let start = Decimal::from_str("1").unwrap();
-        let start_inf = Decimal::from_str("infinity").unwrap();
-        let stop = Decimal::from_str("5").unwrap();
-        let stop_inf = Decimal::from_str("-infinity").unwrap();
-
-        let step = Decimal::from_str("1").unwrap();
-        let step_nan = Decimal::from_str("nan").unwrap();
-        let step_inf = Decimal::from_str("infinity").unwrap();
-        generate_series_decimal(start, stop, step, true).await;
-        generate_series_decimal(start_inf, stop, step, false).await;
-        generate_series_decimal(start_inf, stop_inf, step, false).await;
-        generate_series_decimal(start, stop_inf, step, false).await;
-        generate_series_decimal(start, stop, step_nan, false).await;
-        generate_series_decimal(start, stop, step_inf, false).await;
-        generate_series_decimal(start, stop_inf, step_nan, false).await;
+        generate_series_decimal("1", "5", "1", true).await;
+        generate_series_decimal("inf", "5", "1", false).await;
+        generate_series_decimal("inf", "-inf", "1", false).await;
+        generate_series_decimal("1", "-inf", "1", false).await;
+        generate_series_decimal("1", "5", "nan", false).await;
+        generate_series_decimal("1", "5", "inf", false).await;
+        generate_series_decimal("1", "-inf", "nan", false).await;
     }
 
-    async fn generate_series_decimal(
-        start: Decimal,
-        stop: Decimal,
-        step: Decimal,
-        expect_ok: bool,
-    ) {
-        fn literal(ty: DataType, v: ScalarImpl) -> BoxedExpression {
-            LiteralExpression::new(ty, Some(v)).boxed()
+    async fn generate_series_decimal(start: &str, stop: &str, step: &str, expect_ok: bool) {
+        fn decimal_literal(v: Decimal) -> BoxedExpression {
+            LiteralExpression::new(DataType::Decimal, Some(v.into())).boxed()
         }
         let function = build(
             PbType::GenerateSeries,
             DataType::Decimal,
             CHUNK_SIZE,
             vec![
-                literal(DataType::Decimal, start.into()),
-                literal(DataType::Decimal, stop.into()),
-                literal(DataType::Decimal, step.into()),
+                decimal_literal(start.parse().unwrap()),
+                decimal_literal(stop.parse().unwrap()),
+                decimal_literal(step.parse().unwrap()),
             ],
         )
         .unwrap();
@@ -350,17 +334,13 @@ mod tests {
         let dummy_chunk = DataChunk::new_dummy(1);
         let mut output = function.eval(&dummy_chunk).await;
         while let Some(res) = output.next().await {
-            match res {
-                Ok(_) => {
-                    assert!(expect_ok);
-                }
-                Err(ExprError::InvalidParam { .. }) => {
-                    assert!(!expect_ok);
-                }
-                Err(_) => {
-                    unreachable!();
-                }
-            }
+            let chunk = res.unwrap();
+            let error = check_error(&chunk);
+            assert_eq!(
+                error.is_ok(),
+                expect_ok,
+                "generate_series({start}, {stop}, {step})"
+            );
         }
     }
 }
diff --git a/src/expr/impl/src/table_function/mod.rs b/src/expr/impl/src/table_function/mod.rs
index 4a52ec5bd1f41..dfafb6721e866 100644
--- a/src/expr/impl/src/table_function/mod.rs
+++ b/src/expr/impl/src/table_function/mod.rs
@@ -16,5 +16,6 @@ mod generate_series;
 mod generate_subscripts;
 mod jsonb;
 mod pg_expandarray;
+mod pg_get_keywords;
 mod regexp_matches;
 mod unnest;
diff --git a/src/expr/impl/src/table_function/pg_get_keywords.rs b/src/expr/impl/src/table_function/pg_get_keywords.rs
new file mode 100644
index 0000000000000..bd18660872720
--- /dev/null
+++ b/src/expr/impl/src/table_function/pg_get_keywords.rs
@@ -0,0 +1,58 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_expr::function;
+use risingwave_sqlparser::keywords::{
+    ALL_KEYWORDS_INDEX, RESERVED_FOR_COLUMN_ALIAS, RESERVED_FOR_COLUMN_OR_TABLE_NAME,
+};
+
+/// Returns a set of records describing the SQL keywords recognized by the server.
+///
+/// The word column contains the keyword.
+///
+/// The catcode column contains a category code:
+/// - U for an unreserved keyword
+/// - C for a keyword that can be a column name
+/// - T for a keyword that can be a type or function name
+/// - R for a fully reserved keyword.
+///
+/// The catdesc column contains a possibly-localized string describing the keyword's category.
+///
+/// ```slt
+/// query TTT
+/// select * from pg_get_keywords() where word = 'add';
+/// ----
+/// add U unreserved
+/// ```
+#[function("pg_get_keywords() -> setof struct<word varchar, catcode varchar, catdesc varchar>")]
+fn pg_get_keywords() -> impl Iterator<Item = (Box<str>, &'static str, &'static str)> {
+    ALL_KEYWORDS_INDEX.iter().map(|keyword| {
+        // FIXME: The current category is not correct. Many are different from the PostgreSQL.
+        let catcode = if !RESERVED_FOR_COLUMN_OR_TABLE_NAME.contains(keyword) {
+            "U"
+        } else if !RESERVED_FOR_COLUMN_ALIAS.contains(keyword) {
+            "C"
+        } else {
+            "R"
+        };
+        let catdesc = match catcode {
+            "U" => "unreserved",
+            "C" => "unreserved (cannot be function or type name)",
+            "T" => "reserved (can be function or type name)",
+            "R" => "reserved",
+            _ => unreachable!(),
+        };
+        (keyword.to_string().to_lowercase().into(), catcode, catdesc)
+    })
+}
diff --git a/src/expr/impl/src/udf/deno.rs b/src/expr/impl/src/udf/deno.rs
new file mode 100644
index 0000000000000..8c0c0a4674a8c
--- /dev/null
+++ b/src/expr/impl/src/udf/deno.rs
@@ -0,0 +1,127 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use anyhow::bail;
+use arrow_udf_js_deno::{CallMode, Runtime};
+use futures_async_stream::try_stream;
+use risingwave_common::array::arrow::{ToArrow, UdfArrowConvert};
+
+use super::*;
+
+#[linkme::distributed_slice(UDF_IMPLS)]
+static DENO: UdfImplDescriptor = UdfImplDescriptor {
+    match_fn: |language, runtime, _link| language == "javascript" && runtime == Some("deno"),
+    create_fn: |opts| {
+        let mut body = None;
+        let mut compressed_binary = None;
+        let identifier = opts.name.to_string();
+        match (opts.using_link, opts.using_base64_decoded, opts.as_) {
+            (None, None, Some(as_)) => body = Some(as_.to_string()),
+            (Some(link), None, None) => {
+                let bytes = read_file_from_link(link)?;
+                compressed_binary = Some(zstd::stream::encode_all(bytes.as_slice(), 0)?);
+            }
+            (None, Some(bytes), None) => {
+                compressed_binary = Some(zstd::stream::encode_all(bytes, 0)?);
+            }
+            (None, None, None) => bail!("Either USING or AS must be specified"),
+            _ => bail!("Both USING and AS cannot be specified"),
+        }
+        Ok(CreateFunctionOutput {
+            identifier,
+            body,
+            compressed_binary,
+        })
+    },
+    build_fn: |opts| {
+        let runtime = Runtime::new();
+        let body = match (opts.body, opts.compressed_binary) {
+            (Some(body), _) => body.to_string(),
+            (_, Some(compressed_binary)) => {
+                let binary = zstd::stream::decode_all(compressed_binary)
+                    .context("failed to decompress binary")?;
+                String::from_utf8(binary).context("failed to decode binary")?
+            }
+            _ => bail!("UDF body or compressed binary is required for deno UDF"),
+        };
+
+        let body = format!(
+            "export {} {}({}) {{ {} }}",
+            match opts.function_type {
+                Some("sync") => "function",
+                Some("async") => "async function",
+                Some("generator") => "function*",
+                Some("async_generator") => "async function*",
+                _ if opts.kind.is_table() => "function*",
+                _ => "function",
+            },
+            opts.identifier,
+            opts.arg_names.join(","),
+            body
+        );
+
+        tokio::task::block_in_place(|| {
+            tokio::runtime::Handle::current().block_on(runtime.add_function(
+                opts.identifier,
+                UdfArrowConvert::default().to_arrow_field("", opts.return_type)?,
+                CallMode::CalledOnNullInput,
+                &body,
+            ))
+        })?;
+
+        Ok(Box::new(DenoFunction {
+            runtime,
+            identifier: opts.identifier.to_string(),
+        }))
+    },
+};
+
+#[derive(Debug)]
+struct DenoFunction {
+    runtime: Arc<Runtime>,
+    identifier: String,
+}
+
+#[async_trait::async_trait]
+impl UdfImpl for DenoFunction {
+    async fn call(&self, input: &RecordBatch) -> Result<RecordBatch> {
+        // FIXME(runji): make the future Send
+        tokio::task::block_in_place(|| {
+            tokio::runtime::Handle::current()
+                .block_on(self.runtime.call(&self.identifier, input.clone()))
+        })
+    }
+
+    async fn call_table_function<'a>(
+        &'a self,
+        input: &'a RecordBatch,
+    ) -> Result<BoxStream<'a, Result<RecordBatch>>> {
+        Ok(self.call_table_function_inner(input.clone()))
+    }
+}
+
+impl DenoFunction {
+    #[try_stream(boxed, ok = RecordBatch, error = anyhow::Error)]
+    async fn call_table_function_inner<'a>(&'a self, input: RecordBatch) {
+        let mut stream = self
+            .runtime
+            .call_table_function(&self.identifier, input, 1024)
+            .await?;
+        while let Some(batch) = stream.next().await {
+            yield batch?;
+        }
+    }
+}
diff --git a/src/expr/impl/src/udf/external.rs b/src/expr/impl/src/udf/external.rs
new file mode 100644
index 0000000000000..5c400df26c179
--- /dev/null
+++ b/src/expr/impl/src/udf/external.rs
@@ -0,0 +1,289 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicU8, Ordering};
+use std::sync::{Arc, LazyLock, Weak};
+use std::time::Duration;
+
+use anyhow::bail;
+use arrow_schema::Fields;
+use arrow_udf_flight::Client;
+use futures_util::{StreamExt, TryStreamExt};
+use ginepro::{LoadBalancedChannel, ResolutionStrategy};
+use risingwave_common::array::arrow::{ToArrow, UdfArrowConvert};
+use risingwave_common::util::addr::HostAddr;
+use thiserror_ext::AsReport;
+
+use super::*;
+
+#[linkme::distributed_slice(UDF_IMPLS)]
+static EXTERNAL: UdfImplDescriptor = UdfImplDescriptor {
+    match_fn: |language, _runtime, link| {
+        link.is_some() && matches!(language, "python" | "java" | "")
+    },
+    create_fn: |opts| {
+        let link = opts.using_link.context("USING LINK must be specified")?;
+        let identifier = opts.as_.context("AS must be specified")?.to_string();
+
+        // check UDF server
+        let client = get_or_create_flight_client(link)?;
+        let convert = UdfArrowConvert {
+            legacy: client.protocol_version() == 1,
+        };
+        // A helper function to create a unnamed field from data type.
+        let to_field = |data_type| convert.to_arrow_field("", data_type);
+        let args = arrow_schema::Schema::new(
+            opts.arg_types
+                .iter()
+                .map(to_field)
+                .try_collect::<Fields>()?,
+        );
+        let returns = arrow_schema::Schema::new(if opts.kind.is_table() {
+            vec![
+                arrow_schema::Field::new("row", arrow_schema::DataType::Int32, true),
+                to_field(opts.return_type)?,
+            ]
+        } else {
+            vec![to_field(opts.return_type)?]
+        });
+        let function = tokio::task::block_in_place(|| {
+            tokio::runtime::Handle::current().block_on(client.get(&identifier))
+        })
+        .context("failed to check UDF signature")?;
+        if !data_types_match(&function.args, &args) {
+            bail!(
+                "argument type mismatch, expect: {:?}, actual: {:?}",
+                args,
+                function.args,
+            );
+        }
+        if !data_types_match(&function.returns, &returns) {
+            bail!(
+                "return type mismatch, expect: {:?}, actual: {:?}",
+                returns,
+                function.returns,
+            );
+        }
+        Ok(CreateFunctionOutput {
+            identifier,
+            body: None,
+            compressed_binary: None,
+        })
+    },
+    build_fn: |opts| {
+        let link = opts.link.context("link is required")?;
+        let client = get_or_create_flight_client(link)?;
+        Ok(Box::new(ExternalFunction {
+            identifier: opts.identifier.to_string(),
+            client,
+            disable_retry_count: AtomicU8::new(INITIAL_RETRY_COUNT),
+            always_retry_on_network_error: opts.always_retry_on_network_error,
+        }))
+    },
+};
+
+#[derive(Debug)]
+struct ExternalFunction {
+    identifier: String,
+    client: Arc<Client>,
+    /// Number of remaining successful calls until retry is enabled.
+    /// This parameter is designed to prevent continuous retry on every call, which would increase delay.
+    /// Logic:
+    /// It resets to `INITIAL_RETRY_COUNT` after a single failure and then decrements with each call, enabling retry when it reaches zero.
+    /// If non-zero, we will not retry on connection errors to prevent blocking the stream.
+    /// On each connection error, the count will be reset to `INITIAL_RETRY_COUNT`.
+    /// On each successful call, the count will be decreased by 1.
+    /// Link:
+    /// See <https://github.com/risingwavelabs/risingwave/issues/13791>.
+    disable_retry_count: AtomicU8,
+    /// Always retry. Overrides `disable_retry_count`.
+    always_retry_on_network_error: bool,
+}
+
+const INITIAL_RETRY_COUNT: u8 = 16;
+
+#[async_trait::async_trait]
+impl UdfImpl for ExternalFunction {
+    fn is_legacy(&self) -> bool {
+        // see <https://github.com/risingwavelabs/risingwave/pull/16619> for details
+        self.client.protocol_version() == 1
+    }
+
+    async fn call(&self, input: &RecordBatch) -> Result<RecordBatch> {
+        let disable_retry_count = self.disable_retry_count.load(Ordering::Relaxed);
+        let result = if self.always_retry_on_network_error {
+            self.call_with_always_retry_on_network_error(input).await
+        } else {
+            let result = if disable_retry_count != 0 {
+                self.client.call(&self.identifier, input).await
+            } else {
+                self.call_with_retry(input).await
+            };
+            let disable_retry_count = self.disable_retry_count.load(Ordering::Relaxed);
+            let connection_error = matches!(&result, Err(e) if is_connection_error(e));
+            if connection_error && disable_retry_count != INITIAL_RETRY_COUNT {
+                // reset count on connection error
+                self.disable_retry_count
+                    .store(INITIAL_RETRY_COUNT, Ordering::Relaxed);
+            } else if !connection_error && disable_retry_count != 0 {
+                // decrease count on success, ignore if exchange failed
+                _ = self.disable_retry_count.compare_exchange(
+                    disable_retry_count,
+                    disable_retry_count - 1,
+                    Ordering::Relaxed,
+                    Ordering::Relaxed,
+                );
+            }
+            result
+        };
+        result.map_err(|e| e.into())
+    }
+
+    async fn call_table_function<'a>(
+        &'a self,
+        input: &'a RecordBatch,
+    ) -> Result<BoxStream<'a, Result<RecordBatch>>> {
+        let stream = self
+            .client
+            .call_table_function(&self.identifier, input)
+            .await?;
+        Ok(stream.map_err(|e| e.into()).boxed())
+    }
+}
+
+/// Get or create a client for the given UDF service.
+///
+/// There is a global cache for clients, so that we can reuse the same client for the same service.
+fn get_or_create_flight_client(link: &str) -> Result<Arc<Client>> {
+    static CLIENTS: LazyLock<std::sync::Mutex<HashMap<String, Weak<Client>>>> =
+        LazyLock::new(Default::default);
+    let mut clients = CLIENTS.lock().unwrap();
+    if let Some(client) = clients.get(link).and_then(|c| c.upgrade()) {
+        // reuse existing client
+        Ok(client)
+    } else {
+        // create new client
+        let client = Arc::new(tokio::task::block_in_place(|| {
+            tokio::runtime::Handle::current().block_on(async {
+                let channel = connect_tonic(link).await?;
+                Ok(Client::new(channel).await?) as Result<_>
+            })
+        })?);
+        clients.insert(link.to_owned(), Arc::downgrade(&client));
+        Ok(client)
+    }
+}
+
+/// Connect to a UDF service and return a tonic `Channel`.
+async fn connect_tonic(mut addr: &str) -> Result<tonic::transport::Channel> {
+    // Interval between two successive probes of the UDF DNS.
+    const DNS_PROBE_INTERVAL_SECS: u64 = 5;
+    // Timeout duration for performing an eager DNS resolution.
+    const EAGER_DNS_RESOLVE_TIMEOUT_SECS: u64 = 5;
+    const REQUEST_TIMEOUT_SECS: u64 = 5;
+    const CONNECT_TIMEOUT_SECS: u64 = 5;
+
+    if let Some(s) = addr.strip_prefix("http://") {
+        addr = s;
+    }
+    if let Some(s) = addr.strip_prefix("https://") {
+        addr = s;
+    }
+    let host_addr = addr.parse::<HostAddr>()?;
+    let channel = LoadBalancedChannel::builder((host_addr.host.clone(), host_addr.port))
+        .dns_probe_interval(std::time::Duration::from_secs(DNS_PROBE_INTERVAL_SECS))
+        .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECS))
+        .connect_timeout(Duration::from_secs(CONNECT_TIMEOUT_SECS))
+        .resolution_strategy(ResolutionStrategy::Eager {
+            timeout: tokio::time::Duration::from_secs(EAGER_DNS_RESOLVE_TIMEOUT_SECS),
+        })
+        .channel()
+        .await
+        .with_context(|| format!("failed to create LoadBalancedChannel, address: {host_addr}"))?;
+    Ok(channel.into())
+}
+
+impl ExternalFunction {
+    /// Call a function, retry up to 5 times / 3s if connection is broken.
+    async fn call_with_retry(
+        &self,
+        input: &RecordBatch,
+    ) -> Result<RecordBatch, arrow_udf_flight::Error> {
+        let mut backoff = Duration::from_millis(100);
+        for i in 0..5 {
+            match self.client.call(&self.identifier, input).await {
+                Err(err) if is_connection_error(&err) && i != 4 => {
+                    tracing::error!(?backoff, error = %err.as_report(), "UDF connection error. retry...");
+                }
+                ret => return ret,
+            }
+            tokio::time::sleep(backoff).await;
+            backoff *= 2;
+        }
+        unreachable!()
+    }
+
+    /// Always retry on connection error
+    async fn call_with_always_retry_on_network_error(
+        &self,
+        input: &RecordBatch,
+    ) -> Result<RecordBatch, arrow_udf_flight::Error> {
+        let mut backoff = Duration::from_millis(100);
+        loop {
+            match self.client.call(&self.identifier, input).await {
+                Err(err) if is_tonic_error(&err) => {
+                    tracing::error!(?backoff, error = %err.as_report(), "UDF tonic error. retry...");
+                }
+                ret => {
+                    if ret.is_err() {
+                        tracing::error!(error = %ret.as_ref().unwrap_err().as_report(), "UDF error. exiting...");
+                    }
+                    return ret;
+                }
+            }
+            tokio::time::sleep(backoff).await;
+            backoff *= 2;
+        }
+    }
+}
+
+/// Returns true if the arrow flight error is caused by a connection error.
+fn is_connection_error(err: &arrow_udf_flight::Error) -> bool {
+    match err {
+        // Connection refused
+        arrow_udf_flight::Error::Tonic(status) if status.code() == tonic::Code::Unavailable => true,
+        _ => false,
+    }
+}
+
+fn is_tonic_error(err: &arrow_udf_flight::Error) -> bool {
+    matches!(
+        err,
+        arrow_udf_flight::Error::Tonic(_)
+            | arrow_udf_flight::Error::Flight(arrow_flight::error::FlightError::Tonic(_))
+    )
+}
+
+/// Check if two list of data types match, ignoring field names.
+fn data_types_match(a: &arrow_schema::Schema, b: &arrow_schema::Schema) -> bool {
+    if a.fields().len() != b.fields().len() {
+        return false;
+    }
+    #[allow(clippy::disallowed_methods)]
+    a.fields()
+        .iter()
+        .zip(b.fields())
+        .all(|(a, b)| a.data_type().equals_datatype(b.data_type()))
+}
diff --git a/src/expr/impl/src/udf/mod.rs b/src/expr/impl/src/udf/mod.rs
new file mode 100644
index 0000000000000..8ae9b6e98b69f
--- /dev/null
+++ b/src/expr/impl/src/udf/mod.rs
@@ -0,0 +1,46 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(dead_code, unused_imports)]
+
+// common imports for submodules
+use anyhow::{Context as _, Result};
+use arrow_array::{ArrayRef, BooleanArray, RecordBatch};
+use futures_util::stream::BoxStream;
+use risingwave_expr::sig::{
+    CreateFunctionOptions, CreateFunctionOutput, UdfImpl, UdfImplDescriptor, UDF_IMPLS,
+};
+
+#[cfg(feature = "deno-udf")]
+mod deno;
+#[cfg(feature = "external-udf")]
+#[cfg(not(madsim))]
+mod external;
+#[cfg(feature = "python-udf")]
+mod python;
+#[cfg(feature = "js-udf")]
+mod quickjs;
+#[cfg(feature = "wasm-udf")]
+mod wasm;
+
+/// Download wasm binary from a link.
+fn read_file_from_link(link: &str) -> Result<Vec<u8>> {
+    // currently only local file system is supported
+    let path = link
+        .strip_prefix("fs://")
+        .context("only 'fs://' is supported")?;
+    let content =
+        std::fs::read(path).context("failed to read wasm binary from local file system")?;
+    Ok(content)
+}
diff --git a/src/expr/impl/src/udf/python.rs b/src/expr/impl/src/udf/python.rs
new file mode 100644
index 0000000000000..9ca4b2a5b2a24
--- /dev/null
+++ b/src/expr/impl/src/udf/python.rs
@@ -0,0 +1,97 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use arrow_schema::{DataType, Field};
+use arrow_udf_python::{CallMode, Runtime};
+use futures_util::StreamExt;
+use risingwave_common::array::arrow::{ToArrow, UdfArrowConvert};
+
+use super::*;
+
+#[linkme::distributed_slice(UDF_IMPLS)]
+static PYTHON: UdfImplDescriptor = UdfImplDescriptor {
+    match_fn: |language, _runtime, link| language == "python" && link.is_none(),
+    create_fn: |opts| {
+        Ok(CreateFunctionOutput {
+            identifier: opts.name.to_string(),
+            body: Some(opts.as_.context("AS must be specified")?.to_string()),
+            compressed_binary: None,
+        })
+    },
+    build_fn: |opts| {
+        let mut runtime = Runtime::builder().sandboxed(true).build()?;
+        if opts.kind.is_aggregate() {
+            runtime.add_aggregate(
+                opts.identifier,
+                Field::new("state", DataType::Binary, true).with_metadata(
+                    [("ARROW:extension:name".into(), "arrowudf.pickle".into())].into(),
+                ),
+                UdfArrowConvert::default().to_arrow_field("", opts.return_type)?,
+                CallMode::CalledOnNullInput,
+                opts.body.context("body is required")?,
+            )?;
+        } else {
+            runtime.add_function(
+                opts.identifier,
+                UdfArrowConvert::default().to_arrow_field("", opts.return_type)?,
+                CallMode::CalledOnNullInput,
+                opts.body.context("body is required")?,
+            )?;
+        }
+        Ok(Box::new(PythonFunction {
+            runtime,
+            identifier: opts.identifier.to_string(),
+        }))
+    },
+};
+
+#[derive(Debug)]
+struct PythonFunction {
+    runtime: Runtime,
+    identifier: String,
+}
+
+#[async_trait::async_trait]
+impl UdfImpl for PythonFunction {
+    async fn call(&self, input: &RecordBatch) -> Result<RecordBatch> {
+        self.runtime.call(&self.identifier, input)
+    }
+
+    async fn call_table_function<'a>(
+        &'a self,
+        input: &'a RecordBatch,
+    ) -> Result<BoxStream<'a, Result<RecordBatch>>> {
+        self.runtime
+            .call_table_function(&self.identifier, input, 1024)
+            .map(|s| futures_util::stream::iter(s).boxed())
+    }
+
+    fn call_agg_create_state(&self) -> Result<ArrayRef> {
+        self.runtime.create_state(&self.identifier)
+    }
+
+    fn call_agg_accumulate_or_retract(
+        &self,
+        state: &ArrayRef,
+        ops: &BooleanArray,
+        input: &RecordBatch,
+    ) -> Result<ArrayRef> {
+        self.runtime
+            .accumulate_or_retract(&self.identifier, state, ops, input)
+    }
+
+    fn call_agg_finish(&self, state: &ArrayRef) -> Result<ArrayRef> {
+        self.runtime.finish(&self.identifier, state)
+    }
+}
diff --git a/src/expr/impl/src/udf/quickjs.rs b/src/expr/impl/src/udf/quickjs.rs
new file mode 100644
index 0000000000000..7faa4dec8ae9f
--- /dev/null
+++ b/src/expr/impl/src/udf/quickjs.rs
@@ -0,0 +1,110 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use arrow_schema::{DataType, Field};
+use arrow_udf_js::{CallMode, Runtime};
+use futures_util::StreamExt;
+use risingwave_common::array::arrow::{ToArrow, UdfArrowConvert};
+
+use super::*;
+
+#[linkme::distributed_slice(UDF_IMPLS)]
+static QUICKJS: UdfImplDescriptor = UdfImplDescriptor {
+    match_fn: |language, runtime, _link| {
+        language == "javascript" && matches!(runtime, None | Some("quickjs"))
+    },
+    create_fn: |opts| {
+        Ok(CreateFunctionOutput {
+            identifier: opts.name.to_string(),
+            body: Some(opts.as_.context("AS must be specified")?.to_string()),
+            compressed_binary: None,
+        })
+    },
+    build_fn: |opts| {
+        let mut runtime = Runtime::new()?;
+        if opts.kind.is_aggregate() {
+            runtime.add_aggregate(
+                opts.identifier,
+                Field::new("state", DataType::Binary, true).with_metadata(
+                    [("ARROW:extension:name".into(), "arrowudf.json".into())].into(),
+                ),
+                UdfArrowConvert::default().to_arrow_field("", opts.return_type)?,
+                CallMode::CalledOnNullInput,
+                opts.body.context("body is required")?,
+            )?;
+        } else {
+            let body = format!(
+                "export function{} {}({}) {{ {} }}",
+                if opts.kind.is_table() { "*" } else { "" },
+                opts.identifier,
+                opts.arg_names.join(","),
+                opts.body.context("body is required")?,
+            );
+            runtime.add_function(
+                opts.identifier,
+                UdfArrowConvert::default().to_arrow_field("", opts.return_type)?,
+                CallMode::CalledOnNullInput,
+                &body,
+            )?;
+        }
+        Ok(Box::new(QuickJsFunction {
+            runtime,
+            identifier: opts.identifier.to_string(),
+        }))
+    },
+};
+
+#[derive(Debug)]
+struct QuickJsFunction {
+    runtime: Runtime,
+    identifier: String,
+}
+
+#[async_trait::async_trait]
+impl UdfImpl for QuickJsFunction {
+    async fn call(&self, input: &RecordBatch) -> Result<RecordBatch> {
+        self.runtime.call(&self.identifier, input)
+    }
+
+    async fn call_table_function<'a>(
+        &'a self,
+        input: &'a RecordBatch,
+    ) -> Result<BoxStream<'a, Result<RecordBatch>>> {
+        self.runtime
+            .call_table_function(&self.identifier, input, 1024)
+            .map(|s| futures_util::stream::iter(s).boxed())
+    }
+
+    fn call_agg_create_state(&self) -> Result<ArrayRef> {
+        self.runtime.create_state(&self.identifier)
+    }
+
+    fn call_agg_accumulate_or_retract(
+        &self,
+        state: &ArrayRef,
+        ops: &BooleanArray,
+        input: &RecordBatch,
+    ) -> Result<ArrayRef> {
+        self.runtime
+            .accumulate_or_retract(&self.identifier, state, ops, input)
+    }
+
+    fn call_agg_finish(&self, state: &ArrayRef) -> Result<ArrayRef> {
+        self.runtime.finish(&self.identifier, state)
+    }
+
+    fn memory_usage(&self) -> usize {
+        self.runtime.memory_usage().malloc_size as usize
+    }
+}
diff --git a/src/expr/impl/src/udf/wasm.rs b/src/expr/impl/src/udf/wasm.rs
new file mode 100644
index 0000000000000..bd84cfa004326
--- /dev/null
+++ b/src/expr/impl/src/udf/wasm.rs
@@ -0,0 +1,283 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::borrow::Cow;
+use std::sync::{Arc, LazyLock};
+use std::time::Duration;
+
+use anyhow::{anyhow, bail};
+use arrow_udf_wasm::Runtime;
+use futures_util::StreamExt;
+use itertools::Itertools;
+use risingwave_common::types::DataType;
+use risingwave_expr::sig::UdfOptions;
+
+use super::*;
+
+#[linkme::distributed_slice(UDF_IMPLS)]
+static WASM: UdfImplDescriptor = UdfImplDescriptor {
+    match_fn: |language, _runtime, _link| language == "wasm",
+    create_fn: create_wasm,
+    build_fn: build,
+};
+
+#[linkme::distributed_slice(UDF_IMPLS)]
+static RUST: UdfImplDescriptor = UdfImplDescriptor {
+    match_fn: |language, _runtime, _link| language == "rust",
+    create_fn: create_rust,
+    build_fn: build,
+};
+
+fn create_wasm(opts: CreateFunctionOptions<'_>) -> Result<CreateFunctionOutput> {
+    let wasm_binary: Cow<'_, [u8]> = if let Some(link) = opts.using_link {
+        read_file_from_link(link)?.into()
+    } else if let Some(bytes) = opts.using_base64_decoded {
+        bytes.into()
+    } else {
+        bail!("USING must be specified")
+    };
+    let runtime = get_or_create_wasm_runtime(&wasm_binary)?;
+    if runtime.abi_version().0 <= 2 {
+        bail!("legacy arrow-udf is no longer supported. please update arrow-udf to 0.3+");
+    }
+    let identifier_v1 = wasm_identifier_v1(
+        opts.name,
+        opts.arg_types,
+        opts.return_type,
+        opts.kind.is_table(),
+    );
+    let identifier = find_wasm_identifier_v2(&runtime, &identifier_v1)?;
+    let compressed_binary = Some(zstd::stream::encode_all(&*wasm_binary, 0)?);
+    Ok(CreateFunctionOutput {
+        identifier,
+        body: None,
+        compressed_binary,
+    })
+}
+
+fn create_rust(opts: CreateFunctionOptions<'_>) -> Result<CreateFunctionOutput> {
+    if opts.using_link.is_some() {
+        bail!("USING is not supported for rust function");
+    }
+    let identifier_v1 = wasm_identifier_v1(
+        opts.name,
+        opts.arg_types,
+        opts.return_type,
+        opts.kind.is_table(),
+    );
+    // if the function returns a struct, users need to add `#[function]` macro by themselves.
+    // otherwise, we add it automatically. the code should start with `fn ...`.
+    let function_macro = if opts.return_type.is_struct() {
+        String::new()
+    } else {
+        format!("#[function(\"{}\")]", identifier_v1)
+    };
+    let script = format!(
+        "use arrow_udf::{{function, types::*}};\n{}\n{}",
+        function_macro,
+        opts.as_.context("AS must be specified")?
+    );
+    let body = Some(script.clone());
+
+    let wasm_binary = std::thread::spawn(move || {
+        let mut opts = arrow_udf_wasm::build::BuildOpts::default();
+        opts.arrow_udf_version = Some("0.3".to_string());
+        opts.script = script;
+        // use a fixed tempdir to reuse the build cache
+        opts.tempdir = Some(std::env::temp_dir().join("risingwave-rust-udf"));
+
+        arrow_udf_wasm::build::build_with(&opts)
+    })
+    .join()
+    .unwrap()
+    .context("failed to build rust function")?;
+
+    let runtime = get_or_create_wasm_runtime(&wasm_binary)?;
+    let identifier = find_wasm_identifier_v2(&runtime, &identifier_v1)?;
+
+    let compressed_binary = Some(zstd::stream::encode_all(wasm_binary.as_slice(), 0)?);
+    Ok(CreateFunctionOutput {
+        identifier,
+        body,
+        compressed_binary,
+    })
+}
+
+fn build(opts: UdfOptions<'_>) -> Result<Box<dyn UdfImpl>> {
+    let compressed_binary = opts
+        .compressed_binary
+        .context("compressed binary is required")?;
+    let wasm_binary =
+        zstd::stream::decode_all(compressed_binary).context("failed to decompress wasm binary")?;
+    let runtime = get_or_create_wasm_runtime(&wasm_binary)?;
+    Ok(Box::new(WasmFunction {
+        runtime,
+        identifier: opts.identifier.to_string(),
+    }))
+}
+
+#[derive(Debug)]
+struct WasmFunction {
+    runtime: Arc<Runtime>,
+    identifier: String,
+}
+
+#[async_trait::async_trait]
+impl UdfImpl for WasmFunction {
+    async fn call(&self, input: &RecordBatch) -> Result<RecordBatch> {
+        self.runtime.call(&self.identifier, input)
+    }
+
+    async fn call_table_function<'a>(
+        &'a self,
+        input: &'a RecordBatch,
+    ) -> Result<BoxStream<'a, Result<RecordBatch>>> {
+        self.runtime
+            .call_table_function(&self.identifier, input)
+            .map(|s| futures_util::stream::iter(s).boxed())
+    }
+
+    fn is_legacy(&self) -> bool {
+        // see <https://github.com/risingwavelabs/risingwave/pull/16619> for details
+        self.runtime.abi_version().0 <= 2
+    }
+}
+
+/// Get or create a wasm runtime.
+///
+/// Runtimes returned by this function are cached inside for at least 60 seconds.
+/// Later calls with the same binary will reuse the same runtime.
+fn get_or_create_wasm_runtime(binary: &[u8]) -> Result<Arc<Runtime>> {
+    static RUNTIMES: LazyLock<moka::sync::Cache<md5::Digest, Arc<Runtime>>> = LazyLock::new(|| {
+        moka::sync::Cache::builder()
+            .time_to_idle(Duration::from_secs(60))
+            .build()
+    });
+
+    let md5 = md5::compute(binary);
+    if let Some(runtime) = RUNTIMES.get(&md5) {
+        return Ok(runtime.clone());
+    }
+
+    let runtime = Arc::new(arrow_udf_wasm::Runtime::new(binary)?);
+    RUNTIMES.insert(md5, runtime.clone());
+    Ok(runtime)
+}
+
+/// Convert a v0.1 function identifier to v0.2 format.
+///
+/// In arrow-udf v0.1 format, struct type is inline in the identifier. e.g.
+///
+/// ```text
+/// keyvalue(varchar,varchar)->struct<key:varchar,value:varchar>
+/// ```
+///
+/// However, since arrow-udf v0.2, struct type is no longer inline.
+/// The above identifier is divided into a function and a type.
+///
+/// ```text
+/// keyvalue(varchar,varchar)->struct KeyValue
+/// KeyValue=key:varchar,value:varchar
+/// ```
+///
+/// For compatibility, we should call `find_wasm_identifier_v2` to
+/// convert v0.1 identifiers to v0.2 format before looking up the function.
+fn find_wasm_identifier_v2(
+    runtime: &arrow_udf_wasm::Runtime,
+    inlined_signature: &str,
+) -> Result<String> {
+    // Inline types in function signature.
+    //
+    // # Example
+    //
+    // ```text
+    // types = { "KeyValue": "key:varchar,value:varchar" }
+    // input = "keyvalue(varchar, varchar) -> struct KeyValue"
+    // output = "keyvalue(varchar, varchar) -> struct<key:varchar,value:varchar>"
+    // ```
+    let inline_types = |s: &str| -> String {
+        let mut inlined = s.to_string();
+        // iteratively replace `struct Xxx` with `struct<...>` until no replacement is made.
+        loop {
+            let replaced = inlined.clone();
+            for (k, v) in runtime.types() {
+                inlined = inlined.replace(&format!("struct {k}"), &format!("struct<{v}>"));
+            }
+            if replaced == inlined {
+                return inlined;
+            }
+        }
+    };
+    // Function signature in arrow-udf is case sensitive.
+    // However, SQL identifiers are usually case insensitive and stored in lowercase.
+    // So we should convert the signature to lowercase before comparison.
+    let identifier = runtime
+        .functions()
+        .find(|f| inline_types(f).to_lowercase() == inlined_signature)
+        .ok_or_else(|| {
+            anyhow!(
+                "function not found in wasm binary: \"{}\"\nHINT: available functions:\n  {}\navailable types:\n  {}",
+                inlined_signature,
+                runtime.functions().join("\n  "),
+                runtime.types().map(|(k, v)| format!("{k}: {v}")).join("\n  "),
+            )
+        })?;
+    Ok(identifier.into())
+}
+
+/// Generate a function identifier in v0.1 format from the function signature.
+fn wasm_identifier_v1(
+    name: &str,
+    args: &[DataType],
+    ret: &DataType,
+    table_function: bool,
+) -> String {
+    format!(
+        "{}({}){}{}",
+        name,
+        args.iter().map(datatype_name).join(","),
+        if table_function { "->>" } else { "->" },
+        datatype_name(ret)
+    )
+}
+
+/// Convert a data type to string used in identifier.
+fn datatype_name(ty: &DataType) -> String {
+    match ty {
+        DataType::Boolean => "boolean".to_string(),
+        DataType::Int16 => "int16".to_string(),
+        DataType::Int32 => "int32".to_string(),
+        DataType::Int64 => "int64".to_string(),
+        DataType::Float32 => "float32".to_string(),
+        DataType::Float64 => "float64".to_string(),
+        DataType::Date => "date32".to_string(),
+        DataType::Time => "time64".to_string(),
+        DataType::Timestamp => "timestamp".to_string(),
+        DataType::Timestamptz => "timestamptz".to_string(),
+        DataType::Interval => "interval".to_string(),
+        DataType::Decimal => "decimal".to_string(),
+        DataType::Jsonb => "json".to_string(),
+        DataType::Serial => "serial".to_string(),
+        DataType::Int256 => "int256".to_string(),
+        DataType::Bytea => "binary".to_string(),
+        DataType::Varchar => "string".to_string(),
+        DataType::List(inner) => format!("{}[]", datatype_name(inner)),
+        DataType::Struct(s) => format!(
+            "struct<{}>",
+            s.iter()
+                .map(|(name, ty)| format!("{}:{}", name, datatype_name(ty)))
+                .join(",")
+        ),
+    }
+}
diff --git a/src/expr/core/src/window_function/state/aggregate.rs b/src/expr/impl/src/window_function/aggregate.rs
similarity index 88%
rename from src/expr/core/src/window_function/state/aggregate.rs
rename to src/expr/impl/src/window_function/aggregate.rs
index 45d3729702932..9942581357f77 100644
--- a/src/expr/core/src/window_function/state/aggregate.rs
+++ b/src/expr/impl/src/window_function/aggregate.rs
@@ -20,16 +20,18 @@ use risingwave_common::types::{DataType, Datum};
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::{bail, must_match};
 use risingwave_common_estimate_size::{EstimateSize, KvSize};
+use risingwave_expr::aggregate::{
+    AggCall, AggregateFunction, AggregateState as AggImplState, BoxedAggregateFunction,
+};
+use risingwave_expr::sig::FUNCTION_REGISTRY;
+use risingwave_expr::window_function::{
+    BoxedWindowState, FrameBounds, StateEvictHint, StateKey, StatePos, WindowFuncCall,
+    WindowFuncKind, WindowState,
+};
+use risingwave_expr::Result;
 use smallvec::SmallVec;
 
-use super::buffer::{RangeWindow, RowsWindow, WindowBuffer, WindowImpl};
-use super::{BoxedWindowState, StateEvictHint, StateKey, StatePos, WindowState};
-use crate::aggregate::{
-    AggArgs, AggCall, AggregateFunction, AggregateState as AggImplState, BoxedAggregateFunction,
-};
-use crate::sig::FUNCTION_REGISTRY;
-use crate::window_function::{FrameBounds, WindowFuncCall, WindowFuncKind};
-use crate::Result;
+use super::buffer::{RangeWindow, RowsWindow, SessionWindow, WindowBuffer, WindowImpl};
 
 type StateValue = SmallVec<[Datum; 2]>;
 
@@ -52,12 +54,7 @@ pub(super) fn new(call: &WindowFuncCall) -> Result<BoxedWindowState> {
     let arg_data_types = call.args.arg_types().to_vec();
     let agg_call = AggCall {
         kind: agg_kind,
-        args: match &call.args {
-            // convert args to [0] or [0, 1]
-            AggArgs::None => AggArgs::None,
-            AggArgs::Unary(data_type, _) => AggArgs::Unary(data_type.to_owned(), 0),
-            AggArgs::Binary(data_types, _) => AggArgs::Binary(data_types.to_owned(), [0, 1]),
-        },
+        args: call.args.clone(),
         return_type: call.return_type.clone(),
         column_orders: Vec::new(), // the input is already sorted
         // TODO(rc): support filter on window function call
@@ -65,6 +62,7 @@ pub(super) fn new(call: &WindowFuncCall) -> Result<BoxedWindowState> {
         // TODO(rc): support distinct on window function call? PG doesn't support it either.
         distinct: false,
         direct_args: vec![],
+        user_defined: None,
     };
     let agg_func_sig = FUNCTION_REGISTRY
         .get(agg_kind, &arg_data_types, &call.return_type)
@@ -72,7 +70,7 @@ pub(super) fn new(call: &WindowFuncCall) -> Result<BoxedWindowState> {
     let agg_func = agg_func_sig.build_aggregate(&agg_call)?;
     let (agg_impl, enable_delta) =
         if agg_func_sig.is_retractable() && call.frame.exclusion.is_no_others() {
-            let init_state = agg_func.create_state();
+            let init_state = agg_func.create_state()?;
             (AggImpl::Incremental(init_state), true)
         } else {
             (AggImpl::Full, false)
@@ -101,6 +99,17 @@ pub(super) fn new(call: &WindowFuncCall) -> Result<BoxedWindowState> {
             ),
             buffer_heap_size: KvSize::new(),
         }) as BoxedWindowState,
+        FrameBounds::Session(frame_bounds) => Box::new(AggregateState {
+            agg_func,
+            agg_impl,
+            arg_data_types,
+            buffer: WindowBuffer::<SessionWindow<StateValue>>::new(
+                SessionWindow::new(frame_bounds.clone()),
+                call.frame.exclusion,
+                enable_delta,
+            ),
+            buffer_heap_size: KvSize::new(),
+        }) as BoxedWindowState,
     };
     Ok(this)
 }
@@ -212,7 +221,7 @@ impl AggregatorWrapper<'_> {
     where
         V: AsRef<[Datum]>,
     {
-        let mut state = self.agg_func.create_state();
+        let mut state = self.agg_func.create_state()?;
         self.update(
             &mut state,
             values.into_iter().map(|args| (Op::Insert, args)),
diff --git a/src/expr/core/src/window_function/state/buffer.rs b/src/expr/impl/src/window_function/buffer.rs
similarity index 59%
rename from src/expr/core/src/window_function/state/buffer.rs
rename to src/expr/impl/src/window_function/buffer.rs
index ba064042c0021..bd1c10d162b23 100644
--- a/src/expr/core/src/window_function/state/buffer.rs
+++ b/src/expr/impl/src/window_function/buffer.rs
@@ -18,12 +18,12 @@ use std::ops::Range;
 use educe::Educe;
 use risingwave_common::array::Op;
 use risingwave_common::types::Sentinelled;
-use risingwave_common::util::memcmp_encoding;
+use risingwave_common::util::memcmp_encoding::{self, MemcmpEncoded};
+use risingwave_expr::window_function::{
+    FrameExclusion, RangeFrameBounds, RowsFrameBounds, SessionFrameBounds, StateKey,
+};
 
-use super::range_utils::range_except;
-use super::StateKey;
-use crate::window_function::state::range_utils::range_diff;
-use crate::window_function::{FrameExclusion, RangeFrameBounds, RowsFrameBounds};
+use super::range_utils::{range_diff, range_except};
 
 /// A common sliding window buffer.
 pub(super) struct WindowBuffer<W: WindowImpl> {
@@ -89,7 +89,7 @@ impl<W: WindowImpl> WindowBuffer<W> {
 
     /// Get the current window info.
     pub fn curr_window(&self) -> CurrWindow<'_, W::Key> {
-        let buffer_ref = BufferRef {
+        let window = BufferRef {
             buffer: &self.buffer,
             curr_idx: self.curr_idx,
             left_idx: self.left_idx,
@@ -97,8 +97,8 @@ impl<W: WindowImpl> WindowBuffer<W> {
         };
         CurrWindow {
             key: self.curr_key(),
-            preceding_saturated: self.window_impl.preceding_saturated(buffer_ref),
-            following_saturated: self.window_impl.following_saturated(buffer_ref),
+            preceding_saturated: self.window_impl.preceding_saturated(window),
+            following_saturated: self.window_impl.following_saturated(window),
         }
     }
 
@@ -148,7 +148,7 @@ impl<W: WindowImpl> WindowBuffer<W> {
         let old_outer = self.curr_window_outer();
 
         self.buffer.push_back(Entry { key, value });
-        self.recalculate_left_right();
+        self.recalculate_left_right(RecalculateHint::Append);
 
         if self.curr_delta.is_some() {
             self.maintain_delta(old_outer, self.curr_window_outer());
@@ -161,7 +161,7 @@ impl<W: WindowImpl> WindowBuffer<W> {
         let old_outer = self.curr_window_outer();
 
         self.curr_idx += 1;
-        self.recalculate_left_right();
+        self.recalculate_left_right(RecalculateHint::Slide);
 
         if self.curr_delta.is_some() {
             self.maintain_delta(old_outer, self.curr_window_outer());
@@ -171,6 +171,9 @@ impl<W: WindowImpl> WindowBuffer<W> {
         self.curr_idx -= min_needed_idx;
         self.left_idx -= min_needed_idx;
         self.right_excl_idx -= min_needed_idx;
+
+        self.window_impl.shift_indices(min_needed_idx);
+
         self.buffer
             .drain(0..min_needed_idx)
             .map(|Entry { key, value }| (key, value))
@@ -189,14 +192,14 @@ impl<W: WindowImpl> WindowBuffer<W> {
         }
     }
 
-    fn recalculate_left_right(&mut self) {
-        let buffer_ref = BufferRefMut {
+    fn recalculate_left_right(&mut self, hint: RecalculateHint) {
+        let window = BufferRefMut {
             buffer: &self.buffer,
             curr_idx: &mut self.curr_idx,
             left_idx: &mut self.left_idx,
             right_excl_idx: &mut self.right_excl_idx,
         };
-        self.window_impl.recalculate_left_right(buffer_ref);
+        self.window_impl.recalculate_left_right(window, hint);
     }
 }
 
@@ -218,6 +221,12 @@ pub(super) struct BufferRefMut<'a, K: Ord, V: Clone> {
     right_excl_idx: &'a mut usize,
 }
 
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub(super) enum RecalculateHint {
+    Append,
+    Slide,
+}
+
 /// A trait for sliding window implementations. This trait is used by [`WindowBuffer`] to
 /// determine the status of current window and how to slide the window.
 pub(super) trait WindowImpl {
@@ -227,14 +236,22 @@ pub(super) trait WindowImpl {
     /// Whether the preceding half of the current window is saturated.
     /// By "saturated" we mean that every row that is possible to be in the preceding half of the
     /// current window is already in the buffer.
-    fn preceding_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool;
+    fn preceding_saturated(&self, window: BufferRef<'_, Self::Key, Self::Value>) -> bool;
 
     /// Whether the following half of the current window is saturated.
-    fn following_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool;
+    fn following_saturated(&self, window: BufferRef<'_, Self::Key, Self::Value>) -> bool;
 
     /// Recalculate the left and right indices of the current window, according to the latest
-    /// `curr_idx`. The indices are indices in the buffer vector.
-    fn recalculate_left_right(&self, buffer_ref: BufferRefMut<'_, Self::Key, Self::Value>);
+    /// `curr_idx` and the hint.
+    fn recalculate_left_right(
+        &mut self,
+        window: BufferRefMut<'_, Self::Key, Self::Value>,
+        hint: RecalculateHint,
+    );
+
+    /// Shift the indices stored by the [`WindowImpl`] by `n` positions. This should be called
+    /// after evicting rows from the buffer.
+    fn shift_indices(&mut self, n: usize);
 }
 
 /// The sliding window implementation for `ROWS` frames.
@@ -258,8 +275,8 @@ impl<K: Ord, V: Clone> WindowImpl for RowsWindow<K, V> {
     type Key = K;
     type Value = V;
 
-    fn preceding_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool {
-        buffer_ref.curr_idx < buffer_ref.buffer.len() && {
+    fn preceding_saturated(&self, window: BufferRef<'_, Self::Key, Self::Value>) -> bool {
+        window.curr_idx < window.buffer.len() && {
             let start_off = self.frame_bounds.start.to_offset();
             if let Some(start_off) = start_off {
                 if start_off >= 0 {
@@ -269,9 +286,9 @@ impl<K: Ord, V: Clone> WindowImpl for RowsWindow<K, V> {
                     // the following can be simplified.
                     #[allow(clippy::nonminimal_bool)]
                     {
-                        assert!(buffer_ref.curr_idx >= buffer_ref.left_idx);
+                        assert!(window.curr_idx >= window.left_idx);
                     }
-                    buffer_ref.curr_idx - buffer_ref.left_idx >= start_off.unsigned_abs()
+                    window.curr_idx - window.left_idx >= start_off.unsigned_abs()
                 }
             } else {
                 false // unbounded frame start, never preceding-saturated
@@ -279,8 +296,8 @@ impl<K: Ord, V: Clone> WindowImpl for RowsWindow<K, V> {
         }
     }
 
-    fn following_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool {
-        buffer_ref.curr_idx < buffer_ref.buffer.len() && {
+    fn following_saturated(&self, window: BufferRef<'_, Self::Key, Self::Value>) -> bool {
+        window.curr_idx < window.buffer.len() && {
             let end_off = self.frame_bounds.end.to_offset();
             if let Some(end_off) = end_off {
                 if end_off <= 0 {
@@ -289,11 +306,11 @@ impl<K: Ord, V: Clone> WindowImpl for RowsWindow<K, V> {
                     // FIXME(rc): Ditto.
                     #[allow(clippy::nonminimal_bool)]
                     {
-                        assert!(buffer_ref.right_excl_idx > 0);
-                        assert!(buffer_ref.right_excl_idx > buffer_ref.curr_idx);
-                        assert!(buffer_ref.right_excl_idx <= buffer_ref.buffer.len());
+                        assert!(window.right_excl_idx > 0);
+                        assert!(window.right_excl_idx > window.curr_idx);
+                        assert!(window.right_excl_idx <= window.buffer.len());
                     }
-                    buffer_ref.right_excl_idx - 1 - buffer_ref.curr_idx >= end_off as usize
+                    window.right_excl_idx - 1 - window.curr_idx >= end_off as usize
                 }
             } else {
                 false // unbounded frame end, never following-saturated
@@ -301,39 +318,44 @@ impl<K: Ord, V: Clone> WindowImpl for RowsWindow<K, V> {
         }
     }
 
-    fn recalculate_left_right(&self, buffer_ref: BufferRefMut<'_, Self::Key, Self::Value>) {
-        if buffer_ref.buffer.is_empty() {
-            *buffer_ref.left_idx = 0;
-            *buffer_ref.right_excl_idx = 0;
+    fn recalculate_left_right(
+        &mut self,
+        window: BufferRefMut<'_, Self::Key, Self::Value>,
+        _hint: RecalculateHint,
+    ) {
+        if window.buffer.is_empty() {
+            *window.left_idx = 0;
+            *window.right_excl_idx = 0;
         }
 
         let start_off = self.frame_bounds.start.to_offset();
         let end_off = self.frame_bounds.end.to_offset();
         if let Some(start_off) = start_off {
-            let logical_left_idx = *buffer_ref.curr_idx as isize + start_off;
+            let logical_left_idx = *window.curr_idx as isize + start_off;
             if logical_left_idx >= 0 {
-                *buffer_ref.left_idx =
-                    std::cmp::min(logical_left_idx as usize, buffer_ref.buffer.len());
+                *window.left_idx = std::cmp::min(logical_left_idx as usize, window.buffer.len());
             } else {
-                *buffer_ref.left_idx = 0;
+                *window.left_idx = 0;
             }
         } else {
             // unbounded start
-            *buffer_ref.left_idx = 0;
+            *window.left_idx = 0;
         }
         if let Some(end_off) = end_off {
-            let logical_right_excl_idx = *buffer_ref.curr_idx as isize + end_off + 1;
+            let logical_right_excl_idx = *window.curr_idx as isize + end_off + 1;
             if logical_right_excl_idx >= 0 {
-                *buffer_ref.right_excl_idx =
-                    std::cmp::min(logical_right_excl_idx as usize, buffer_ref.buffer.len());
+                *window.right_excl_idx =
+                    std::cmp::min(logical_right_excl_idx as usize, window.buffer.len());
             } else {
-                *buffer_ref.right_excl_idx = 0;
+                *window.right_excl_idx = 0;
             }
         } else {
             // unbounded end
-            *buffer_ref.right_excl_idx = buffer_ref.buffer.len();
+            *window.right_excl_idx = window.buffer.len();
         }
     }
+
+    fn shift_indices(&mut self, _n: usize) {}
 }
 
 /// The sliding window implementation for `RANGE` frames.
@@ -355,36 +377,40 @@ impl<V: Clone> WindowImpl for RangeWindow<V> {
     type Key = StateKey;
     type Value = V;
 
-    fn preceding_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool {
-        buffer_ref.curr_idx < buffer_ref.buffer.len() && {
+    fn preceding_saturated(&self, window: BufferRef<'_, Self::Key, Self::Value>) -> bool {
+        window.curr_idx < window.buffer.len() && {
             // XXX(rc): It seems that preceding saturation is not important, may remove later.
             true
         }
     }
 
-    fn following_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool {
-        buffer_ref.curr_idx < buffer_ref.buffer.len()
+    fn following_saturated(&self, window: BufferRef<'_, Self::Key, Self::Value>) -> bool {
+        window.curr_idx < window.buffer.len()
             && {
                 // Left OK? (note that `left_idx` can be greater than `right_idx`)
                 // The following line checks whether the left value is the last one in the buffer.
                 // Here we adopt a conservative approach, which means we assume the next future value
                 // is likely to be the same as the last value in the current window, in which case
                 // we can't say the current window is saturated.
-                buffer_ref.left_idx < buffer_ref.buffer.len() /* non-zero */ - 1
+                window.left_idx < window.buffer.len() /* non-zero */ - 1
             }
             && {
                 // Right OK? Ditto.
-                buffer_ref.right_excl_idx < buffer_ref.buffer.len()
+                window.right_excl_idx < window.buffer.len()
             }
     }
 
-    fn recalculate_left_right(&self, buffer_ref: BufferRefMut<'_, Self::Key, Self::Value>) {
-        if buffer_ref.buffer.is_empty() {
-            *buffer_ref.left_idx = 0;
-            *buffer_ref.right_excl_idx = 0;
+    fn recalculate_left_right(
+        &mut self,
+        window: BufferRefMut<'_, Self::Key, Self::Value>,
+        _hint: RecalculateHint,
+    ) {
+        if window.buffer.is_empty() {
+            *window.left_idx = 0;
+            *window.right_excl_idx = 0;
         }
 
-        let Some(entry) = buffer_ref.buffer.get(*buffer_ref.curr_idx) else {
+        let Some(entry) = window.buffer.get(*window.curr_idx) else {
             // If the current index has been moved to a future position, we can't touch anything
             // because the next coming key may equal to the previous one which means the left and
             // right indices will be the same.
@@ -403,7 +429,7 @@ impl<V: Clone> WindowImpl for RangeWindow<V> {
             Sentinelled::Smallest => {
                 // unbounded frame start
                 assert_eq!(
-                    *buffer_ref.left_idx, 0,
+                    *window.left_idx, 0,
                     "for unbounded start, left index should always be 0"
                 );
             }
@@ -411,7 +437,7 @@ impl<V: Clone> WindowImpl for RangeWindow<V> {
                 // bounded, find the start position
                 let value_enc = memcmp_encoding::encode_value(value, self.frame_bounds.order_type)
                     .expect("no reason to fail here");
-                *buffer_ref.left_idx = buffer_ref
+                *window.left_idx = window
                     .buffer
                     .partition_point(|elem| elem.key.order_key < value_enc);
             }
@@ -421,29 +447,214 @@ impl<V: Clone> WindowImpl for RangeWindow<V> {
         match self.frame_bounds.frame_end_of(curr_order_value) {
             Sentinelled::Largest => {
                 // unbounded frame end
-                *buffer_ref.right_excl_idx = buffer_ref.buffer.len();
+                *window.right_excl_idx = window.buffer.len();
             }
             Sentinelled::Normal(value) => {
                 // bounded, find the end position
                 let value_enc = memcmp_encoding::encode_value(value, self.frame_bounds.order_type)
                     .expect("no reason to fail here");
-                *buffer_ref.right_excl_idx = buffer_ref
+                *window.right_excl_idx = window
                     .buffer
                     .partition_point(|elem| elem.key.order_key <= value_enc);
             }
             Sentinelled::Smallest => unreachable!("frame end never be UNBOUNDED PRECEDING"),
         }
     }
+
+    fn shift_indices(&mut self, _n: usize) {}
+}
+
+pub(super) struct SessionWindow<V: Clone> {
+    frame_bounds: SessionFrameBounds,
+    /// The latest session is the rightmost session in the buffer, which is updated during appending.
+    latest_session: Option<LatestSession>,
+    /// The sizes of recognized but not consumed sessions in the buffer. It's updated during appending.
+    /// The first element, if any, should be the size of the "current session window". When sliding,
+    /// the front should be popped.
+    recognized_session_sizes: VecDeque<usize>,
+    _phantom: std::marker::PhantomData<V>,
+}
+
+#[derive(Debug)]
+struct LatestSession {
+    /// The starting index of the latest session.
+    start_idx: usize,
+
+    /// Minimal next start means the minimal order value that can start a new session.
+    /// If a row has an order value less than this, it should be in the current session.
+    minimal_next_start: MemcmpEncoded,
+}
+
+impl<V: Clone> SessionWindow<V> {
+    pub fn new(frame_bounds: SessionFrameBounds) -> Self {
+        Self {
+            frame_bounds,
+            latest_session: None,
+            recognized_session_sizes: Default::default(),
+            _phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+impl<V: Clone> WindowImpl for SessionWindow<V> {
+    type Key = StateKey;
+    type Value = V;
+
+    fn preceding_saturated(&self, window: BufferRef<'_, Self::Key, Self::Value>) -> bool {
+        window.curr_idx < window.buffer.len() && {
+            // XXX(rc): It seems that preceding saturation is not important, may remove later.
+            true
+        }
+    }
+
+    fn following_saturated(&self, window: BufferRef<'_, Self::Key, Self::Value>) -> bool {
+        window.curr_idx < window.buffer.len() && {
+            // For session window, `left_idx` is always smaller than `right_excl_idx`.
+            assert!(window.left_idx <= window.curr_idx);
+            assert!(window.curr_idx < window.right_excl_idx);
+
+            // The following expression checks whether the current window is the latest session.
+            // If it is, we don't think it's saturated because the next row may be still in the
+            // same session. Otherwise, we can safely say it's saturated.
+            self.latest_session
+                .as_ref()
+                .map_or(false, |LatestSession { start_idx, .. }| {
+                    window.left_idx < *start_idx
+                })
+        }
+    }
+
+    fn recalculate_left_right(
+        &mut self,
+        window: BufferRefMut<'_, Self::Key, Self::Value>,
+        hint: RecalculateHint,
+    ) {
+        // Terms:
+        // - Session: A continuous range of rows among any two of which the difference of order values
+        //   is less than the session gap. This is a concept on the whole stream. Sessions are recognized
+        //   during appending.
+        // - Current window: The range of rows that are represented by the indices in `window`. It is a
+        //   status of the `WindowBuffer`. If the current window happens to be the last session in the
+        //   buffer, it will be updated during appending. Otherwise it will only be updated during sliding.
+
+        match hint {
+            RecalculateHint::Append => {
+                assert!(!window.buffer.is_empty()); // because we just appended a row
+                let appended_idx = window.buffer.len() - 1;
+                let appended_key = &window.buffer[appended_idx].key;
+
+                let minimal_next_start_of_appended = self.frame_bounds.minimal_next_start_of(
+                    memcmp_encoding::decode_value(
+                        &self.frame_bounds.order_data_type,
+                        &appended_key.order_key,
+                        self.frame_bounds.order_type,
+                    )
+                    .expect("no reason to fail here because we just encoded it in memory"),
+                );
+                let minimal_next_start_enc_of_appended = memcmp_encoding::encode_value(
+                    minimal_next_start_of_appended,
+                    self.frame_bounds.order_type,
+                )
+                .expect("no reason to fail here");
+
+                if let Some(LatestSession {
+                    ref start_idx,
+                    minimal_next_start,
+                }) = self.latest_session.as_mut()
+                {
+                    if &appended_key.order_key >= minimal_next_start {
+                        // the appended row starts a new session
+                        self.recognized_session_sizes
+                            .push_back(appended_idx - start_idx);
+                        self.latest_session = Some(LatestSession {
+                            start_idx: appended_idx,
+                            minimal_next_start: minimal_next_start_enc_of_appended,
+                        });
+                        // no need to update the current window because it's now corresponding
+                        // to some previous session
+                    } else {
+                        // the appended row belongs to the latest session
+                        *minimal_next_start = minimal_next_start_enc_of_appended;
+
+                        if *start_idx == *window.left_idx {
+                            // the current window is the latest session, we should extend it
+                            *window.right_excl_idx = appended_idx + 1;
+                        }
+                    }
+                } else {
+                    // no session yet, the current window should be empty
+                    let left_idx = *window.left_idx;
+                    let curr_idx = *window.curr_idx;
+                    let old_right_excl_idx = *window.right_excl_idx;
+                    assert_eq!(left_idx, curr_idx);
+                    assert_eq!(left_idx, old_right_excl_idx);
+                    assert_eq!(old_right_excl_idx, window.buffer.len() - 1);
+
+                    // now we put the first row into the current window
+                    *window.right_excl_idx = window.buffer.len();
+
+                    // and start to recognize the latest session
+                    self.latest_session = Some(LatestSession {
+                        start_idx: left_idx,
+                        minimal_next_start: minimal_next_start_enc_of_appended,
+                    });
+                }
+            }
+            RecalculateHint::Slide => {
+                let old_left_idx = *window.left_idx;
+                let new_curr_idx = *window.curr_idx;
+                let old_right_excl_idx = *window.right_excl_idx;
+
+                if new_curr_idx < old_right_excl_idx {
+                    // the current row is still in the current session window, no need to slide
+                } else {
+                    let old_session_size = self.recognized_session_sizes.pop_front();
+                    let next_session_size = self.recognized_session_sizes.front().copied();
+
+                    if let Some(old_session_size) = old_session_size {
+                        assert_eq!(old_session_size, old_right_excl_idx - old_left_idx);
+
+                        // slide the window to the next session
+                        if let Some(next_session_size) = next_session_size {
+                            // the next session is fully recognized, so we know the ending index
+                            *window.left_idx = old_right_excl_idx;
+                            *window.right_excl_idx = old_right_excl_idx + next_session_size;
+                        } else {
+                            // the next session is still in recognition, so we end the window at the end of buffer
+                            *window.left_idx = old_right_excl_idx;
+                            *window.right_excl_idx = window.buffer.len();
+                        }
+                    } else {
+                        // no recognized session yet, meaning the current window is the last session in the buffer
+                        assert_eq!(old_right_excl_idx, window.buffer.len());
+                        *window.left_idx = old_right_excl_idx;
+                        *window.right_excl_idx = old_right_excl_idx;
+                        self.latest_session = None;
+                    }
+                }
+            }
+        }
+    }
+
+    fn shift_indices(&mut self, n: usize) {
+        if let Some(LatestSession { start_idx, .. }) = self.latest_session.as_mut() {
+            *start_idx -= n;
+        }
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use itertools::Itertools;
-
-    use super::*;
-    use crate::window_function::FrameBound::{
+    use risingwave_common::row::OwnedRow;
+    use risingwave_common::types::{DataType, ScalarImpl};
+    use risingwave_common::util::sort_util::OrderType;
+    use risingwave_expr::window_function::FrameBound::{
         CurrentRow, Following, Preceding, UnboundedFollowing, UnboundedPreceding,
     };
+    use risingwave_expr::window_function::SessionFrameGap;
+
+    use super::*;
 
     #[test]
     fn test_rows_frame_unbounded_preceding_to_current_row() {
@@ -735,4 +946,114 @@ mod tests {
             vec!["hello"]
         );
     }
+
+    #[test]
+    fn test_session_frame() {
+        let order_data_type = DataType::Int64;
+        let order_type = OrderType::ascending();
+        let gap_data_type = DataType::Int64;
+
+        let mut buffer = WindowBuffer::<SessionWindow<_>>::new(
+            SessionWindow::new(SessionFrameBounds {
+                order_data_type: order_data_type.clone(),
+                order_type,
+                gap_data_type: gap_data_type.clone(),
+                gap: SessionFrameGap::new_for_test(
+                    ScalarImpl::Int64(5),
+                    &order_data_type,
+                    &gap_data_type,
+                ),
+            }),
+            FrameExclusion::NoOthers,
+            true,
+        );
+
+        let key = |key: i64| -> StateKey {
+            StateKey {
+                order_key: memcmp_encoding::encode_value(&Some(ScalarImpl::from(key)), order_type)
+                    .unwrap(),
+                pk: OwnedRow::empty().into(),
+            }
+        };
+
+        assert!(buffer.curr_key().is_none());
+
+        buffer.append(key(1), "hello");
+        buffer.append(key(3), "session");
+        let window = buffer.curr_window();
+        assert_eq!(window.key, Some(&key(1)));
+        assert!(window.preceding_saturated);
+        assert!(!window.following_saturated);
+        assert_eq!(
+            buffer.curr_window_values().cloned().collect_vec(),
+            vec!["hello", "session"]
+        );
+
+        buffer.append(key(8), "window"); // start a new session
+        let window = buffer.curr_window();
+        assert!(window.following_saturated);
+        assert_eq!(
+            buffer.curr_window_values().cloned().collect_vec(),
+            vec!["hello", "session"]
+        );
+
+        buffer.append(key(15), "and");
+        buffer.append(key(16), "world");
+        assert_eq!(
+            buffer.curr_window_values().cloned().collect_vec(),
+            vec!["hello", "session"]
+        );
+
+        let removed_keys = buffer.slide().map(|(k, _)| k).collect_vec();
+        assert!(removed_keys.is_empty());
+        let window = buffer.curr_window();
+        assert_eq!(window.key, Some(&key(3)));
+        assert!(window.preceding_saturated);
+        assert!(window.following_saturated);
+        assert_eq!(
+            buffer.curr_window_values().cloned().collect_vec(),
+            vec!["hello", "session"]
+        );
+
+        let removed_keys = buffer.slide().map(|(k, _)| k).collect_vec();
+        assert_eq!(removed_keys, vec![key(1), key(3)]);
+        assert_eq!(buffer.smallest_key(), Some(&key(8)));
+        let window = buffer.curr_window();
+        assert_eq!(window.key, Some(&key(8)));
+        assert!(window.preceding_saturated);
+        assert!(window.following_saturated);
+        assert_eq!(
+            buffer.curr_window_values().cloned().collect_vec(),
+            vec!["window"]
+        );
+
+        let removed_keys = buffer.slide().map(|(k, _)| k).collect_vec();
+        assert_eq!(removed_keys, vec![key(8)]);
+        assert_eq!(buffer.smallest_key(), Some(&key(15)));
+        let window = buffer.curr_window();
+        assert_eq!(window.key, Some(&key(15)));
+        assert!(window.preceding_saturated);
+        assert!(!window.following_saturated);
+        assert_eq!(
+            buffer.curr_window_values().cloned().collect_vec(),
+            vec!["and", "world"]
+        );
+
+        let removed_keys = buffer.slide().map(|(k, _)| k).collect_vec();
+        assert!(removed_keys.is_empty());
+        assert_eq!(buffer.curr_key(), Some(&key(16)));
+        assert_eq!(
+            buffer.curr_window_values().cloned().collect_vec(),
+            vec!["and", "world"]
+        );
+
+        let removed_keys = buffer.slide().map(|(k, _)| k).collect_vec();
+        assert_eq!(removed_keys, vec![key(15), key(16)]);
+        assert!(buffer.curr_key().is_none());
+        assert!(buffer
+            .curr_window_values()
+            .cloned()
+            .collect_vec()
+            .is_empty());
+    }
 }
diff --git a/src/expr/impl/src/window_function/mod.rs b/src/expr/impl/src/window_function/mod.rs
new file mode 100644
index 0000000000000..1153a9111a1ea
--- /dev/null
+++ b/src/expr/impl/src/window_function/mod.rs
@@ -0,0 +1,45 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use itertools::Itertools;
+use risingwave_expr::window_function::{
+    BoxedWindowState, WindowFuncCall, WindowFuncKind, WINDOW_STATE_BUILDERS,
+};
+use risingwave_expr::{ExprError, Result};
+
+mod aggregate;
+mod buffer;
+mod range_utils;
+mod rank;
+
+#[linkme::distributed_slice(WINDOW_STATE_BUILDERS)]
+fn create_window_state_impl(call: &WindowFuncCall) -> Result<BoxedWindowState> {
+    assert!(call.frame.bounds.validate().is_ok());
+
+    use WindowFuncKind::*;
+    Ok(match call.kind {
+        RowNumber => Box::new(rank::RankState::<rank::RowNumber>::new(call)),
+        Rank => Box::new(rank::RankState::<rank::Rank>::new(call)),
+        DenseRank => Box::new(rank::RankState::<rank::DenseRank>::new(call)),
+        Aggregate(_) => aggregate::new(call)?,
+        kind => {
+            return Err(ExprError::UnsupportedFunction(format!(
+                "{}({}) -> {}",
+                kind,
+                call.args.arg_types().iter().format(", "),
+                &call.return_type,
+            )));
+        }
+    })
+}
diff --git a/src/expr/core/src/window_function/state/range_utils.rs b/src/expr/impl/src/window_function/range_utils.rs
similarity index 100%
rename from src/expr/core/src/window_function/state/range_utils.rs
rename to src/expr/impl/src/window_function/range_utils.rs
diff --git a/src/expr/core/src/window_function/state/rank.rs b/src/expr/impl/src/window_function/rank.rs
similarity index 95%
rename from src/expr/core/src/window_function/state/rank.rs
rename to src/expr/impl/src/window_function/rank.rs
index 238af18699315..9b25a15c35590 100644
--- a/src/expr/core/src/window_function/state/rank.rs
+++ b/src/expr/impl/src/window_function/rank.rs
@@ -18,12 +18,13 @@ use risingwave_common::types::Datum;
 use risingwave_common::util::memcmp_encoding::MemcmpEncoded;
 use risingwave_common_estimate_size::collections::EstimatedVecDeque;
 use risingwave_common_estimate_size::EstimateSize;
+use risingwave_expr::window_function::{
+    StateEvictHint, StateKey, StatePos, WindowFuncCall, WindowState,
+};
+use risingwave_expr::Result;
 use smallvec::SmallVec;
 
 use self::private::RankFuncCount;
-use super::{StateEvictHint, StateKey, StatePos, WindowState};
-use crate::window_function::WindowFuncCall;
-use crate::Result;
 
 mod private {
     use super::*;
@@ -34,7 +35,7 @@ mod private {
 }
 
 #[derive(Default, EstimateSize)]
-pub struct RowNumber {
+pub(super) struct RowNumber {
     prev_rank: i64,
 }
 
@@ -47,7 +48,7 @@ impl RankFuncCount for RowNumber {
 }
 
 #[derive(EstimateSize)]
-pub struct Rank {
+pub(super) struct Rank {
     prev_order_key: Option<MemcmpEncoded>,
     prev_rank: i64,
     prev_pos_in_peer_group: i64,
@@ -83,7 +84,7 @@ impl RankFuncCount for Rank {
 }
 
 #[derive(Default, EstimateSize)]
-pub struct DenseRank {
+pub(super) struct DenseRank {
     prev_order_key: Option<MemcmpEncoded>,
     prev_rank: i64,
 }
@@ -107,7 +108,7 @@ impl RankFuncCount for DenseRank {
 
 /// Generic state for rank window functions including `row_number`, `rank` and `dense_rank`.
 #[derive(EstimateSize)]
-pub struct RankState<RF: RankFuncCount> {
+pub(super) struct RankState<RF: RankFuncCount> {
     /// First state key of the partition.
     first_key: Option<StateKey>,
     /// State keys that are waiting to be outputted.
@@ -176,10 +177,10 @@ mod tests {
     use risingwave_common::types::{DataType, ScalarImpl};
     use risingwave_common::util::memcmp_encoding;
     use risingwave_common::util::sort_util::OrderType;
+    use risingwave_expr::aggregate::AggArgs;
+    use risingwave_expr::window_function::{Frame, FrameBound, WindowFuncKind};
 
     use super::*;
-    use crate::aggregate::AggArgs;
-    use crate::window_function::{Frame, FrameBound, WindowFuncKind};
 
     fn create_state_key(order: i64, pk: i64) -> StateKey {
         StateKey {
@@ -197,7 +198,7 @@ mod tests {
     fn test_rank_state_bad_use() {
         let call = WindowFuncCall {
             kind: WindowFuncKind::RowNumber,
-            args: AggArgs::None,
+            args: AggArgs::default(),
             return_type: DataType::Int64,
             frame: Frame::rows(
                 FrameBound::UnboundedPreceding,
@@ -214,7 +215,7 @@ mod tests {
     fn test_row_number_state() {
         let call = WindowFuncCall {
             kind: WindowFuncKind::RowNumber,
-            args: AggArgs::None,
+            args: AggArgs::default(),
             return_type: DataType::Int64,
             frame: Frame::rows(
                 FrameBound::UnboundedPreceding,
@@ -256,7 +257,7 @@ mod tests {
     fn test_rank_state() {
         let call = WindowFuncCall {
             kind: WindowFuncKind::Rank,
-            args: AggArgs::None,
+            args: AggArgs::default(),
             return_type: DataType::Int64,
             frame: Frame::rows(
                 FrameBound::UnboundedPreceding,
@@ -297,7 +298,7 @@ mod tests {
     fn test_dense_rank_state() {
         let call = WindowFuncCall {
             kind: WindowFuncKind::DenseRank,
-            args: AggArgs::None,
+            args: AggArgs::default(),
             return_type: DataType::Int64,
             frame: Frame::rows(
                 FrameBound::UnboundedPreceding,
diff --git a/src/expr/macro/src/gen.rs b/src/expr/macro/src/gen.rs
index ca3203033a63e..8bf39c17e763b 100644
--- a/src/expr/macro/src/gen.rs
+++ b/src/expr/macro/src/gen.rs
@@ -342,10 +342,45 @@ impl FunctionAttr {
             // no prebuilt argument
             (None, _) => quote! {},
         };
-        let variadic_args = variadic.then(|| quote! { variadic_row, });
+        let variadic_args = variadic.then(|| quote! { &variadic_row, });
         let context = user_fn.context.then(|| quote! { &self.context, });
         let writer = user_fn.write.then(|| quote! { &mut writer, });
         let await_ = user_fn.async_.then(|| quote! { .await });
+
+        let record_error = {
+            // Uniform arguments into `DatumRef`.
+            #[allow(clippy::disallowed_methods)] // allow zip
+            let inputs_args = inputs
+                .iter()
+                .zip(user_fn.args_option.iter())
+                .map(|(input, opt)| {
+                    if *opt {
+                        quote! { #input.map(|s| ScalarRefImpl::from(s)) }
+                    } else {
+                        quote! { Some(ScalarRefImpl::from(#input)) }
+                    }
+                });
+            let inputs_args = quote! {
+                let args: &[DatumRef<'_>] = &[#(#inputs_args),*];
+                let args = args.iter().copied();
+            };
+            let var_args = variadic.then(|| {
+                quote! {
+                    let args = args.chain(variadic_row.iter());
+                }
+            });
+
+            quote! {
+                #inputs_args
+                #var_args
+                errors.push(ExprError::function(
+                    stringify!(#fn_name),
+                    args,
+                    e,
+                ));
+            }
+        };
+
         // call the user defined function
         // inputs: [ Option<impl ScalarRef> ]
         let mut output = quote! { #fn_name #generic(
@@ -365,13 +400,19 @@ impl FunctionAttr {
             ReturnTypeKind::Result => quote! {
                 match #output {
                     Ok(x) => Some(x),
-                    Err(e) => { errors.push(e); None }
+                    Err(e) => {
+                        #record_error
+                        None
+                    }
                 }
             },
             ReturnTypeKind::ResultOption => quote! {
                 match #output {
                     Ok(x) => x,
-                    Err(e) => { errors.push(e); None }
+                    Err(e) => {
+                        #record_error
+                        None
+                    }
                 }
             },
         };
@@ -484,10 +525,6 @@ impl FunctionAttr {
             }
         } else {
             // no optimization
-            let array_zip = match children_indices.len() {
-                0 => quote! { std::iter::repeat(()).take(input.capacity()) },
-                _ => quote! { multizip((#(#arrays.iter(),)*)) },
-            };
             let let_variadic = variadic.then(|| {
                 quote! {
                     let variadic_row = variadic_input.row_at_unchecked_vis(i);
@@ -497,18 +534,18 @@ impl FunctionAttr {
                 let mut builder = #builder_type::with_type(input.capacity(), self.context.return_type.clone());
 
                 if input.is_compacted() {
-                    for (i, (#(#inputs,)*)) in #array_zip.enumerate() {
+                    for i in 0..input.capacity() {
+                        #(let #inputs = unsafe { #arrays.value_at_unchecked(i) };)*
                         #let_variadic
                         #append_output
                     }
                 } else {
-                    // allow using `zip` for performance
-                    #[allow(clippy::disallowed_methods)]
-                    for (i, ((#(#inputs,)*), visible)) in #array_zip.zip(input.visibility().iter()).enumerate() {
-                        if !visible {
+                    for i in 0..input.capacity() {
+                        if unsafe { !input.visibility().is_set_unchecked(i) } {
                             builder.append_null();
                             continue;
                         }
+                        #(let #inputs = unsafe { #arrays.value_at_unchecked(i) };)*
                         #let_variadic
                         #append_output
                     }
@@ -725,15 +762,15 @@ impl FunctionAttr {
         };
         let create_state = if custom_state.is_some() {
             quote! {
-                fn create_state(&self) -> AggregateState {
-                    AggregateState::Any(Box::<#state_type>::default())
+                fn create_state(&self) -> Result<AggregateState> {
+                    Ok(AggregateState::Any(Box::<#state_type>::default()))
                 }
             }
         } else if let Some(state) = &self.init_state {
             let state: TokenStream2 = state.parse().unwrap();
             quote! {
-                fn create_state(&self) -> AggregateState {
-                    AggregateState::Datum(Some(#state.into()))
+                fn create_state(&self) -> Result<AggregateState> {
+                    Ok(AggregateState::Datum(Some(#state.into())))
                 }
             }
         } else {
@@ -1085,11 +1122,29 @@ impl FunctionAttr {
         let iter = quote! { #fn_name(#(#inputs,)* #prebuilt_arg #context) };
         let mut iter = match user_fn.return_type_kind {
             ReturnTypeKind::T => quote! { #iter },
-            ReturnTypeKind::Result => quote! { #iter? },
-            ReturnTypeKind::Option => quote! { if let Some(it) = #iter { it } else { continue; } },
-            ReturnTypeKind::ResultOption => {
-                quote! { if let Some(it) = #iter? { it } else { continue; } }
-            }
+            ReturnTypeKind::Option => quote! { match #iter {
+                Some(it) => it,
+                None => continue,
+            } },
+            ReturnTypeKind::Result => quote! { match #iter {
+                Ok(it) => it,
+                Err(e) => {
+                    index_builder.append(Some(i as i32));
+                    #(#builders.append_null();)*
+                    error_builder.append_display(Some(e.as_report()));
+                    continue;
+                }
+            } },
+            ReturnTypeKind::ResultOption => quote! { match #iter {
+                Ok(Some(it)) => it,
+                Ok(None) => continue,
+                Err(e) => {
+                    index_builder.append(Some(i as i32));
+                    #(#builders.append_null();)*
+                    error_builder.append_display(Some(e.as_report()));
+                    continue;
+                }
+            } },
         };
         // if user function accepts non-option arguments, we assume the function
         // returns empty on null input, so we need to unwrap the inputs before calling.
@@ -1116,11 +1171,24 @@ impl FunctionAttr {
                 "expect `impl Iterator` in return type",
             )
         })?;
-        let output = match iterator_item_type {
-            ReturnTypeKind::T => quote! { Some(output) },
-            ReturnTypeKind::Option => quote! { output },
-            ReturnTypeKind::Result => quote! { Some(output?) },
-            ReturnTypeKind::ResultOption => quote! { output? },
+        let append_output = match iterator_item_type {
+            ReturnTypeKind::T => quote! {
+                let (#(#outputs),*) = output;
+                #(#builders.append(#optioned_outputs);)* error_builder.append_null();
+            },
+            ReturnTypeKind::Option => quote! { match output {
+                Some((#(#outputs),*)) => { #(#builders.append(#optioned_outputs);)* error_builder.append_null(); }
+                None => { #(#builders.append_null();)* error_builder.append_null(); }
+            } },
+            ReturnTypeKind::Result => quote! { match output {
+                Ok((#(#outputs),*)) => { #(#builders.append(#optioned_outputs);)* error_builder.append_null(); }
+                Err(e) => { #(#builders.append_null();)* error_builder.append_display(Some(e.as_report())); }
+            } },
+            ReturnTypeKind::ResultOption => quote! { match output {
+                Ok(Some((#(#outputs),*))) => { #(#builders.append(#optioned_outputs);)* error_builder.append_null(); }
+                Ok(None) => { #(#builders.append_null();)* error_builder.append_null(); }
+                Err(e) => { #(#builders.append_null();)* error_builder.append_display(Some(e.as_report())); }
+            } },
         };
 
         Ok(quote! {
@@ -1178,24 +1246,28 @@ impl FunctionAttr {
 
                         let mut index_builder = I32ArrayBuilder::new(self.chunk_size);
                         #(let mut #builders = #builder_types::with_type(self.chunk_size, #return_types);)*
+                        let mut error_builder = Utf8ArrayBuilder::new(self.chunk_size);
 
-                        for (i, ((#(#inputs,)*), visible)) in multizip((#(#arrays.iter(),)*)).zip_eq_fast(input.visibility().iter()).enumerate() {
-                            if !visible {
+                        for i in 0..input.capacity() {
+                            if unsafe { !input.visibility().is_set_unchecked(i) } {
                                 continue;
                             }
+                            #(let #inputs = unsafe { #arrays.value_at_unchecked(i) };)*
                             for output in #iter {
                                 index_builder.append(Some(i as i32));
-                                match #output {
-                                    Some((#(#outputs),*)) => { #(#builders.append(#optioned_outputs);)* }
-                                    None => { #(#builders.append_null();)* }
-                                }
+                                #append_output
 
                                 if index_builder.len() == self.chunk_size {
                                     let len = index_builder.len();
                                     let index_array = std::mem::replace(&mut index_builder, I32ArrayBuilder::new(self.chunk_size)).finish().into_ref();
                                     let value_arrays = [#(std::mem::replace(&mut #builders, #builder_types::with_type(self.chunk_size, #return_types)).finish().into_ref()),*];
                                     #build_value_array
-                                    yield DataChunk::new(vec![index_array, value_array], self.chunk_size);
+                                    let error_array = std::mem::replace(&mut error_builder, Utf8ArrayBuilder::new(self.chunk_size)).finish().into_ref();
+                                    if error_array.null_bitmap().any() {
+                                        yield DataChunk::new(vec![index_array, value_array, error_array], self.chunk_size);
+                                    } else {
+                                        yield DataChunk::new(vec![index_array, value_array], self.chunk_size);
+                                    }
                                 }
                             }
                         }
@@ -1205,7 +1277,12 @@ impl FunctionAttr {
                             let index_array = index_builder.finish().into_ref();
                             let value_arrays = [#(#builders.finish().into_ref()),*];
                             #build_value_array
-                            yield DataChunk::new(vec![index_array, value_array], len);
+                            let error_array = error_builder.finish().into_ref();
+                            if error_array.null_bitmap().any() {
+                                yield DataChunk::new(vec![index_array, value_array, error_array], len);
+                            } else {
+                                yield DataChunk::new(vec![index_array, value_array], len);
+                            }
                         }
                     }
                 }
diff --git a/src/expr/udf/Cargo.toml b/src/expr/udf/Cargo.toml
deleted file mode 100644
index b17ad7acadfc1..0000000000000
--- a/src/expr/udf/Cargo.toml
+++ /dev/null
@@ -1,35 +0,0 @@
-[package]
-name = "risingwave_udf"
-version = "0.1.0"
-edition = "2021"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[package.metadata.cargo-machete]
-ignored = ["workspace-hack"]
-
-[package.metadata.cargo-udeps.ignore]
-normal = ["workspace-hack"]
-
-[dependencies]
-arrow-array = { workspace = true }
-arrow-flight = { workspace = true }
-arrow-schema = { workspace = true }
-arrow-select = { workspace = true }
-cfg-or-panic = "0.2"
-futures = "0.3"
-futures-util = "0.3.28"
-ginepro = "0.7.0"
-prometheus = "0.13"
-risingwave_common = { workspace = true }
-static_assertions = "1"
-thiserror = "1"
-thiserror-ext = { workspace = true }
-tokio = { version = "0.2", package = "madsim-tokio", features = [
-    "rt",
-    "macros",
-] }
-tonic = { workspace = true }
-tracing = "0.1"
-
-[lints]
-workspace = true
diff --git a/src/expr/udf/README-js.md b/src/expr/udf/README-js.md
deleted file mode 100644
index 902bce4ef52ee..0000000000000
--- a/src/expr/udf/README-js.md
+++ /dev/null
@@ -1,83 +0,0 @@
-# Use UDFs in JavaScript
-
-This article provides a step-by-step guide for defining JavaScript functions in RisingWave.
-
-JavaScript code is inlined in `CREATE FUNCTION` statement and then run on the embedded QuickJS virtual machine in RisingWave. It does not support access to external networks and is limited to computational tasks only.
-Compared to other languages, JavaScript UDFs offer the easiest way to define UDFs in RisingWave.
-
-## Define your functions
-
-You can use the `CREATE FUNCTION` statement to create JavaScript UDFs. The syntax is as follows:
-
-```sql
-CREATE FUNCTION function_name ( arg_name arg_type [, ...] )
-    [ RETURNS return_type | RETURNS TABLE ( column_name column_type [, ...] ) ]
-    LANGUAGE javascript
-    AS [ $$ function_body $$ | 'function_body' ];
-```
-
-The argument names you define can be used in the function body. For example:
-
-```sql
-CREATE FUNCTION gcd(a int, b int) RETURNS int LANGUAGE javascript AS $$
-    if(a == null || b == null) {
-        return null;
-    }
-    while (b != 0) {
-        let t = b;
-        b = a % b;
-        a = t;
-    }
-    return a;
-$$;
-```
-
-The correspondence between SQL types and JavaScript types can be found in the [appendix table](#appendix-type-mapping). You need to ensure that the type of the return value is either `null` or consistent with the type in the `RETURNS` clause.
-
-If the function you define returns a table, you need to use the `yield` statement to return the data of each row. For example:
-
-```sql
-CREATE FUNCTION series(n int) RETURNS TABLE (x int) LANGUAGE javascript AS $$
-    for(let i = 0; i < n; i++) {
-        yield i;
-    }
-$$;
-```
-
-## Use your functions
-
-Once the UDFs are created in RisingWave, you can use them in SQL queries just like any built-in functions. For example:
-
-```sql
-SELECT gcd(25, 15);
-SELECT * from series(5);
-```
-
-## Appendix: Type Mapping
-
-The following table shows the type mapping between SQL and JavaScript:
-
-| SQL Type              | JS Type       | Note                  |
-| --------------------- | ------------- | --------------------- |
-| boolean               | boolean       |                       |
-| smallint              | number        |                       |
-| int                   | number        |                       |
-| bigint                | number        |                       |
-| real                  | number        |                       |
-| double precision      | number        |                       |
-| decimal               | BigDecimal    |                       |
-| date                  |               | not supported yet     |
-| time                  |               | not supported yet     |
-| timestamp             |               | not supported yet     |
-| timestamptz           |               | not supported yet     |
-| interval              |               | not supported yet     |
-| varchar               | string        |                       |
-| bytea                 | Uint8Array    |                       |
-| jsonb                 | null, boolean, number, string, array or object | `JSON.parse(string)`  |
-| smallint[]            | Int16Array    |                       |
-| int[]                 | Int32Array    |                       |
-| bigint[]              | BigInt64Array |                       |
-| real[]                | Float32Array  |                       |
-| double precision[]    | Float64Array  |                       |
-| others[]              | array         |                       |
-| struct<..>            | object        |                       |
diff --git a/src/expr/udf/README.md b/src/expr/udf/README.md
deleted file mode 100644
index d9428cc547249..0000000000000
--- a/src/expr/udf/README.md
+++ /dev/null
@@ -1,118 +0,0 @@
-# Use UDFs in Rust
-
-This article provides a step-by-step guide for defining Rust functions in RisingWave.
-
-Rust functions are compiled into WebAssembly modules and then run on the embedded WebAssembly virtual machine in RisingWave. Compared to Python and Java, Rust UDFs offer **higher performance** (near native) and are **managed by the RisingWave kernel**, eliminating the need for additional maintenance. However, since they run embedded in the kernel, for security reasons, Rust UDFs currently **do not support access to external networks and are limited to computational tasks only**, with restricted CPU and memory resources. Therefore, we recommend using Rust UDFs for **computationally intensive tasks**, such as packet parsing and format conversion.
-
-## Prerequisites
-
-- Ensure that you have [Rust toolchain](https://rustup.rs) (stable channel) installed on your computer.
-- Ensure that the Rust standard library for `wasm32-wasi` target is installed:
-    ```shell
-    rustup target add wasm32-wasi
-    ```
-
-## 1. Create a project
-
-Create a Rust project named `udf`:
-
-```shell
-cargo new --lib udf
-cd udf
-```
-
-Add the following lines to `Cargo.toml`:
-
-```toml
-[lib]
-crate-type = ["cdylib"]
-
-[dependencies]
-arrow-udf = "0.1"
-```
-
-## 2. Define your functions
-
-In `src/lib.rs`, define your functions using the `function` macro:
-
-```rust
-use arrow_udf::function;
-
-// define a scalar function
-#[function("gcd(int, int) -> int")]
-fn gcd(mut x: i32, mut y: i32) -> i32 {
-    while y != 0 {
-        (x, y) = (y, x % y);
-    }
-    x
-}
-
-// define a table function
-#[function("series(int) -> setof int")]
-fn series(n: i32) -> impl Iterator<Item = i32> {
-    0..n
-}
-```
-
-You can find more usages in the [documentation](https://docs.rs/arrow_udf/0.1.0/arrow_udf/attr.function.html) and more examples in the [tests](https://github.com/risingwavelabs/arrow-udf/blob/main/arrow-udf/tests/tests.rs).
-
-Currently we only support a limited set of data types. `timestamptz` and complex array types are not supported yet.
-
-## 3. Build the project
-
-Build your functions into a WebAssembly module:
-
-```shell
-cargo build --release --target wasm32-wasi
-```
-
-You can find the generated WASM module at `target/wasm32-wasi/release/udf.wasm`.
-
-Optional: It is recommended to strip the binary to reduce its size:
-
-```shell
-# Install wasm-tools
-cargo install wasm-tools
-
-# Strip the binary
-wasm-tools strip ./target/wasm32-wasi/release/udf.wasm > udf.wasm
-```
-
-## 4. Declare your functions in RisingWave
-
-In RisingWave, use the `CREATE FUNCTION` command to declare the functions you defined.
-
-There are two ways to load the WASM module:
-
-1. The WASM binary can be embedded in the SQL statement using the base64 encoding.
-You can use the following shell script to encode the binary and generate the SQL statement:
-    ```shell
-    encoded=$(base64 -i udf.wasm)
-    sql="CREATE FUNCTION gcd(int, int) RETURNS int LANGUAGE wasm USING BASE64 '$encoded';"
-    echo "$sql" > create_function.sql
-    ```
-    When created successfully, the WASM binary will be automatically uploaded to the object store.
-
-2. The WASM binary can be loaded from the object store.
-    ```sql
-    CREATE FUNCTION gcd(int, int) RETURNS int
-    LANGUAGE wasm USING LINK 's3://bucket/path/to/udf.wasm';
-
-    CREATE FUNCTION series(int) RETURNS TABLE (x int)
-    LANGUAGE wasm USING LINK 's3://bucket/path/to/udf.wasm';
-    ```
-
-    Or if you run RisingWave locally, you can use the local file system:
-    ```sql
-    CREATE FUNCTION gcd(int, int) RETURNS int
-    LANGUAGE wasm USING LINK 'fs://path/to/udf.wasm';
-    ```
-
-## 5. Use your functions in RisingWave
-
-Once the UDFs are created in RisingWave, you can use them in SQL queries just like any built-in functions. For example:
-
-```sql
-SELECT gcd(25, 15);
-SELECT series(5);
-```
diff --git a/src/expr/udf/examples/client.rs b/src/expr/udf/examples/client.rs
deleted file mode 100644
index 92f93ae13614e..0000000000000
--- a/src/expr/udf/examples/client.rs
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use arrow_array::{Int32Array, RecordBatch};
-use arrow_schema::{DataType, Field, Schema};
-use risingwave_udf::ArrowFlightUdfClient;
-
-#[tokio::main]
-async fn main() {
-    let addr = "http://localhost:8815";
-    let client = ArrowFlightUdfClient::connect(addr).await.unwrap();
-
-    // build `RecordBatch` to send (equivalent to our `DataChunk`)
-    let array1 = Arc::new(Int32Array::from_iter(vec![1, 6, 10]));
-    let array2 = Arc::new(Int32Array::from_iter(vec![3, 4, 15]));
-    let array3 = Arc::new(Int32Array::from_iter(vec![6, 8, 3]));
-    let input2_schema = Schema::new(vec![
-        Field::new("a", DataType::Int32, true),
-        Field::new("b", DataType::Int32, true),
-    ]);
-    let input3_schema = Schema::new(vec![
-        Field::new("a", DataType::Int32, true),
-        Field::new("b", DataType::Int32, true),
-        Field::new("c", DataType::Int32, true),
-    ]);
-    let output_schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]);
-
-    // check function
-    client
-        .check("gcd", &input2_schema, &output_schema)
-        .await
-        .unwrap();
-    client
-        .check("gcd3", &input3_schema, &output_schema)
-        .await
-        .unwrap();
-
-    let input2 = RecordBatch::try_new(
-        Arc::new(input2_schema),
-        vec![array1.clone(), array2.clone()],
-    )
-    .unwrap();
-
-    let output = client
-        .call("gcd", input2)
-        .await
-        .expect("failed to call function");
-
-    println!("{:?}", output);
-
-    let input3 = RecordBatch::try_new(
-        Arc::new(input3_schema),
-        vec![array1.clone(), array2.clone(), array3.clone()],
-    )
-    .unwrap();
-
-    let output = client
-        .call("gcd3", input3)
-        .await
-        .expect("failed to call function");
-
-    println!("{:?}", output);
-}
diff --git a/src/expr/udf/python/.gitignore b/src/expr/udf/python/.gitignore
deleted file mode 100644
index 75b18b1dc1919..0000000000000
--- a/src/expr/udf/python/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-/dist
-/risingwave.egg-info
diff --git a/src/expr/udf/python/CHANGELOG.md b/src/expr/udf/python/CHANGELOG.md
deleted file mode 100644
index a20411e69d83e..0000000000000
--- a/src/expr/udf/python/CHANGELOG.md
+++ /dev/null
@@ -1,37 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [Unreleased]
-
-## [0.1.1] - 2023-12-06
-
-### Fixed
-
-- Fix decimal type output.
-
-## [0.1.0] - 2023-12-01
-
-### Fixed
-
-- Fix unconstrained decimal type.
-
-## [0.0.12] - 2023-11-28
-
-### Changed
-
-- Change the default struct field name to `f{i}`.
-
-### Fixed
-
-- Fix parsing nested struct type.
-
-
-## [0.0.11] - 2023-11-06
-
-### Fixed
-
-- Hook SIGTERM to stop the UDF server gracefully.
diff --git a/src/expr/udf/python/README.md b/src/expr/udf/python/README.md
deleted file mode 100644
index d1655be05350b..0000000000000
--- a/src/expr/udf/python/README.md
+++ /dev/null
@@ -1,112 +0,0 @@
-# RisingWave Python UDF SDK
-
-This library provides a Python SDK for creating user-defined functions (UDF) in [RisingWave](https://www.risingwave.com/).
-
-For a detailed guide on how to use Python UDF in RisingWave, please refer to [this doc](https://docs.risingwave.com/docs/current/udf-python/).
-
-## Introduction
-
-RisingWave supports user-defined functions implemented as external functions.
-With the RisingWave Python UDF SDK, users can define custom UDFs using Python and start a Python process as a UDF server.
-RisingWave can then remotely access the UDF server to execute the defined functions.
-
-## Installation
-
-```sh
-pip install risingwave
-```
-
-## Usage
-
-Define functions in a Python file:
-
-```python
-# udf.py
-from risingwave.udf import udf, udtf, UdfServer
-import struct
-import socket
-
-# Define a scalar function
-@udf(input_types=['INT', 'INT'], result_type='INT')
-def gcd(x, y):
-    while y != 0:
-        (x, y) = (y, x % y)
-    return x
-
-# Define a scalar function that returns multiple values (within a struct)
-@udf(input_types=['BYTEA'], result_type='STRUCT<VARCHAR, VARCHAR, SMALLINT, SMALLINT>')
-def extract_tcp_info(tcp_packet: bytes):
-    src_addr, dst_addr = struct.unpack('!4s4s', tcp_packet[12:20])
-    src_port, dst_port = struct.unpack('!HH', tcp_packet[20:24])
-    src_addr = socket.inet_ntoa(src_addr)
-    dst_addr = socket.inet_ntoa(dst_addr)
-    return src_addr, dst_addr, src_port, dst_port
-
-# Define a table function
-@udtf(input_types='INT', result_types='INT')
-def series(n):
-    for i in range(n):
-        yield i
-
-# Start a UDF server
-if __name__ == '__main__':
-    server = UdfServer(location="0.0.0.0:8815")
-    server.add_function(gcd)
-    server.add_function(series)
-    server.serve()
-```
-
-Start the UDF server:
-
-```sh
-python3 udf.py
-```
-
-To create functions in RisingWave, use the following syntax:
-
-```sql
-create function <name> ( <arg_type>[, ...] )
-    [ returns <ret_type> | returns table ( <column_name> <column_type> [, ...] ) ]
-    as <name_defined_in_server> using link '<udf_server_address>';
-```
-
-- The `as` parameter specifies the function name defined in the UDF server.
-- The `link` parameter specifies the address of the UDF server.
-
-For example:
-
-```sql
-create function gcd(int, int) returns int
-as gcd using link 'http://localhost:8815';
-
-create function series(int) returns table (x int)
-as series using link 'http://localhost:8815';
-
-select gcd(25, 15);
-
-select * from series(10);
-```
-
-## Data Types
-
-The RisingWave Python UDF SDK supports the following data types:
-
-| SQL Type         | Python Type                    | Notes              |
-| ---------------- | -----------------------------  | ------------------ |
-| BOOLEAN          | bool                           |                    |
-| SMALLINT         | int                            |                    |
-| INT              | int                            |                    |
-| BIGINT           | int                            |                    |
-| REAL             | float                          |                    |
-| DOUBLE PRECISION | float                          |                    |
-| DECIMAL          | decimal.Decimal                |                    |
-| DATE             | datetime.date                  |                    |
-| TIME             | datetime.time                  |                    |
-| TIMESTAMP        | datetime.datetime              |                    |
-| INTERVAL         | MonthDayNano / (int, int, int) | Fields can be obtained by `months()`, `days()` and `nanoseconds()` from `MonthDayNano` |
-| VARCHAR          | str                            |                    |
-| BYTEA            | bytes                          |                    |
-| JSONB            | any                            |                    |
-| T[]              | list[T]                        |                    |
-| STRUCT<>         | tuple                          |                    |
-| ...others        |                                | Not supported yet. |
diff --git a/src/expr/udf/python/publish.md b/src/expr/udf/python/publish.md
deleted file mode 100644
index 0bc22d713906f..0000000000000
--- a/src/expr/udf/python/publish.md
+++ /dev/null
@@ -1,19 +0,0 @@
-# How to publish this library
-
-Install the build tool:
-
-```sh
-pip3 install build
-```
-
-Build the library:
-
-```sh
-python3 -m build
-```
-
-Upload the library to PyPI:
-
-```sh
-twine upload dist/*
-```
diff --git a/src/expr/udf/python/pyproject.toml b/src/expr/udf/python/pyproject.toml
deleted file mode 100644
index b535355168363..0000000000000
--- a/src/expr/udf/python/pyproject.toml
+++ /dev/null
@@ -1,20 +0,0 @@
-[build-system]
-requires = ["setuptools", "wheel"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "risingwave"
-version = "0.1.1"
-authors = [{ name = "RisingWave Labs" }]
-description = "RisingWave Python API"
-readme = "README.md"
-license = { text = "Apache Software License" }
-classifiers = [
-    "Programming Language :: Python",
-    "License :: OSI Approved :: Apache Software License",
-]
-requires-python = ">=3.8"
-dependencies = ["pyarrow"]
-
-[project.optional-dependencies]
-test = ["pytest"]
diff --git a/src/expr/udf/python/risingwave/__init__.py b/src/expr/udf/python/risingwave/__init__.py
deleted file mode 100644
index 3d60f2f96d025..0000000000000
--- a/src/expr/udf/python/risingwave/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 RisingWave Labs
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/src/expr/udf/python/risingwave/test_udf.py b/src/expr/udf/python/risingwave/test_udf.py
deleted file mode 100644
index e3c2029d3d1f9..0000000000000
--- a/src/expr/udf/python/risingwave/test_udf.py
+++ /dev/null
@@ -1,240 +0,0 @@
-# Copyright 2024 RisingWave Labs
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from multiprocessing import Process
-import pytest
-from risingwave.udf import udf, UdfServer, _to_data_type
-import pyarrow as pa
-import pyarrow.flight as flight
-import time
-import datetime
-from typing import Any
-
-
-def flight_server():
-    server = UdfServer(location="localhost:8815")
-    server.add_function(add)
-    server.add_function(wait)
-    server.add_function(wait_concurrent)
-    server.add_function(return_all)
-    return server
-
-
-def flight_client():
-    client = flight.FlightClient(("localhost", 8815))
-    return client
-
-
-# Define a scalar function
-@udf(input_types=["INT", "INT"], result_type="INT")
-def add(x, y):
-    return x + y
-
-
-@udf(input_types=["INT"], result_type="INT")
-def wait(x):
-    time.sleep(0.01)
-    return 0
-
-
-@udf(input_types=["INT"], result_type="INT", io_threads=32)
-def wait_concurrent(x):
-    time.sleep(0.01)
-    return 0
-
-
-@udf(
-    input_types=[
-        "BOOLEAN",
-        "SMALLINT",
-        "INT",
-        "BIGINT",
-        "FLOAT4",
-        "FLOAT8",
-        "DECIMAL",
-        "DATE",
-        "TIME",
-        "TIMESTAMP",
-        "INTERVAL",
-        "VARCHAR",
-        "BYTEA",
-        "JSONB",
-    ],
-    result_type="""struct<
-        BOOLEAN,
-        SMALLINT,
-        INT,
-        BIGINT,
-        FLOAT4,
-        FLOAT8,
-        DECIMAL,
-        DATE,
-        TIME,
-        TIMESTAMP,
-        INTERVAL,
-        VARCHAR,
-        BYTEA,
-        JSONB
-    >""",
-)
-def return_all(
-    bool,
-    i16,
-    i32,
-    i64,
-    f32,
-    f64,
-    decimal,
-    date,
-    time,
-    timestamp,
-    interval,
-    varchar,
-    bytea,
-    jsonb,
-):
-    return (
-        bool,
-        i16,
-        i32,
-        i64,
-        f32,
-        f64,
-        decimal,
-        date,
-        time,
-        timestamp,
-        interval,
-        varchar,
-        bytea,
-        jsonb,
-    )
-
-
-def test_simple():
-    LEN = 64
-    data = pa.Table.from_arrays(
-        [pa.array(range(0, LEN)), pa.array(range(0, LEN))], names=["x", "y"]
-    )
-
-    batches = data.to_batches(max_chunksize=512)
-
-    with flight_client() as client, flight_server() as server:
-        flight_info = flight.FlightDescriptor.for_path(b"add")
-        writer, reader = client.do_exchange(descriptor=flight_info)
-        with writer:
-            writer.begin(schema=data.schema)
-            for batch in batches:
-                writer.write_batch(batch)
-            writer.done_writing()
-
-            chunk = reader.read_chunk()
-            assert len(chunk.data) == LEN
-            assert chunk.data.column("output").equals(
-                pa.array(range(0, LEN * 2, 2), type=pa.int32())
-            )
-
-
-def test_io_concurrency():
-    LEN = 64
-    data = pa.Table.from_arrays([pa.array(range(0, LEN))], names=["x"])
-    batches = data.to_batches(max_chunksize=512)
-
-    with flight_client() as client, flight_server() as server:
-        # Single-threaded function takes a long time
-        flight_info = flight.FlightDescriptor.for_path(b"wait")
-        writer, reader = client.do_exchange(descriptor=flight_info)
-        with writer:
-            writer.begin(schema=data.schema)
-            for batch in batches:
-                writer.write_batch(batch)
-            writer.done_writing()
-            start_time = time.time()
-
-            total_len = 0
-            for chunk in reader:
-                total_len += len(chunk.data)
-
-            assert total_len == LEN
-
-            elapsed_time = time.time() - start_time  # ~0.64s
-            assert elapsed_time > 0.5
-
-        # Multi-threaded I/O bound function will take a much shorter time
-        flight_info = flight.FlightDescriptor.for_path(b"wait_concurrent")
-        writer, reader = client.do_exchange(descriptor=flight_info)
-        with writer:
-            writer.begin(schema=data.schema)
-            for batch in batches:
-                writer.write_batch(batch)
-            writer.done_writing()
-            start_time = time.time()
-
-            total_len = 0
-            for chunk in reader:
-                total_len += len(chunk.data)
-
-            assert total_len == LEN
-
-            elapsed_time = time.time() - start_time
-            assert elapsed_time < 0.25
-
-
-def test_all_types():
-    arrays = [
-        pa.array([True], type=pa.bool_()),
-        pa.array([1], type=pa.int16()),
-        pa.array([1], type=pa.int32()),
-        pa.array([1], type=pa.int64()),
-        pa.array([1], type=pa.float32()),
-        pa.array([1], type=pa.float64()),
-        pa.array(["12345678901234567890.1234567890"], type=pa.large_binary()),
-        pa.array([datetime.date(2023, 6, 1)], type=pa.date32()),
-        pa.array([datetime.time(1, 2, 3, 456789)], type=pa.time64("us")),
-        pa.array(
-            [datetime.datetime(2023, 6, 1, 1, 2, 3, 456789)],
-            type=pa.timestamp("us"),
-        ),
-        pa.array([(1, 2, 3)], type=pa.month_day_nano_interval()),
-        pa.array(["string"], type=pa.string()),
-        pa.array(["bytes"], type=pa.binary()),
-        pa.array(['{ "key": 1 }'], type=pa.large_string()),
-    ]
-    batch = pa.RecordBatch.from_arrays(arrays, names=["" for _ in arrays])
-
-    with flight_client() as client, flight_server() as server:
-        flight_info = flight.FlightDescriptor.for_path(b"return_all")
-        writer, reader = client.do_exchange(descriptor=flight_info)
-        with writer:
-            writer.begin(schema=batch.schema)
-            writer.write_batch(batch)
-            writer.done_writing()
-
-            chunk = reader.read_chunk()
-            assert [v.as_py() for _, v in chunk.data.column(0)[0].items()] == [
-                True,
-                1,
-                1,
-                1,
-                1.0,
-                1.0,
-                b"12345678901234567890.1234567890",
-                datetime.date(2023, 6, 1),
-                datetime.time(1, 2, 3, 456789),
-                datetime.datetime(2023, 6, 1, 1, 2, 3, 456789),
-                (1, 2, 3),
-                "string",
-                b"bytes",
-                '{"key": 1}',
-            ]
diff --git a/src/expr/udf/python/risingwave/udf.py b/src/expr/udf/python/risingwave/udf.py
deleted file mode 100644
index aad53e25e0c98..0000000000000
--- a/src/expr/udf/python/risingwave/udf.py
+++ /dev/null
@@ -1,552 +0,0 @@
-# Copyright 2024 RisingWave Labs
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import *
-import pyarrow as pa
-import pyarrow.flight
-import pyarrow.parquet
-import inspect
-import traceback
-import json
-from concurrent.futures import ThreadPoolExecutor
-import concurrent
-from decimal import Decimal
-import signal
-
-
-class UserDefinedFunction:
-    """
-    Base interface for user-defined function.
-    """
-
-    _name: str
-    _input_schema: pa.Schema
-    _result_schema: pa.Schema
-    _io_threads: Optional[int]
-    _executor: Optional[ThreadPoolExecutor]
-
-    def eval_batch(self, batch: pa.RecordBatch) -> Iterator[pa.RecordBatch]:
-        """
-        Apply the function on a batch of inputs.
-        """
-        return iter([])
-
-
-class ScalarFunction(UserDefinedFunction):
-    """
-    Base interface for user-defined scalar function. A user-defined scalar functions maps zero, one,
-    or multiple scalar values to a new scalar value.
-    """
-
-    def __init__(self, *args, **kwargs):
-        self._io_threads = kwargs.pop("io_threads")
-        self._executor = (
-            ThreadPoolExecutor(max_workers=self._io_threads)
-            if self._io_threads is not None
-            else None
-        )
-        super().__init__(*args, **kwargs)
-
-    def eval(self, *args) -> Any:
-        """
-        Method which defines the logic of the scalar function.
-        """
-        pass
-
-    def eval_batch(self, batch: pa.RecordBatch) -> Iterator[pa.RecordBatch]:
-        # parse value from json string for jsonb columns
-        inputs = [[v.as_py() for v in array] for array in batch]
-        inputs = [
-            _process_func(pa.list_(type), False)(array)
-            for array, type in zip(inputs, self._input_schema.types)
-        ]
-        if self._executor is not None:
-            # evaluate the function for each row
-            tasks = [
-                self._executor.submit(self._func, *[col[i] for col in inputs])
-                for i in range(batch.num_rows)
-            ]
-            column = [
-                future.result() for future in concurrent.futures.as_completed(tasks)
-            ]
-        else:
-            # evaluate the function for each row
-            column = [
-                self.eval(*[col[i] for col in inputs]) for i in range(batch.num_rows)
-            ]
-
-        column = _process_func(pa.list_(self._result_schema.types[0]), True)(column)
-
-        array = pa.array(column, type=self._result_schema.types[0])
-        yield pa.RecordBatch.from_arrays([array], schema=self._result_schema)
-
-
-def _process_func(type: pa.DataType, output: bool) -> Callable:
-    """Return a function to process input or output value."""
-    if pa.types.is_list(type):
-        func = _process_func(type.value_type, output)
-        return lambda array: [(func(v) if v is not None else None) for v in array]
-
-    if pa.types.is_struct(type):
-        funcs = [_process_func(field.type, output) for field in type]
-        if output:
-            return lambda tup: tuple(
-                (func(v) if v is not None else None) for v, func in zip(tup, funcs)
-            )
-        else:
-            # the input value of struct type is a dict
-            # we convert it into tuple here
-            return lambda map: tuple(
-                (func(v) if v is not None else None)
-                for v, func in zip(map.values(), funcs)
-            )
-
-    if type.equals(JSONB):
-        if output:
-            return lambda v: json.dumps(v)
-        else:
-            return lambda v: json.loads(v)
-
-    if type.equals(UNCONSTRAINED_DECIMAL):
-        if output:
-
-            def decimal_to_str(v):
-                if not isinstance(v, Decimal):
-                    raise ValueError(f"Expected Decimal, got {v}")
-                # use `f` format to avoid scientific notation, e.g. `1e10`
-                return format(v, "f").encode("utf-8")
-
-            return decimal_to_str
-        else:
-            return lambda v: Decimal(v.decode("utf-8"))
-
-    return lambda v: v
-
-
-class TableFunction(UserDefinedFunction):
-    """
-    Base interface for user-defined table function. A user-defined table functions maps zero, one,
-    or multiple scalar values to a new table value.
-    """
-
-    BATCH_SIZE = 1024
-
-    def eval(self, *args) -> Iterator:
-        """
-        Method which defines the logic of the table function.
-        """
-        yield
-
-    def eval_batch(self, batch: pa.RecordBatch) -> Iterator[pa.RecordBatch]:
-        class RecordBatchBuilder:
-            """A utility class for constructing Arrow RecordBatch by row."""
-
-            schema: pa.Schema
-            columns: List[List]
-
-            def __init__(self, schema: pa.Schema):
-                self.schema = schema
-                self.columns = [[] for _ in self.schema.types]
-
-            def len(self) -> int:
-                """Returns the number of rows in the RecordBatch being built."""
-                return len(self.columns[0])
-
-            def append(self, index: int, value: Any):
-                """Appends a new row to the RecordBatch being built."""
-                self.columns[0].append(index)
-                self.columns[1].append(value)
-
-            def build(self) -> pa.RecordBatch:
-                """Builds the RecordBatch from the accumulated data and clears the state."""
-                # Convert the columns to arrow arrays
-                arrays = [
-                    pa.array(col, type)
-                    for col, type in zip(self.columns, self.schema.types)
-                ]
-                # Reset columns
-                self.columns = [[] for _ in self.schema.types]
-                return pa.RecordBatch.from_arrays(arrays, schema=self.schema)
-
-        builder = RecordBatchBuilder(self._result_schema)
-
-        # Iterate through rows in the input RecordBatch
-        for row_index in range(batch.num_rows):
-            row = tuple(column[row_index].as_py() for column in batch)
-            for result in self.eval(*row):
-                builder.append(row_index, result)
-                if builder.len() == self.BATCH_SIZE:
-                    yield builder.build()
-        if builder.len() != 0:
-            yield builder.build()
-
-
-class UserDefinedScalarFunctionWrapper(ScalarFunction):
-    """
-    Base Wrapper for Python user-defined scalar function.
-    """
-
-    _func: Callable
-
-    def __init__(self, func, input_types, result_type, name=None, io_threads=None):
-        self._func = func
-        self._input_schema = pa.schema(
-            zip(
-                inspect.getfullargspec(func)[0],
-                [_to_data_type(t) for t in _to_list(input_types)],
-            )
-        )
-        self._result_schema = pa.schema([("output", _to_data_type(result_type))])
-        self._name = name or (
-            func.__name__ if hasattr(func, "__name__") else func.__class__.__name__
-        )
-        super().__init__(io_threads=io_threads)
-
-    def __call__(self, *args):
-        return self._func(*args)
-
-    def eval(self, *args):
-        return self._func(*args)
-
-
-class UserDefinedTableFunctionWrapper(TableFunction):
-    """
-    Base Wrapper for Python user-defined table function.
-    """
-
-    _func: Callable
-
-    def __init__(self, func, input_types, result_types, name=None):
-        self._func = func
-        self._name = name or (
-            func.__name__ if hasattr(func, "__name__") else func.__class__.__name__
-        )
-        self._input_schema = pa.schema(
-            zip(
-                inspect.getfullargspec(func)[0],
-                [_to_data_type(t) for t in _to_list(input_types)],
-            )
-        )
-        self._result_schema = pa.schema(
-            [
-                ("row_index", pa.int32()),
-                (
-                    self._name,
-                    pa.struct([("", _to_data_type(t)) for t in result_types])
-                    if isinstance(result_types, list)
-                    else _to_data_type(result_types),
-                ),
-            ]
-        )
-
-    def __call__(self, *args):
-        return self._func(*args)
-
-    def eval(self, *args):
-        return self._func(*args)
-
-
-def _to_list(x):
-    if isinstance(x, list):
-        return x
-    else:
-        return [x]
-
-
-def udf(
-    input_types: Union[List[Union[str, pa.DataType]], Union[str, pa.DataType]],
-    result_type: Union[str, pa.DataType],
-    name: Optional[str] = None,
-    io_threads: Optional[int] = None,
-) -> Callable:
-    """
-    Annotation for creating a user-defined scalar function.
-
-    Parameters:
-    - input_types: A list of strings or Arrow data types that specifies the input data types.
-    - result_type: A string or an Arrow data type that specifies the return value type.
-    - name: An optional string specifying the function name. If not provided, the original name will be used.
-    - io_threads: Number of I/O threads used per data chunk for I/O bound functions.
-
-    Example:
-    ```
-    @udf(input_types=['INT', 'INT'], result_type='INT')
-    def gcd(x, y):
-        while y != 0:
-            (x, y) = (y, x % y)
-        return x
-    ```
-
-    I/O bound Example:
-    ```
-    @udf(input_types=['INT'], result_type='INT', io_threads=64)
-    def external_api(x):
-        response = requests.get(my_endpoint + '?param=' + x)
-        return response["data"]
-    ```
-    """
-
-    if io_threads is not None and io_threads > 1:
-        return lambda f: UserDefinedScalarFunctionWrapper(
-            f, input_types, result_type, name, io_threads=io_threads
-        )
-    else:
-        return lambda f: UserDefinedScalarFunctionWrapper(
-            f, input_types, result_type, name
-        )
-
-
-def udtf(
-    input_types: Union[List[Union[str, pa.DataType]], Union[str, pa.DataType]],
-    result_types: Union[List[Union[str, pa.DataType]], Union[str, pa.DataType]],
-    name: Optional[str] = None,
-) -> Callable:
-    """
-    Annotation for creating a user-defined table function.
-
-    Parameters:
-    - input_types: A list of strings or Arrow data types that specifies the input data types.
-    - result_types A list of strings or Arrow data types that specifies the return value types.
-    - name: An optional string specifying the function name. If not provided, the original name will be used.
-
-    Example:
-    ```
-    @udtf(input_types='INT', result_types='INT')
-    def series(n):
-        for i in range(n):
-            yield i
-    ```
-    """
-
-    return lambda f: UserDefinedTableFunctionWrapper(f, input_types, result_types, name)
-
-
-class UdfServer(pa.flight.FlightServerBase):
-    """
-    A server that provides user-defined functions to clients.
-
-    Example:
-    ```
-    server = UdfServer(location="0.0.0.0:8815")
-    server.add_function(my_udf)
-    server.serve()
-    ```
-    """
-
-    # UDF server based on Apache Arrow Flight protocol.
-    # Reference: https://arrow.apache.org/cookbook/py/flight.html#simple-parquet-storage-service-with-arrow-flight
-
-    _location: str
-    _functions: Dict[str, UserDefinedFunction]
-
-    def __init__(self, location="0.0.0.0:8815", **kwargs):
-        super(UdfServer, self).__init__("grpc://" + location, **kwargs)
-        self._location = location
-        self._functions = {}
-
-    def get_flight_info(self, context, descriptor):
-        """Return the result schema of a function."""
-        udf = self._functions[descriptor.path[0].decode("utf-8")]
-        # return the concatenation of input and output schema
-        full_schema = pa.schema(list(udf._input_schema) + list(udf._result_schema))
-        # we use `total_records` to indicate the number of input arguments
-        return pa.flight.FlightInfo(
-            schema=full_schema,
-            descriptor=descriptor,
-            endpoints=[],
-            total_records=len(udf._input_schema),
-            total_bytes=0,
-        )
-
-    def add_function(self, udf: UserDefinedFunction):
-        """Add a function to the server."""
-        name = udf._name
-        if name in self._functions:
-            raise ValueError("Function already exists: " + name)
-
-        input_types = ",".join(
-            [_data_type_to_string(t) for t in udf._input_schema.types]
-        )
-        if isinstance(udf, TableFunction):
-            output_type = udf._result_schema.types[-1]
-            if isinstance(output_type, pa.StructType):
-                output_type = ",".join(
-                    f"field_{i} {_data_type_to_string(field.type)}"
-                    for i, field in enumerate(output_type)
-                )
-                output_type = f"TABLE({output_type})"
-            else:
-                output_type = _data_type_to_string(output_type)
-                output_type = f"TABLE(output {output_type})"
-        else:
-            output_type = _data_type_to_string(udf._result_schema.types[-1])
-
-        sql = f"CREATE FUNCTION {name}({input_types}) RETURNS {output_type} AS '{name}' USING LINK 'http://{self._location}';"
-        print(f"added function: {name}, corresponding SQL:\n{sql}\n")
-        self._functions[name] = udf
-
-    def do_exchange(self, context, descriptor, reader, writer):
-        """Call a function from the client."""
-        udf = self._functions[descriptor.path[0].decode("utf-8")]
-        writer.begin(udf._result_schema)
-        try:
-            for batch in reader:
-                # print(pa.Table.from_batches([batch.data]))
-                for output_batch in udf.eval_batch(batch.data):
-                    writer.write_batch(output_batch)
-        except Exception as e:
-            print(traceback.print_exc())
-            raise e
-
-    def serve(self):
-        """
-        Block until the server shuts down.
-
-        This method only returns if shutdown() is called or a signal (SIGINT, SIGTERM) received.
-        """
-        print(
-            "Note: You can use arbitrary function names and struct field names in CREATE FUNCTION statements."
-            f"\n\nlistening on {self._location}"
-        )
-        signal.signal(signal.SIGTERM, lambda s, f: self.shutdown())
-        super(UdfServer, self).serve()
-
-
-def _to_data_type(t: Union[str, pa.DataType]) -> pa.DataType:
-    """
-    Convert a SQL data type string or `pyarrow.DataType` to `pyarrow.DataType`.
-    """
-    if isinstance(t, str):
-        return _string_to_data_type(t)
-    else:
-        return t
-
-
-# we use `large_binary` to represent unconstrained decimal type
-UNCONSTRAINED_DECIMAL = pa.large_binary()
-JSONB = pa.large_string()
-
-
-def _string_to_data_type(type_str: str):
-    """
-    Convert a SQL data type string to `pyarrow.DataType`.
-    """
-    type_str = type_str.upper()
-    if type_str.endswith("[]"):
-        return pa.list_(_string_to_data_type(type_str[:-2]))
-    elif type_str in ("BOOLEAN", "BOOL"):
-        return pa.bool_()
-    elif type_str in ("SMALLINT", "INT2"):
-        return pa.int16()
-    elif type_str in ("INT", "INTEGER", "INT4"):
-        return pa.int32()
-    elif type_str in ("BIGINT", "INT8"):
-        return pa.int64()
-    elif type_str in ("FLOAT4", "REAL"):
-        return pa.float32()
-    elif type_str in ("FLOAT8", "DOUBLE PRECISION"):
-        return pa.float64()
-    elif type_str.startswith("DECIMAL") or type_str.startswith("NUMERIC"):
-        if type_str == "DECIMAL" or type_str == "NUMERIC":
-            return UNCONSTRAINED_DECIMAL
-        rest = type_str[8:-1]  # remove "DECIMAL(" and ")"
-        if "," in rest:
-            precision, scale = rest.split(",")
-            return pa.decimal128(int(precision), int(scale))
-        else:
-            return pa.decimal128(int(rest), 0)
-    elif type_str in ("DATE"):
-        return pa.date32()
-    elif type_str in ("TIME", "TIME WITHOUT TIME ZONE"):
-        return pa.time64("us")
-    elif type_str in ("TIMESTAMP", "TIMESTAMP WITHOUT TIME ZONE"):
-        return pa.timestamp("us")
-    elif type_str.startswith("INTERVAL"):
-        return pa.month_day_nano_interval()
-    elif type_str in ("VARCHAR"):
-        return pa.string()
-    elif type_str in ("JSONB"):
-        return JSONB
-    elif type_str in ("BYTEA"):
-        return pa.binary()
-    elif type_str.startswith("STRUCT"):
-        # extract 'STRUCT<INT, VARCHAR, STRUCT<INT>, ...>'
-        type_list = type_str[7:-1]  # strip "STRUCT<>"
-        fields = []
-        elements = []
-        start = 0
-        depth = 0
-        for i, c in enumerate(type_list):
-            if c == "<":
-                depth += 1
-            elif c == ">":
-                depth -= 1
-            elif c == "," and depth == 0:
-                type_str = type_list[start:i].strip()
-                fields.append(pa.field("", _string_to_data_type(type_str)))
-                start = i + 1
-        type_str = type_list[start:].strip()
-        fields.append(pa.field("", _string_to_data_type(type_str)))
-        return pa.struct(fields)
-
-    raise ValueError(f"Unsupported type: {type_str}")
-
-
-def _data_type_to_string(t: pa.DataType) -> str:
-    """
-    Convert a `pyarrow.DataType` to a SQL data type string.
-    """
-    if isinstance(t, pa.ListType):
-        return _data_type_to_string(t.value_type) + "[]"
-    elif t.equals(pa.bool_()):
-        return "BOOLEAN"
-    elif t.equals(pa.int16()):
-        return "SMALLINT"
-    elif t.equals(pa.int32()):
-        return "INT"
-    elif t.equals(pa.int64()):
-        return "BIGINT"
-    elif t.equals(pa.float32()):
-        return "FLOAT4"
-    elif t.equals(pa.float64()):
-        return "FLOAT8"
-    elif t.equals(UNCONSTRAINED_DECIMAL):
-        return "DECIMAL"
-    elif pa.types.is_decimal(t):
-        return f"DECIMAL({t.precision},{t.scale})"
-    elif t.equals(pa.date32()):
-        return "DATE"
-    elif t.equals(pa.time64("us")):
-        return "TIME"
-    elif t.equals(pa.timestamp("us")):
-        return "TIMESTAMP"
-    elif t.equals(pa.month_day_nano_interval()):
-        return "INTERVAL"
-    elif t.equals(pa.string()):
-        return "VARCHAR"
-    elif t.equals(JSONB):
-        return "JSONB"
-    elif t.equals(pa.binary()):
-        return "BYTEA"
-    elif isinstance(t, pa.StructType):
-        return (
-            "STRUCT<"
-            + ",".join(
-                f"f{i+1} {_data_type_to_string(field.type)}"
-                for i, field in enumerate(t)
-            )
-            + ">"
-        )
-    else:
-        raise ValueError(f"Unsupported type: {t}")
diff --git a/src/expr/udf/python/risingwave/udf/health_check.py b/src/expr/udf/python/risingwave/udf/health_check.py
deleted file mode 100644
index ad2d38681a6cc..0000000000000
--- a/src/expr/udf/python/risingwave/udf/health_check.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# Copyright 2024 RisingWave Labs
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from pyarrow.flight import FlightClient
-import sys
-
-
-def check_udf_service_available(addr: str) -> bool:
-    """Check if the UDF service is available at the given address."""
-    try:
-        client = FlightClient(f"grpc://{addr}")
-        client.wait_for_available()
-        return True
-    except Exception as e:
-        print(f"Error connecting to RisingWave UDF service: {str(e)}")
-        return False
-
-
-if __name__ == "__main__":
-    if len(sys.argv) != 2:
-        print("usage: python3 health_check.py <server_address>")
-        sys.exit(1)
-
-    server_address = sys.argv[1]
-    if check_udf_service_available(server_address):
-        print("OK")
-    else:
-        print("unavailable")
-        exit(-1)
diff --git a/src/expr/udf/src/error.rs b/src/expr/udf/src/error.rs
deleted file mode 100644
index fc6733052b137..0000000000000
--- a/src/expr/udf/src/error.rs
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use arrow_flight::error::FlightError;
-use thiserror::Error;
-use thiserror_ext::{Box, Construct};
-
-/// A specialized `Result` type for UDF operations.
-pub type Result<T, E = Error> = std::result::Result<T, E>;
-
-/// The error type for UDF operations.
-#[derive(Error, Debug, Box, Construct)]
-#[thiserror_ext(newtype(name = Error))]
-pub enum ErrorInner {
-    #[error("failed to send requests to UDF service: {0}")]
-    Tonic(#[from] tonic::Status),
-
-    #[error("failed to call UDF: {0}")]
-    Flight(#[from] FlightError),
-
-    #[error("type mismatch: {0}")]
-    TypeMismatch(String),
-
-    #[error("arrow error: {0}")]
-    Arrow(#[from] arrow_schema::ArrowError),
-
-    #[error("UDF unsupported: {0}")]
-    // TODO(error-handling): should prefer use error types than strings.
-    Unsupported(String),
-
-    #[error("UDF service returned no data")]
-    NoReturned,
-
-    #[error("Flight service error: {0}")]
-    ServiceError(String),
-}
-
-impl Error {
-    /// Returns true if the error is caused by a connection error.
-    pub fn is_connection_error(&self) -> bool {
-        match self.inner() {
-            // Connection refused
-            ErrorInner::Tonic(status) if status.code() == tonic::Code::Unavailable => true,
-            _ => false,
-        }
-    }
-
-    pub fn is_tonic_error(&self) -> bool {
-        matches!(
-            self.inner(),
-            ErrorInner::Tonic(_) | ErrorInner::Flight(FlightError::Tonic(_))
-        )
-    }
-}
-
-static_assertions::const_assert_eq!(std::mem::size_of::<Error>(), 8);
diff --git a/src/expr/udf/src/external.rs b/src/expr/udf/src/external.rs
deleted file mode 100644
index 7560638b03985..0000000000000
--- a/src/expr/udf/src/external.rs
+++ /dev/null
@@ -1,260 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::str::FromStr;
-use std::time::Duration;
-
-use arrow_array::RecordBatch;
-use arrow_flight::decode::FlightRecordBatchStream;
-use arrow_flight::encode::FlightDataEncoderBuilder;
-use arrow_flight::error::FlightError;
-use arrow_flight::flight_service_client::FlightServiceClient;
-use arrow_flight::{FlightData, FlightDescriptor};
-use arrow_schema::Schema;
-use cfg_or_panic::cfg_or_panic;
-use futures_util::{stream, FutureExt, Stream, StreamExt, TryStreamExt};
-use ginepro::{LoadBalancedChannel, ResolutionStrategy};
-use risingwave_common::util::addr::HostAddr;
-use thiserror_ext::AsReport;
-use tokio::time::Duration as TokioDuration;
-use tonic::transport::Channel;
-
-use crate::metrics::GLOBAL_METRICS;
-use crate::{Error, Result};
-
-// Interval between two successive probes of the UDF DNS.
-const DNS_PROBE_INTERVAL_SECS: u64 = 5;
-// Timeout duration for performing an eager DNS resolution.
-const EAGER_DNS_RESOLVE_TIMEOUT_SECS: u64 = 5;
-const REQUEST_TIMEOUT_SECS: u64 = 5;
-const CONNECT_TIMEOUT_SECS: u64 = 5;
-
-/// Client for external function service based on Arrow Flight.
-#[derive(Debug)]
-pub struct ArrowFlightUdfClient {
-    client: FlightServiceClient<Channel>,
-    addr: String,
-}
-
-// TODO: support UDF in simulation
-#[cfg_or_panic(not(madsim))]
-impl ArrowFlightUdfClient {
-    /// Connect to a UDF service.
-    pub async fn connect(addr: &str) -> Result<Self> {
-        Self::connect_inner(
-            addr,
-            ResolutionStrategy::Eager {
-                timeout: TokioDuration::from_secs(EAGER_DNS_RESOLVE_TIMEOUT_SECS),
-            },
-        )
-        .await
-    }
-
-    /// Connect to a UDF service lazily (i.e. only when the first request is sent).
-    pub fn connect_lazy(addr: &str) -> Result<Self> {
-        Self::connect_inner(addr, ResolutionStrategy::Lazy)
-            .now_or_never()
-            .unwrap()
-    }
-
-    async fn connect_inner(
-        mut addr: &str,
-        resolution_strategy: ResolutionStrategy,
-    ) -> Result<Self> {
-        if addr.starts_with("http://") {
-            addr = addr.strip_prefix("http://").unwrap();
-        }
-        if addr.starts_with("https://") {
-            addr = addr.strip_prefix("https://").unwrap();
-        }
-        let host_addr = HostAddr::from_str(addr).map_err(|e| {
-            Error::service_error(format!("invalid address: {}, err: {}", addr, e.as_report()))
-        })?;
-        let channel = LoadBalancedChannel::builder((host_addr.host.clone(), host_addr.port))
-            .dns_probe_interval(std::time::Duration::from_secs(DNS_PROBE_INTERVAL_SECS))
-            .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECS))
-            .connect_timeout(Duration::from_secs(CONNECT_TIMEOUT_SECS))
-            .resolution_strategy(resolution_strategy)
-            .channel()
-            .await
-            .map_err(|e| {
-                Error::service_error(format!(
-                    "failed to create LoadBalancedChannel, address: {}, err: {}",
-                    host_addr,
-                    e.as_report()
-                ))
-            })?;
-        let client = FlightServiceClient::new(channel.into());
-        Ok(Self {
-            client,
-            addr: addr.into(),
-        })
-    }
-
-    /// Check if the function is available and the schema is match.
-    pub async fn check(&self, id: &str, args: &Schema, returns: &Schema) -> Result<()> {
-        let descriptor = FlightDescriptor::new_path(vec![id.into()]);
-
-        let response = self.client.clone().get_flight_info(descriptor).await?;
-
-        // check schema
-        let info = response.into_inner();
-        let input_num = info.total_records as usize;
-        let full_schema = Schema::try_from(info).map_err(|e| {
-            FlightError::DecodeError(format!("Error decoding schema: {}", e.as_report()))
-        })?;
-        if input_num > full_schema.fields.len() {
-            return Err(Error::service_error(format!(
-                "function {:?} schema info not consistency: input_num: {}, total_fields: {}",
-                id,
-                input_num,
-                full_schema.fields.len()
-            )));
-        }
-
-        let (input_fields, return_fields) = full_schema.fields.split_at(input_num);
-        let actual_input_types: Vec<_> = input_fields.iter().map(|f| f.data_type()).collect();
-        let actual_result_types: Vec<_> = return_fields.iter().map(|f| f.data_type()).collect();
-        let expect_input_types: Vec<_> = args.fields.iter().map(|f| f.data_type()).collect();
-        let expect_result_types: Vec<_> = returns.fields.iter().map(|f| f.data_type()).collect();
-        if !data_types_match(&expect_input_types, &actual_input_types) {
-            return Err(Error::type_mismatch(format!(
-                "function: {:?}, expect arguments: {:?}, actual: {:?}",
-                id, expect_input_types, actual_input_types
-            )));
-        }
-        if !data_types_match(&expect_result_types, &actual_result_types) {
-            return Err(Error::type_mismatch(format!(
-                "function: {:?}, expect return: {:?}, actual: {:?}",
-                id, expect_result_types, actual_result_types
-            )));
-        }
-        Ok(())
-    }
-
-    /// Call a function.
-    pub async fn call(&self, id: &str, input: RecordBatch) -> Result<RecordBatch> {
-        self.call_internal(id, input).await
-    }
-
-    async fn call_internal(&self, id: &str, input: RecordBatch) -> Result<RecordBatch> {
-        let mut output_stream = self
-            .call_stream_internal(id, stream::once(async { input }))
-            .await?;
-        let mut batches = vec![];
-        while let Some(batch) = output_stream.next().await {
-            batches.push(batch?);
-        }
-        Ok(arrow_select::concat::concat_batches(
-            output_stream.schema().ok_or_else(Error::no_returned)?,
-            batches.iter(),
-        )?)
-    }
-
-    /// Call a function, retry up to 5 times / 3s if connection is broken.
-    pub async fn call_with_retry(&self, id: &str, input: RecordBatch) -> Result<RecordBatch> {
-        let mut backoff = Duration::from_millis(100);
-        for i in 0..5 {
-            match self.call(id, input.clone()).await {
-                Err(err) if err.is_connection_error() && i != 4 => {
-                    tracing::error!(error = %err.as_report(), "UDF connection error. retry...");
-                }
-                ret => return ret,
-            }
-            tokio::time::sleep(backoff).await;
-            backoff *= 2;
-        }
-        unreachable!()
-    }
-
-    /// Always retry on connection error
-    pub async fn call_with_always_retry_on_network_error(
-        &self,
-        id: &str,
-        input: RecordBatch,
-        fragment_id: &str,
-    ) -> Result<RecordBatch> {
-        let mut backoff = Duration::from_millis(100);
-        let metrics = &*GLOBAL_METRICS;
-        let labels: &[&str; 4] = &[&self.addr, "external", id, fragment_id];
-        loop {
-            match self.call(id, input.clone()).await {
-                Err(err) if err.is_tonic_error() => {
-                    tracing::error!(error = %err.as_report(), "UDF tonic error. retry...");
-                }
-                ret => {
-                    if ret.is_err() {
-                        tracing::error!(error = %ret.as_ref().unwrap_err().as_report(), "UDF error. exiting...");
-                    }
-                    return ret;
-                }
-            }
-            metrics.udf_retry_count.with_label_values(labels).inc();
-            tokio::time::sleep(backoff).await;
-            backoff *= 2;
-        }
-    }
-
-    /// Call a function with streaming input and output.
-    #[panic_return = "Result<stream::Empty<_>>"]
-    pub async fn call_stream(
-        &self,
-        id: &str,
-        inputs: impl Stream<Item = RecordBatch> + Send + 'static,
-    ) -> Result<impl Stream<Item = Result<RecordBatch>> + Send + 'static> {
-        Ok(self
-            .call_stream_internal(id, inputs)
-            .await?
-            .map_err(|e| e.into()))
-    }
-
-    async fn call_stream_internal(
-        &self,
-        id: &str,
-        inputs: impl Stream<Item = RecordBatch> + Send + 'static,
-    ) -> Result<FlightRecordBatchStream> {
-        let descriptor = FlightDescriptor::new_path(vec![id.into()]);
-        let flight_data_stream =
-            FlightDataEncoderBuilder::new()
-                .build(inputs.map(Ok))
-                .map(move |res| FlightData {
-                    // TODO: fill descriptor only for the first message
-                    flight_descriptor: Some(descriptor.clone()),
-                    ..res.unwrap()
-                });
-
-        // call `do_exchange` on Flight server
-        let response = self.client.clone().do_exchange(flight_data_stream).await?;
-
-        // decode response
-        let stream = response.into_inner();
-        Ok(FlightRecordBatchStream::new_from_flight_data(
-            // convert tonic::Status to FlightError
-            stream.map_err(|e| e.into()),
-        ))
-    }
-
-    pub fn get_addr(&self) -> &str {
-        &self.addr
-    }
-}
-
-/// Check if two list of data types match, ignoring field names.
-fn data_types_match(a: &[&arrow_schema::DataType], b: &[&arrow_schema::DataType]) -> bool {
-    if a.len() != b.len() {
-        return false;
-    }
-    #[allow(clippy::disallowed_methods)]
-    a.iter().zip(b.iter()).all(|(a, b)| a.equals_datatype(b))
-}
diff --git a/src/expr/udf/src/lib.rs b/src/expr/udf/src/lib.rs
deleted file mode 100644
index ddd8cf1bdeab9..0000000000000
--- a/src/expr/udf/src/lib.rs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#![feature(error_generic_member_access)]
-#![feature(lazy_cell)]
-
-mod error;
-mod external;
-pub mod metrics;
-
-pub use error::{Error, Result};
-pub use external::ArrowFlightUdfClient;
-pub use metrics::GLOBAL_METRICS;
diff --git a/src/expr/udf/src/metrics.rs b/src/expr/udf/src/metrics.rs
deleted file mode 100644
index 50ef1b068307d..0000000000000
--- a/src/expr/udf/src/metrics.rs
+++ /dev/null
@@ -1,111 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::LazyLock;
-
-use prometheus::{
-    exponential_buckets, register_histogram_vec_with_registry,
-    register_int_counter_vec_with_registry, HistogramVec, IntCounterVec, Registry,
-};
-use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
-
-/// Monitor metrics for UDF.
-#[derive(Debug, Clone)]
-pub struct Metrics {
-    /// Number of successful UDF calls.
-    pub udf_success_count: IntCounterVec,
-    /// Number of failed UDF calls.
-    pub udf_failure_count: IntCounterVec,
-    /// Total number of retried UDF calls.
-    pub udf_retry_count: IntCounterVec,
-    /// Input chunk rows of UDF calls.
-    pub udf_input_chunk_rows: HistogramVec,
-    /// The latency of UDF calls in seconds.
-    pub udf_latency: HistogramVec,
-    /// Total number of input rows of UDF calls.
-    pub udf_input_rows: IntCounterVec,
-    /// Total number of input bytes of UDF calls.
-    pub udf_input_bytes: IntCounterVec,
-}
-
-/// Global UDF metrics.
-pub static GLOBAL_METRICS: LazyLock<Metrics> =
-    LazyLock::new(|| Metrics::new(&GLOBAL_METRICS_REGISTRY));
-
-impl Metrics {
-    fn new(registry: &Registry) -> Self {
-        let labels = &["link", "language", "name", "fragment_id"];
-        let udf_success_count = register_int_counter_vec_with_registry!(
-            "udf_success_count",
-            "Total number of successful UDF calls",
-            labels,
-            registry
-        )
-        .unwrap();
-        let udf_failure_count = register_int_counter_vec_with_registry!(
-            "udf_failure_count",
-            "Total number of failed UDF calls",
-            labels,
-            registry
-        )
-        .unwrap();
-        let udf_retry_count = register_int_counter_vec_with_registry!(
-            "udf_retry_count",
-            "Total number of retried UDF calls",
-            labels,
-            registry
-        )
-        .unwrap();
-        let udf_input_chunk_rows = register_histogram_vec_with_registry!(
-            "udf_input_chunk_rows",
-            "Input chunk rows of UDF calls",
-            labels,
-            exponential_buckets(1.0, 2.0, 10).unwrap(), // 1 to 1024
-            registry
-        )
-        .unwrap();
-        let udf_latency = register_histogram_vec_with_registry!(
-            "udf_latency",
-            "The latency(s) of UDF calls",
-            labels,
-            exponential_buckets(0.000001, 2.0, 30).unwrap(), // 1us to 1000s
-            registry
-        )
-        .unwrap();
-        let udf_input_rows = register_int_counter_vec_with_registry!(
-            "udf_input_rows",
-            "Total number of input rows of UDF calls",
-            labels,
-            registry
-        )
-        .unwrap();
-        let udf_input_bytes = register_int_counter_vec_with_registry!(
-            "udf_input_bytes",
-            "Total number of input bytes of UDF calls",
-            labels,
-            registry
-        )
-        .unwrap();
-
-        Metrics {
-            udf_success_count,
-            udf_failure_count,
-            udf_retry_count,
-            udf_input_chunk_rows,
-            udf_latency,
-            udf_input_rows,
-            udf_input_bytes,
-        }
-    }
-}
diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml
index 3cba7afe82660..feb4aa596c55f 100644
--- a/src/frontend/Cargo.toml
+++ b/src/frontend/Cargo.toml
@@ -18,7 +18,6 @@ normal = ["workspace-hack"]
 anyhow = "1"
 arc-swap = "1"
 arrow-schema = { workspace = true }
-arrow-udf-wasm = { workspace = true }
 async-recursion = "1.1.0"
 async-trait = "0.1"
 auto_enums = { workspace = true }
@@ -30,7 +29,7 @@ clap = { workspace = true }
 downcast-rs = "1.2"
 dyn-clone = "1.0.14"
 easy-ext = "1"
-educe = "0.5"
+educe = "0.6"
 either = "1"
 enum-as-inner = "0.6"
 fancy-regex = "0.13.0"
@@ -40,7 +39,7 @@ futures-async-stream = { workspace = true }
 iana-time-zone = "0.1"
 icelake = { workspace = true }
 itertools = { workspace = true }
-jsonbb = "0.1.2"
+jsonbb = { workspace = true }
 linkme = { version = "0.3", features = ["used_linker"] }
 maplit = "1"
 md5 = "0.7.0"
@@ -72,7 +71,6 @@ risingwave_pb = { workspace = true }
 risingwave_rpc_client = { workspace = true }
 risingwave_sqlparser = { workspace = true }
 risingwave_storage = { workspace = true }
-risingwave_udf = { workspace = true }
 risingwave_variables = { workspace = true }
 rw_futures_util = { workspace = true }
 serde = { version = "1", features = ["derive"] }
diff --git a/src/frontend/planner_test/src/lib.rs b/src/frontend/planner_test/src/lib.rs
index ca44fb2b76e8c..ee4f942f09889 100644
--- a/src/frontend/planner_test/src/lib.rs
+++ b/src/frontend/planner_test/src/lib.rs
@@ -21,7 +21,7 @@ risingwave_expr_impl::enable!();
 
 mod resolve_id;
 
-use std::collections::{BTreeMap, HashMap, HashSet};
+use std::collections::{BTreeMap, HashSet};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
@@ -568,9 +568,15 @@ impl TestCase {
                 Statement::CreateSchema {
                     schema_name,
                     if_not_exists,
+                    user_specified,
                 } => {
-                    create_schema::handle_create_schema(handler_args, schema_name, if_not_exists)
-                        .await?;
+                    create_schema::handle_create_schema(
+                        handler_args,
+                        schema_name,
+                        if_not_exists,
+                        user_specified,
+                    )
+                    .await?;
                 }
                 _ => return Err(anyhow!("Unsupported statement type")),
             }
@@ -599,13 +605,13 @@ impl TestCase {
 
         let mut planner = Planner::new(context.clone());
 
-        let mut logical_plan = match planner.plan(bound) {
-            Ok(logical_plan) => {
+        let plan_root = match planner.plan(bound) {
+            Ok(plan_root) => {
                 if self.expected_outputs.contains(&TestType::LogicalPlan) {
                     ret.logical_plan =
-                        Some(explain_plan(&logical_plan.clone().into_unordered_subplan()));
+                        Some(explain_plan(&plan_root.clone().into_unordered_subplan()));
                 }
-                logical_plan
+                plan_root
             }
             Err(err) => {
                 ret.planner_error = Some(err.to_report_string_pretty());
@@ -618,8 +624,9 @@ impl TestCase {
             .contains(&TestType::OptimizedLogicalPlanForBatch)
             || self.expected_outputs.contains(&TestType::OptimizerError)
         {
+            let mut plan_root = plan_root.clone();
             let optimized_logical_plan_for_batch =
-                match logical_plan.gen_optimized_logical_plan_for_batch() {
+                match plan_root.gen_optimized_logical_plan_for_batch() {
                     Ok(optimized_logical_plan_for_batch) => optimized_logical_plan_for_batch,
                     Err(err) => {
                         ret.optimizer_error = Some(err.to_report_string_pretty());
@@ -642,8 +649,9 @@ impl TestCase {
             .contains(&TestType::OptimizedLogicalPlanForStream)
             || self.expected_outputs.contains(&TestType::OptimizerError)
         {
+            let mut plan_root = plan_root.clone();
             let optimized_logical_plan_for_stream =
-                match logical_plan.gen_optimized_logical_plan_for_stream() {
+                match plan_root.gen_optimized_logical_plan_for_stream() {
                     Ok(optimized_logical_plan_for_stream) => optimized_logical_plan_for_stream,
                     Err(err) => {
                         ret.optimizer_error = Some(err.to_report_string_pretty());
@@ -662,15 +670,13 @@ impl TestCase {
         }
 
         'batch: {
-            // if self.batch_plan.is_some()
-            //     || self.batch_plan_proto.is_some()
-            //     || self.batch_error.is_some()
             if self.expected_outputs.contains(&TestType::BatchPlan)
                 || self.expected_outputs.contains(&TestType::BatchPlanProto)
                 || self.expected_outputs.contains(&TestType::BatchError)
             {
-                let batch_plan = match logical_plan.gen_batch_plan() {
-                    Ok(batch_plan) => match logical_plan.gen_batch_distributed_plan(batch_plan) {
+                let mut plan_root = plan_root.clone();
+                let batch_plan = match plan_root.gen_batch_plan() {
+                    Ok(_batch_plan) => match plan_root.gen_batch_distributed_plan() {
                         Ok(batch_plan) => batch_plan,
                         Err(err) => {
                             ret.batch_error = Some(err.to_report_string_pretty());
@@ -701,8 +707,9 @@ impl TestCase {
             if self.expected_outputs.contains(&TestType::BatchLocalPlan)
                 || self.expected_outputs.contains(&TestType::BatchError)
             {
-                let batch_plan = match logical_plan.gen_batch_plan() {
-                    Ok(batch_plan) => match logical_plan.gen_batch_local_plan(batch_plan) {
+                let mut plan_root = plan_root.clone();
+                let batch_plan = match plan_root.gen_batch_plan() {
+                    Ok(_batch_plan) => match plan_root.gen_batch_local_plan() {
                         Ok(batch_plan) => batch_plan,
                         Err(err) => {
                             ret.batch_error = Some(err.to_report_string_pretty());
@@ -728,8 +735,9 @@ impl TestCase {
                 .contains(&TestType::BatchDistributedPlan)
                 || self.expected_outputs.contains(&TestType::BatchError)
             {
-                let batch_plan = match logical_plan.gen_batch_plan() {
-                    Ok(batch_plan) => match logical_plan.gen_batch_distributed_plan(batch_plan) {
+                let mut plan_root = plan_root.clone();
+                let batch_plan = match plan_root.gen_batch_plan() {
+                    Ok(_batch_plan) => match plan_root.gen_batch_distributed_plan() {
                         Ok(batch_plan) => batch_plan,
                         Err(err) => {
                             ret.batch_error = Some(err.to_report_string_pretty());
@@ -823,13 +831,14 @@ impl TestCase {
 
         'sink: {
             if self.expected_outputs.contains(&TestType::SinkPlan) {
+                let mut plan_root = plan_root.clone();
                 let sink_name = "sink_test";
-                let mut options = HashMap::new();
+                let mut options = BTreeMap::new();
                 options.insert("connector".to_string(), "blackhole".to_string());
                 options.insert("type".to_string(), "append-only".to_string());
                 let options = WithOptions::new(options);
                 let format_desc = (&options).try_into().unwrap();
-                match logical_plan.gen_sink_plan(
+                match plan_root.gen_sink_plan(
                     sink_name.to_string(),
                     format!("CREATE SINK {sink_name} AS {}", stmt),
                     options,
diff --git a/src/frontend/planner_test/tests/testdata/input/agg.yaml b/src/frontend/planner_test/tests/testdata/input/agg.yaml
index 154680c8acbc8..70f83549ff18c 100644
--- a/src/frontend/planner_test/tests/testdata/input/agg.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/agg.yaml
@@ -281,12 +281,14 @@
     select distinct v1 from t;
   expected_outputs:
   - logical_plan
+  - stream_plan
 - name: distinct with agg
   sql: |
     create table t (v1 int, v2 int);
     select distinct sum(v1) from t group by v2;
   expected_outputs:
   - logical_plan
+  - stream_plan
 - name: distinct on
   sql: |
     create table t (v1 int, v2 int, v3 int);
diff --git a/src/frontend/planner_test/tests/testdata/input/expr.yaml b/src/frontend/planner_test/tests/testdata/input/expr.yaml
index e19ee4a29126b..42664321bfa37 100644
--- a/src/frontend/planner_test/tests/testdata/input/expr.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/expr.yaml
@@ -443,6 +443,11 @@
     select * from max();
   expected_outputs:
   - binder_error
+- sql: |
+    create table t (k int, j jsonb);
+    select a * k + b from t, jsonb_populate_record(null::struct<a int, b int>, j);
+  expected_outputs:
+  - batch_plan
 - name: Grafana issue-10134
   sql: |
     SELECT * FROM
diff --git a/src/frontend/planner_test/tests/testdata/input/index_selection.yaml b/src/frontend/planner_test/tests/testdata/input/index_selection.yaml
index 22343381dd926..95495a75fcaf4 100644
--- a/src/frontend/planner_test/tests/testdata/input/index_selection.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/index_selection.yaml
@@ -453,3 +453,10 @@
     select ts::timestamptz from t where ts > now();
   expected_outputs:
   - batch_plan
+- name: funtional index with timezone2
+  sql: |
+    create table t (a int, ts timestamp without time zone);
+    create index idx on t (CAST(ts AS timestamptz));
+    select * from t order by ts;
+  expected_outputs:
+    - batch_plan
diff --git a/src/frontend/planner_test/tests/testdata/input/over_window_function.yaml b/src/frontend/planner_test/tests/testdata/input/over_window_function.yaml
index 4406089b273b8..09f9c1aa6ecc4 100644
--- a/src/frontend/planner_test/tests/testdata/input/over_window_function.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/over_window_function.yaml
@@ -591,3 +591,53 @@
     from t;
   expected_outputs:
     - binder_error
+
+# Session frames
+- sql: |
+    create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval);
+    select
+        first_value(i) over (partition by bi order by i session with gap 10) as col1,
+        first_value(bi) over (partition by i order by bi session with gap 10) as col2,
+        first_value(i) over (partition by bi order by d session with gap 1.5) as col3,
+        first_value(i) over (partition by bi order by f session with gap 1.5) as col4,
+        -- first_value(i) over (partition by bi order by da session with gap '1 day') as col5, -- `date` not supported yet
+        -- first_value(i) over (partition by bi order by t session with gap '1 min') as col6, -- `time` not supported yet
+        first_value(i) over (partition by bi order by ts session with gap '1 day 1 hour') as col7,
+        first_value(i) over (partition by bi order by tstz session with gap '1 min') as col8
+    from t;
+  expected_outputs:
+    - logical_plan
+    - optimized_logical_plan_for_stream
+    - stream_error # not supported yet
+    - batch_plan
+- sql: |
+    create table t (i int, bi bigint, ts timestamp, watermark for ts as ts - interval '1 minute') append only;
+    select
+        first_value(i) over (partition by bi order by ts session with gap '10 minutes') as window_start,
+        last_value(i) over (partition by bi order by ts session with gap '10 minutes') as window_end
+    from t;
+  expected_outputs:
+    - logical_plan
+    - eowc_stream_plan
+    - batch_plan
+- sql: |
+    create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval);
+    select
+        count(*) over (partition by 1::int order by da session with gap '1 day') -- `date` not supported yet
+    from t;
+  expected_outputs:
+    - binder_error
+- sql: |
+    create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval);
+    select
+        count(*) over (partition by 1::int order by t session with gap '1 min') -- `time` not supported yet
+    from t;
+  expected_outputs:
+    - binder_error
+- sql: |
+    create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval);
+    select
+        count(*) over (partition by 1::int order by tstz session with gap '1 day 1 hour') -- `timestamptz` +/- 'x month x day' not supported yet
+    from t;
+  expected_outputs:
+    - binder_error
diff --git a/src/frontend/planner_test/tests/testdata/input/recursive_cte.yaml b/src/frontend/planner_test/tests/testdata/input/recursive_cte.yaml
index 411d129d415ff..bdd6f11f0b7be 100644
--- a/src/frontend/planner_test/tests/testdata/input/recursive_cte.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/recursive_cte.yaml
@@ -1,12 +1,16 @@
 - name: basic
   sql: WITH RECURSIVE t1 AS (SELECT 1 AS a UNION ALL SELECT a + 1 FROM t1 WHERE a < 10) SELECT * FROM t1;
   expected_outputs:
-  - planner_error
+  - logical_plan
 - name: output column follows lhs
   sql: WITH RECURSIVE t1 AS (SELECT 1 AS a UNION ALL SELECT a + 1 FROM t1 WHERE a < 10) SELECT a FROM t1;
   expected_outputs:
-  - planner_error
+  - logical_plan
+- name: with normal column
+  sql: WITH RECURSIVE t(a) AS (VALUES(1) UNION ALL SELECT a + 1 FROM t WHERE a < 100) SELECT * FROM t;
+  expected_outputs:
+  - logical_plan
 - name: name a is leaked outside
   sql: WITH RECURSIVE t1 AS (SELECT 1 AS a UNION ALL SELECT a + 1 FROM t1 WHERE a < 10) SELECT a;
   expected_outputs:
-  - binder_error
+  - binder_error
\ No newline at end of file
diff --git a/src/frontend/planner_test/tests/testdata/input/shared_source.yml b/src/frontend/planner_test/tests/testdata/input/shared_source.yml
index 0f68cc25f6288..952ae8dcc5aa0 100644
--- a/src/frontend/planner_test/tests/testdata/input/shared_source.yml
+++ b/src/frontend/planner_test/tests/testdata/input/shared_source.yml
@@ -25,7 +25,7 @@
 # We use with_config_map to control the config when CREATE SOURCE, and use another SET statement to change the config for CREATE MV
 #
 # batch: All 4 plans should be the same.
-# stream: StreamSourceScan (with backfill) should be used only for the last 1. All other 3 use StreamSource.
+# stream: StreamSourceScan (with backfill) should be used only for the last 2. The first 2 use StreamSource. rw_enable_shared_source changes the behavior of CREATE SOURCE, but not CREATE MATERIALIZED VIEW
 - with_config_map:
     rw_enable_shared_source: false
   before:
diff --git a/src/frontend/planner_test/tests/testdata/input/short_circuit.yaml b/src/frontend/planner_test/tests/testdata/input/short_circuit.yaml
index 08b4b08471116..83e518e395b51 100644
--- a/src/frontend/planner_test/tests/testdata/input/short_circuit.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/short_circuit.yaml
@@ -40,6 +40,12 @@
   expected_outputs:
   - logical_plan
   - batch_plan
+- id : short_circuit_optimize_time
+  sql: |
+    select true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true;
+  expected_outputs:
+    - logical_plan
+    - batch_plan
 
 # should *not* be identified as const
 # otherwise the *semantic* will be inconsistent
diff --git a/src/frontend/planner_test/tests/testdata/input/subquery_expr_correlated.yaml b/src/frontend/planner_test/tests/testdata/input/subquery_expr_correlated.yaml
index 5ba9930be99b9..f35703128ae7d 100644
--- a/src/frontend/planner_test/tests/testdata/input/subquery_expr_correlated.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/subquery_expr_correlated.yaml
@@ -508,3 +508,49 @@
   expected_outputs:
     - batch_plan
     - stream_plan
+- name: improve multi scalar subqueries optimization time. issue 16952. case 1.
+  sql: |
+    create table t1(a int, b int);
+    create table t2(c int primary key, d int);
+    select
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col1,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col2,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col3,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col4,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col5,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col6,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col7,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col8,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col9,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col10,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col11,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col12,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col13,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col14,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col15,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col16,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col17,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col18
+    from t1;
+  expected_outputs:
+    - batch_plan
+    - stream_plan
+- name: improve multi scalar subqueries optimization time. issue 16952. case 2.
+  sql: |
+    create table t1(a int, b int);
+    create table t2(c int primary key, d int);
+    create table t3(e int, f int);
+    create table t4(g int, h int);
+    create table t5(i int, j int);
+    create table t6(k int, l int);
+    select
+    COALESCE((SELECT sum(d) FROM t2 left join t3 on e = a and f = c left join t4 on g = a and h = c left join t5 on i = a and j = c WHERE t1.a = t2.c and t1.a = t2.c and j in (select k from t6 where b = l) ), 0) col1,
+    COALESCE((SELECT sum(d) FROM t2 left join t3 on e = a and f = c left join t4 on g = a and h = c left join t5 on i = a and j = c WHERE t1.a = t2.c and t1.a = t2.c and j in (select k from t6 where b = l) ), 0) col2,
+    COALESCE((SELECT sum(d) FROM t2 left join t3 on e = a and f = c left join t4 on g = a and h = c left join t5 on i = a and j = c WHERE t1.a = t2.c and t1.a = t2.c and j in (select k from t6 where b = l) ), 0) col3,
+    COALESCE((SELECT sum(d) FROM t2 left join t3 on e = a and f = c left join t4 on g = a and h = c left join t5 on i = a and j = c WHERE t1.a = t2.c and t1.a = t2.c and j in (select k from t6 where b = l) ), 0) col4,
+    COALESCE((SELECT sum(d) FROM t2 left join t3 on e = a and f = c left join t4 on g = a and h = c left join t5 on i = a and j = c WHERE t1.a = t2.c and t1.a = t2.c and j in (select k from t6 where b = l) ), 0) col5,
+    COALESCE((SELECT sum(d) FROM t2 left join t3 on e = a and f = c left join t4 on g = a and h = c left join t5 on i = a and j = c WHERE t1.a = t2.c and t1.a = t2.c and j in (select k from t6 where b = l) ), 0) col6
+    from t1;
+  expected_outputs:
+    - batch_plan
+    - stream_plan
diff --git a/src/frontend/planner_test/tests/testdata/output/agg.yaml b/src/frontend/planner_test/tests/testdata/output/agg.yaml
index e618a58500783..4c75b83187741 100644
--- a/src/frontend/planner_test/tests/testdata/output/agg.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/agg.yaml
@@ -451,6 +451,12 @@
     LogicalAgg { group_key: [t.v1], aggs: [] }
     └─LogicalProject { exprs: [t.v1] }
       └─LogicalScan { table: t, columns: [t.v1, t.v2, t._row_id] }
+  stream_plan: |-
+    StreamMaterialize { columns: [v1], stream_key: [v1], pk_columns: [v1], pk_conflict: NoCheck }
+    └─StreamProject { exprs: [t.v1], noop_update_hint: true }
+      └─StreamHashAgg { group_key: [t.v1], aggs: [count] }
+        └─StreamExchange { dist: HashShard(t.v1) }
+          └─StreamTableScan { table: t, columns: [t.v1, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
 - name: distinct with agg
   sql: |
     create table t (v1 int, v2 int);
@@ -461,6 +467,15 @@
       └─LogicalAgg { group_key: [t.v2], aggs: [sum(t.v1)] }
         └─LogicalProject { exprs: [t.v2, t.v1] }
           └─LogicalScan { table: t, columns: [t.v1, t.v2, t._row_id] }
+  stream_plan: |-
+    StreamMaterialize { columns: [sum], stream_key: [sum], pk_columns: [sum], pk_conflict: NoCheck }
+    └─StreamProject { exprs: [sum(t.v1)], noop_update_hint: true }
+      └─StreamHashAgg { group_key: [sum(t.v1)], aggs: [count] }
+        └─StreamExchange { dist: HashShard(sum(t.v1)) }
+          └─StreamProject { exprs: [t.v2, sum(t.v1)] }
+            └─StreamHashAgg { group_key: [t.v2], aggs: [sum(t.v1), count] }
+              └─StreamExchange { dist: HashShard(t.v2) }
+                └─StreamTableScan { table: t, columns: [t.v1, t.v2, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
 - name: distinct on
   sql: |
     create table t (v1 int, v2 int, v3 int);
@@ -576,7 +591,7 @@
     └─LogicalScan { table: t, columns: [t.v1] }
   stream_plan: |-
     StreamMaterialize { columns: [v1], stream_key: [v1], pk_columns: [v1], pk_conflict: NoCheck }
-    └─StreamProject { exprs: [t.v1] }
+    └─StreamProject { exprs: [t.v1], noop_update_hint: true }
       └─StreamHashAgg { group_key: [t.v1], aggs: [count] }
         └─StreamExchange { dist: HashShard(t.v1) }
           └─StreamTableScan { table: t, columns: [t.v1, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
@@ -1278,7 +1293,7 @@
   logical_plan: |-
     LogicalProject { exprs: [Case((count(t.v1) <= 1:Int64), null:Decimal, Sqrt(((sum($expr1)::Decimal - ((sum(t.v1)::Decimal * sum(t.v1)::Decimal) / count(t.v1)::Decimal)) / (count(t.v1) - 1:Int64)::Decimal))) as $expr2, Sqrt(((sum($expr1)::Decimal - ((sum(t.v1)::Decimal * sum(t.v1)::Decimal) / count(t.v1)::Decimal)) / count(t.v1)::Decimal)) as $expr3] }
     └─LogicalAgg { aggs: [sum($expr1), sum(t.v1), count(t.v1)] }
-      └─LogicalProject { exprs: [t.v1, (t.v1 * t.v1) as $expr1] }
+      └─LogicalProject { exprs: [(t.v1 * t.v1) as $expr1, t.v1] }
         └─LogicalScan { table: t, columns: [t.v1, t._row_id] }
   batch_plan: |-
     BatchProject { exprs: [Case((sum0(count(t.v1)) <= 1:Int64), null:Decimal, Sqrt(((sum(sum($expr1))::Decimal - (($expr2 * $expr2) / $expr3)) / (sum0(count(t.v1)) - 1:Int64)::Decimal))) as $expr4, Sqrt(((sum(sum($expr1))::Decimal - (($expr2 * $expr2) / $expr3)) / $expr3)) as $expr5] }
@@ -1286,14 +1301,14 @@
       └─BatchSimpleAgg { aggs: [sum(sum($expr1)), sum(sum(t.v1)), sum0(count(t.v1))] }
         └─BatchExchange { order: [], dist: Single }
           └─BatchSimpleAgg { aggs: [sum($expr1), sum(t.v1), count(t.v1)] }
-            └─BatchProject { exprs: [t.v1, (t.v1 * t.v1) as $expr1] }
+            └─BatchProject { exprs: [(t.v1 * t.v1) as $expr1, t.v1] }
               └─BatchScan { table: t, columns: [t.v1], distribution: SomeShard }
   batch_local_plan: |-
     BatchProject { exprs: [Case((count(t.v1) <= 1:Int64), null:Decimal, Sqrt(((sum($expr1)::Decimal - (($expr2 * $expr2) / $expr3)) / (count(t.v1) - 1:Int64)::Decimal))) as $expr4, Sqrt(((sum($expr1)::Decimal - (($expr2 * $expr2) / $expr3)) / $expr3)) as $expr5] }
     └─BatchProject { exprs: [sum($expr1), sum(t.v1), count(t.v1), sum(t.v1)::Decimal as $expr2, count(t.v1)::Decimal as $expr3] }
       └─BatchSimpleAgg { aggs: [sum($expr1), sum(t.v1), count(t.v1)] }
         └─BatchExchange { order: [], dist: Single }
-          └─BatchProject { exprs: [t.v1, (t.v1 * t.v1) as $expr1] }
+          └─BatchProject { exprs: [(t.v1 * t.v1) as $expr1, t.v1] }
             └─BatchScan { table: t, columns: [t.v1], distribution: SomeShard }
   stream_plan: |-
     StreamMaterialize { columns: [stddev_samp, stddev_pop], stream_key: [], pk_columns: [], pk_conflict: NoCheck }
@@ -1302,7 +1317,7 @@
         └─StreamSimpleAgg { aggs: [sum(sum($expr1)), sum(sum(t.v1)), sum0(count(t.v1)), count] }
           └─StreamExchange { dist: Single }
             └─StreamStatelessSimpleAgg { aggs: [sum($expr1), sum(t.v1), count(t.v1)] }
-              └─StreamProject { exprs: [t.v1, (t.v1 * t.v1) as $expr1, t._row_id] }
+              └─StreamProject { exprs: [(t.v1 * t.v1) as $expr1, t.v1, t._row_id] }
                 └─StreamTableScan { table: t, columns: [t.v1, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
 - name: stddev_samp with other columns
   sql: |
@@ -1310,7 +1325,7 @@
   logical_plan: |-
     LogicalProject { exprs: [count('':Varchar), Case((count(1:Int32) <= 1:Int64), null:Decimal, Sqrt(((sum($expr1)::Decimal - ((sum(1:Int32)::Decimal * sum(1:Int32)::Decimal) / count(1:Int32)::Decimal)) / (count(1:Int32) - 1:Int64)::Decimal))) as $expr2] }
     └─LogicalAgg { aggs: [count('':Varchar), sum($expr1), sum(1:Int32), count(1:Int32)] }
-      └─LogicalProject { exprs: ['':Varchar, 1:Int32, (1:Int32 * 1:Int32) as $expr1] }
+      └─LogicalProject { exprs: ['':Varchar, (1:Int32 * 1:Int32) as $expr1, 1:Int32] }
         └─LogicalValues { rows: [[]], schema: Schema { fields: [] } }
 - name: stddev_samp with group
   sql: |
@@ -1319,7 +1334,7 @@
   logical_plan: |-
     LogicalProject { exprs: [Case((count(t.v) <= 1:Int64), null:Decimal, Sqrt(((sum($expr1)::Decimal - ((sum(t.v)::Decimal * sum(t.v)::Decimal) / count(t.v)::Decimal)) / (count(t.v) - 1:Int64)::Decimal))) as $expr2] }
     └─LogicalAgg { group_key: [t.w], aggs: [sum($expr1), sum(t.v), count(t.v)] }
-      └─LogicalProject { exprs: [t.w, t.v, (t.v * t.v) as $expr1] }
+      └─LogicalProject { exprs: [t.w, (t.v * t.v) as $expr1, t.v] }
         └─LogicalScan { table: t, columns: [t.v, t.w, t._row_id] }
 - name: force two phase aggregation should succeed with UpstreamHashShard and SomeShard (batch only).
   sql: |
@@ -1797,7 +1812,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [a, row_number], stream_key: [a], pk_columns: [a], pk_conflict: NoCheck }
     └─StreamOverWindow { window_functions: [row_number() OVER(PARTITION BY t.a ORDER BY t.a DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)] }
-      └─StreamProject { exprs: [t.a] }
+      └─StreamProject { exprs: [t.a], noop_update_hint: true }
         └─StreamHashAgg { group_key: [t.a], aggs: [count] }
           └─StreamExchange { dist: HashShard(t.a) }
             └─StreamTableScan { table: t, columns: [t.a, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
diff --git a/src/frontend/planner_test/tests/testdata/output/cse_expr.yaml b/src/frontend/planner_test/tests/testdata/output/cse_expr.yaml
index ceb706446f986..0e5d72b3499a3 100644
--- a/src/frontend/planner_test/tests/testdata/output/cse_expr.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/cse_expr.yaml
@@ -67,7 +67,7 @@
           └─BatchSimpleAgg { aggs: [sum(sum($expr1)), sum(sum(t.v)), sum0(count(t.v))] }
             └─BatchExchange { order: [], dist: Single }
               └─BatchSimpleAgg { aggs: [sum($expr1), sum(t.v), count(t.v)] }
-                └─BatchProject { exprs: [t.v, (t.v * t.v) as $expr1] }
+                └─BatchProject { exprs: [(t.v * t.v) as $expr1, t.v] }
                   └─BatchScan { table: t, columns: [t.v], distribution: SomeShard }
   stream_plan: |-
     StreamMaterialize { columns: [stddev_pop, stddev_samp, var_pop, var_samp], stream_key: [], pk_columns: [], pk_conflict: NoCheck }
@@ -78,7 +78,7 @@
             └─StreamSimpleAgg { aggs: [sum(sum($expr1)), sum(sum(t.v)), sum0(count(t.v)), count] }
               └─StreamExchange { dist: Single }
                 └─StreamStatelessSimpleAgg { aggs: [sum($expr1), sum(t.v), count(t.v)] }
-                  └─StreamProject { exprs: [t.v, (t.v * t.v) as $expr1, t._row_id] }
+                  └─StreamProject { exprs: [(t.v * t.v) as $expr1, t.v, t._row_id] }
                     └─StreamTableScan { table: t, columns: [t.v, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
 - name: Common sub expression shouldn't extract partial expression of `some`/`all`. See 11766
   sql: |
diff --git a/src/frontend/planner_test/tests/testdata/output/dynamic_filter.yaml b/src/frontend/planner_test/tests/testdata/output/dynamic_filter.yaml
index f819990200576..89aea24f2bd80 100644
--- a/src/frontend/planner_test/tests/testdata/output/dynamic_filter.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/dynamic_filter.yaml
@@ -40,7 +40,7 @@
   stream_error: |-
     Not supported: streaming nested-loop join
     HINT: The non-equal join in the query requires a nested-loop join executor, which could be very expensive to run. Consider rewriting the query to use dynamic filter as a substitute if possible.
-    See also: https://github.com/risingwavelabs/rfcs/blob/main/rfcs/0033-dynamic-filter.md
+    See also: https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/
 - name: With Simple Agg upstream materialized view on inner side
   before:
   - create_tables
@@ -97,7 +97,7 @@
   stream_error: |-
     Not supported: streaming nested-loop join
     HINT: The non-equal join in the query requires a nested-loop join executor, which could be very expensive to run. Consider rewriting the query to use dynamic filter as a substitute if possible.
-    See also: https://github.com/risingwavelabs/rfcs/blob/main/rfcs/0033-dynamic-filter.md
+    See also: https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/
 - name: Ensure error on output columns from inner
   before:
   - create_tables
@@ -111,7 +111,7 @@
   stream_error: |-
     Not supported: streaming nested-loop join
     HINT: The non-equal join in the query requires a nested-loop join executor, which could be very expensive to run. Consider rewriting the query to use dynamic filter as a substitute if possible.
-    See also: https://github.com/risingwavelabs/rfcs/blob/main/rfcs/0033-dynamic-filter.md
+    See also: https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/
 - name: Use Inner Join for equi condition
   before:
   - create_tables
diff --git a/src/frontend/planner_test/tests/testdata/output/except.yaml b/src/frontend/planner_test/tests/testdata/output/except.yaml
index 58b4f757955c3..d19ec9882c42e 100644
--- a/src/frontend/planner_test/tests/testdata/output/except.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/except.yaml
@@ -25,7 +25,7 @@
           └─BatchScan { table: t2, columns: [t2.a, t2.b, t2.c], distribution: SomeShard }
   stream_plan: |-
     StreamMaterialize { columns: [a, b, c], stream_key: [a, b, c], pk_columns: [a, b, c], pk_conflict: NoCheck }
-    └─StreamProject { exprs: [t1.a, t1.b, t1.c] }
+    └─StreamProject { exprs: [t1.a, t1.b, t1.c], noop_update_hint: true }
       └─StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] }
         └─StreamHashJoin { type: LeftAnti, predicate: t1.a IS NOT DISTINCT FROM t2.a AND t1.b IS NOT DISTINCT FROM t2.b AND t1.c IS NOT DISTINCT FROM t2.c, output: all }
           ├─StreamExchange { dist: HashShard(t1.a, t1.b, t1.c) }
@@ -35,7 +35,7 @@
   stream_dist_plan: |+
     Fragment 0
     StreamMaterialize { columns: [a, b, c], stream_key: [a, b, c], pk_columns: [a, b, c], pk_conflict: NoCheck } { tables: [ Materialize: 4294967294 ] }
-    └── StreamProject { exprs: [t1.a, t1.b, t1.c] }
+    └── StreamProject { exprs: [t1.a, t1.b, t1.c], noop_update_hint: true }
         └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } { tables: [ HashAggState: 0 ] }
             └── StreamHashJoin { type: LeftAnti, predicate: t1.a IS NOT DISTINCT FROM t2.a AND t1.b IS NOT DISTINCT FROM t2.b AND t1.c IS NOT DISTINCT FROM t2.c, output: all }
                 ├── tables: [ HashJoinLeft: 1, HashJoinDegreeLeft: 2, HashJoinRight: 3, HashJoinDegreeRight: 4 ]
diff --git a/src/frontend/planner_test/tests/testdata/output/expr.yaml b/src/frontend/planner_test/tests/testdata/output/expr.yaml
index 924547196357e..4ba572a54e600 100644
--- a/src/frontend/planner_test/tests/testdata/output/expr.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/expr.yaml
@@ -88,7 +88,7 @@
     create table t (v1 int);
     SELECT 1 in (3, 0.5*2, min(v1)) from t;
   batch_plan: |-
-    BatchProject { exprs: [true:Boolean] }
+    BatchProject { exprs: [(true:Boolean OR (1:Int32 = min(min(t.v1)))) as $expr1] }
     └─BatchSimpleAgg { aggs: [min(min(t.v1))] }
       └─BatchExchange { order: [], dist: Single }
         └─BatchSimpleAgg { aggs: [min(t.v1)] }
@@ -593,8 +593,9 @@
   batch_error: |
     Expr error
 
-    Caused by:
-      Division by zero
+    Caused by these errors (recent errors listed first):
+      1: error while evaluating expression `general_div('1', '0')`
+      2: Division by zero
 - sql: |
     select * from abs(-1);
   batch_plan: 'BatchValues { rows: [[1:Int32]] }'
@@ -615,6 +616,23 @@
 - sql: |
     select * from max();
   binder_error: 'Invalid input syntax: aggregate functions are not allowed in FROM'
+- sql: |
+    create table t (k int, j jsonb);
+    select a * k + b from t, jsonb_populate_record(null::struct<a int, b int>, j);
+  batch_plan: |-
+    BatchExchange { order: [], dist: Single }
+    └─BatchProject { exprs: [((Field($expr1, 0:Int32) * t.k) + Field($expr1, 1:Int32)) as $expr2] }
+      └─BatchHashJoin { type: Inner, predicate: t.j IS NOT DISTINCT FROM t.j, output: all }
+        ├─BatchExchange { order: [], dist: HashShard(t.j) }
+        │ └─BatchScan { table: t, columns: [t.k, t.j], distribution: SomeShard }
+        └─BatchExchange { order: [], dist: HashShard(t.j) }
+          └─BatchProject { exprs: [t.j, JsonbPopulateRecord(null:Struct(StructType { field_names: ["a", "b"], field_types: [Int32, Int32] }), t.j) as $expr1] }
+            └─BatchNestedLoopJoin { type: Inner, predicate: true, output: all }
+              ├─BatchExchange { order: [], dist: Single }
+              │ └─BatchHashAgg { group_key: [t.j], aggs: [] }
+              │   └─BatchExchange { order: [], dist: HashShard(t.j) }
+              │     └─BatchScan { table: t, columns: [t.j], distribution: SomeShard }
+              └─BatchValues { rows: [[]] }
 - name: Grafana issue-10134
   sql: |
     SELECT * FROM
@@ -631,7 +649,7 @@
   stream_error: |-
     Not supported: streaming nested-loop join
     HINT: The non-equal join in the query requires a nested-loop join executor, which could be very expensive to run. Consider rewriting the query to use dynamic filter as a substitute if possible.
-    See also: https://github.com/risingwavelabs/rfcs/blob/main/rfcs/0033-dynamic-filter.md
+    See also: https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/
 - name: without variadic keyword
   sql: |
     create table t(a varchar, b varchar);
diff --git a/src/frontend/planner_test/tests/testdata/output/format.yaml b/src/frontend/planner_test/tests/testdata/output/format.yaml
index 89b32834c381e..513e377911caf 100644
--- a/src/frontend/planner_test/tests/testdata/output/format.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/format.yaml
@@ -21,8 +21,9 @@
   batch_error: |
     Expr error
 
-    Caused by:
-      too few arguments for format()
+    Caused by these errors (recent errors listed first):
+      1: error while evaluating expression `format('one', 'two')`
+      2: too few arguments for format()
 - sql: |
     SELECT format('Testing %s, %s, %s, %', 'one', 'two', 'three');
   batch_error: |
diff --git a/src/frontend/planner_test/tests/testdata/output/index_selection.yaml b/src/frontend/planner_test/tests/testdata/output/index_selection.yaml
index 7840e6b48e5f8..82c2a5bbf7ec6 100644
--- a/src/frontend/planner_test/tests/testdata/output/index_selection.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/index_selection.yaml
@@ -766,3 +766,12 @@
     BatchExchange { order: [], dist: Single }
     └─BatchProject { exprs: [AtTimeZone(i.ts, 'UTC':Varchar) as $expr1] }
       └─BatchScan { table: i, columns: [i.ts], scan_ranges: [i.AT_TIME_ZONE > Timestamptz(Timestamptz(1617235200000000))], distribution: SomeShard }
+- name: funtional index with timezone2
+  sql: |
+    create table t (a int, ts timestamp without time zone);
+    create index idx on t (CAST(ts AS timestamptz));
+    select * from t order by ts;
+  batch_plan: |-
+    BatchExchange { order: [t.ts ASC], dist: Single }
+    └─BatchSort { order: [t.ts ASC] }
+      └─BatchScan { table: t, columns: [t.a, t.ts], distribution: SomeShard }
diff --git a/src/frontend/planner_test/tests/testdata/output/intersect.yaml b/src/frontend/planner_test/tests/testdata/output/intersect.yaml
index 87834b2ca2d2e..678091bbb2e47 100644
--- a/src/frontend/planner_test/tests/testdata/output/intersect.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/intersect.yaml
@@ -25,7 +25,7 @@
           └─BatchScan { table: t2, columns: [t2.a, t2.b, t2.c], distribution: SomeShard }
   stream_plan: |-
     StreamMaterialize { columns: [a, b, c], stream_key: [a, b, c], pk_columns: [a, b, c], pk_conflict: NoCheck }
-    └─StreamProject { exprs: [t1.a, t1.b, t1.c] }
+    └─StreamProject { exprs: [t1.a, t1.b, t1.c], noop_update_hint: true }
       └─StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] }
         └─StreamHashJoin { type: LeftSemi, predicate: t1.a IS NOT DISTINCT FROM t2.a AND t1.b IS NOT DISTINCT FROM t2.b AND t1.c IS NOT DISTINCT FROM t2.c, output: all }
           ├─StreamExchange { dist: HashShard(t1.a, t1.b, t1.c) }
@@ -35,7 +35,7 @@
   stream_dist_plan: |+
     Fragment 0
     StreamMaterialize { columns: [a, b, c], stream_key: [a, b, c], pk_columns: [a, b, c], pk_conflict: NoCheck } { tables: [ Materialize: 4294967294 ] }
-    └── StreamProject { exprs: [t1.a, t1.b, t1.c] }
+    └── StreamProject { exprs: [t1.a, t1.b, t1.c], noop_update_hint: true }
         └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } { tables: [ HashAggState: 0 ] }
             └── StreamHashJoin { type: LeftSemi, predicate: t1.a IS NOT DISTINCT FROM t2.a AND t1.b IS NOT DISTINCT FROM t2.b AND t1.c IS NOT DISTINCT FROM t2.c, output: all }
                 ├── tables: [ HashJoinLeft: 1, HashJoinDegreeLeft: 2, HashJoinRight: 3, HashJoinDegreeRight: 4 ]
diff --git a/src/frontend/planner_test/tests/testdata/output/lateral_subquery.yaml b/src/frontend/planner_test/tests/testdata/output/lateral_subquery.yaml
index acac3cc236994..39639b3ebb647 100644
--- a/src/frontend/planner_test/tests/testdata/output/lateral_subquery.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/lateral_subquery.yaml
@@ -164,7 +164,7 @@
         ├─StreamExchange { dist: HashShard(t.arr) }
         │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
         └─StreamProjectSet { select_list: [$0, Unnest($0)] }
-          └─StreamProject { exprs: [t.arr] }
+          └─StreamProject { exprs: [t.arr], noop_update_hint: true }
             └─StreamHashAgg { group_key: [t.arr], aggs: [count] }
               └─StreamExchange { dist: HashShard(t.arr) }
                 └─StreamTableScan { table: t, columns: [t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
@@ -193,7 +193,7 @@
               │   │     └─StreamTableScan { table: t1, columns: [t1.n, t1.id, t1._row_id, t1.c], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
               │   └─StreamExchange { dist: HashShard(Unnest(Case(($1 <> '':Varchar), ArrayAppend(StringToArray(Trim($1, ',':Varchar), ',':Varchar), $3), Array($3)))) }
               │     └─StreamProjectSet { select_list: [$0, $1, $2, $3, Unnest(Case(($1 <> '':Varchar), ArrayAppend(StringToArray(Trim($1, ',':Varchar), ',':Varchar), $3), Array($3)))] }
-              │       └─StreamProject { exprs: [t2.p, t2.p, t2.d, t2.d] }
+              │       └─StreamProject { exprs: [t2.p, t2.p, t2.d, t2.d], noop_update_hint: true }
               │         └─StreamHashAgg { group_key: [t2.p, t2.p, t2.d, t2.d], aggs: [count] }
               │           └─StreamExchange { dist: HashShard(t2.p, t2.p, t2.d, t2.d) }
               │             └─StreamProject { exprs: [t2.p, t2.p, t2.d, t2.d, t2._row_id] }
@@ -226,7 +226,7 @@
           └─StreamHashAgg { group_key: [t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5], aggs: [sum($expr4) filter(((t2.c7 * t1.c3) <= (t2.c7 * t2.c8))), sum(t1.c2) filter(((t2.c7 * t1.c3) <= (t2.c7 * t2.c8))), count] }
             └─StreamHashJoin { type: LeftOuter, predicate: t2.c7 IS NOT DISTINCT FROM t2.c7 AND t2.c7 IS NOT DISTINCT FROM t2.c7 AND t2.c8 IS NOT DISTINCT FROM t2.c8 AND t2.c7 IS NOT DISTINCT FROM t2.c7 AND t2.c7 IS NOT DISTINCT FROM t2.c7 AND t2.c8 IS NOT DISTINCT FROM t2.c8 AND t2.c1 IS NOT DISTINCT FROM t2.c1 AND t2.c5 IS NOT DISTINCT FROM t2.c5 AND t2.c5 IS NOT DISTINCT FROM t2.c5, output: [t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5, t1.c3, $expr4, t1.c2, t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5, t1._row_id, $expr1] }
               ├─StreamExchange { dist: HashShard(t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5) }
-              │ └─StreamProject { exprs: [t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5] }
+              │ └─StreamProject { exprs: [t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5], noop_update_hint: true }
               │   └─StreamHashAgg { group_key: [t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5], aggs: [count] }
               │     └─StreamExchange { dist: HashShard(t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5) }
               │       └─StreamTableScan { table: t2, columns: [t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5, t2._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t2._row_id], pk: [_row_id], dist: UpstreamHashShard(t2._row_id) }
@@ -235,7 +235,7 @@
                   └─StreamFilter { predicate: (t2.c5 <= $expr2) }
                     └─StreamHashJoin { type: Inner, predicate: t2.c1 = t1.c1 AND $expr1 = $expr3, output: all }
                       ├─StreamExchange { dist: HashShard(t2.c1, $expr1) }
-                      │ └─StreamProject { exprs: [t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5, AtTimeZone(t2.c5, 'UTC':Varchar)::Date as $expr1] }
+                      │ └─StreamProject { exprs: [t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5, AtTimeZone(t2.c5, 'UTC':Varchar)::Date as $expr1], noop_update_hint: true }
                       │   └─StreamHashAgg { group_key: [t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5], aggs: [count] }
                       │     └─StreamExchange { dist: HashShard(t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5) }
                       │       └─StreamTableScan { table: t2, columns: [t2.c7, t2.c7, t2.c8, t2.c7, t2.c7, t2.c8, t2.c1, t2.c5, t2.c5, t2._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t2._row_id], pk: [_row_id], dist: UpstreamHashShard(t2._row_id) }
diff --git a/src/frontend/planner_test/tests/testdata/output/nexmark.yaml b/src/frontend/planner_test/tests/testdata/output/nexmark.yaml
index dcdd34ed2c153..298653450f659 100644
--- a/src/frontend/planner_test/tests/testdata/output/nexmark.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/nexmark.yaml
@@ -50,8 +50,7 @@
     └─BatchScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time], distribution: SomeShard }
   sink_plan: |-
     StreamSink { type: append-only, columns: [auction, bidder, price, date_time, bid._row_id(hidden)] }
-    └─StreamExchange { dist: Single }
-      └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
+    └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
   stream_plan: |-
     StreamMaterialize { columns: [auction, bidder, price, date_time, bid._row_id(hidden)], stream_key: [bid._row_id], pk_columns: [bid._row_id], pk_conflict: NoCheck }
     └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
@@ -83,9 +82,8 @@
       └─BatchScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time], distribution: SomeShard }
   sink_plan: |-
     StreamSink { type: append-only, columns: [auction, bidder, price, date_time, bid._row_id(hidden)] }
-    └─StreamExchange { dist: Single }
-      └─StreamProject { exprs: [bid.auction, bid.bidder, (0.908:Decimal * bid.price::Decimal) as $expr1, bid.date_time, bid._row_id] }
-        └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
+    └─StreamProject { exprs: [bid.auction, bid.bidder, (0.908:Decimal * bid.price::Decimal) as $expr1, bid.date_time, bid._row_id] }
+      └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
   stream_plan: |-
     StreamMaterialize { columns: [auction, bidder, price, date_time, bid._row_id(hidden)], stream_key: [bid._row_id], pk_columns: [bid._row_id], pk_conflict: NoCheck }
     └─StreamProject { exprs: [bid.auction, bid.bidder, (0.908:Decimal * bid.price::Decimal) as $expr1, bid.date_time, bid._row_id] }
@@ -113,9 +111,8 @@
       └─BatchScan { table: bid, columns: [bid.auction, bid.price], distribution: SomeShard }
   sink_plan: |-
     StreamSink { type: append-only, columns: [auction, price, bid._row_id(hidden)] }
-    └─StreamExchange { dist: Single }
-      └─StreamFilter { predicate: (((((bid.auction = 1007:Int32) OR (bid.auction = 1020:Int32)) OR (bid.auction = 2001:Int32)) OR (bid.auction = 2019:Int32)) OR (bid.auction = 2087:Int32)) }
-        └─StreamTableScan { table: bid, columns: [bid.auction, bid.price, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
+    └─StreamFilter { predicate: (((((bid.auction = 1007:Int32) OR (bid.auction = 1020:Int32)) OR (bid.auction = 2001:Int32)) OR (bid.auction = 2019:Int32)) OR (bid.auction = 2087:Int32)) }
+      └─StreamTableScan { table: bid, columns: [bid.auction, bid.price, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
   stream_plan: |-
     StreamMaterialize { columns: [auction, price, bid._row_id(hidden)], stream_key: [bid._row_id], pk_columns: [bid._row_id], pk_conflict: NoCheck }
     └─StreamFilter { predicate: (((((bid.auction = 1007:Int32) OR (bid.auction = 1020:Int32)) OR (bid.auction = 2001:Int32)) OR (bid.auction = 2019:Int32)) OR (bid.auction = 2087:Int32)) }
@@ -751,7 +748,7 @@
         │     └─StreamProject { exprs: [person.id, person.name, $expr1, ($expr1 + '00:00:10':Interval) as $expr2] }
         │       └─StreamProject { exprs: [person.id, person.name, person.date_time, TumbleStart(person.date_time, '00:00:10':Interval) as $expr1] }
         │         └─StreamTableScan { table: person, columns: [person.id, person.name, person.date_time], stream_scan_type: ArrangementBackfill, stream_key: [person.id], pk: [id], dist: UpstreamHashShard(person.id) }
-        └─StreamProject { exprs: [auction.seller, $expr3, $expr4] }
+        └─StreamProject { exprs: [auction.seller, $expr3, $expr4], noop_update_hint: true }
           └─StreamHashAgg { group_key: [auction.seller, $expr3, $expr4], aggs: [count] }
             └─StreamExchange { dist: HashShard(auction.seller, $expr3, $expr4) }
               └─StreamProject { exprs: [auction.seller, $expr3, ($expr3 + '00:00:10':Interval) as $expr4, auction.id] }
@@ -767,7 +764,7 @@
     StreamHashJoin { type: Inner, predicate: person.id = auction.seller AND $expr1 = $expr3 AND $expr2 = $expr4, output: [person.id, internal_last_seen_value(person.name), $expr1, $expr2, auction.seller, $expr3, $expr4] }
     ├── tables: [ HashJoinLeft: 0, HashJoinDegreeLeft: 1, HashJoinRight: 2, HashJoinDegreeRight: 3 ]
     ├── StreamExchange Hash([0, 1, 2]) from 2
-    └── StreamProject { exprs: [auction.seller, $expr3, $expr4] }
+    └── StreamProject { exprs: [auction.seller, $expr3, $expr4], noop_update_hint: true }
         └── StreamHashAgg { group_key: [auction.seller, $expr3, $expr4], aggs: [count] } { tables: [ HashAggState: 6 ] }
             └── StreamExchange Hash([0, 1, 2]) from 3
 
@@ -911,9 +908,8 @@
       └─BatchScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time], distribution: SomeShard }
   sink_plan: |-
     StreamSink { type: append-only, columns: [auction, bidder, price, date_time, date, time, bid._row_id(hidden)] }
-    └─StreamExchange { dist: Single }
-      └─StreamProject { exprs: [bid.auction, bid.bidder, bid.price, bid.date_time, ToChar(bid.date_time, 'YYYY-MM-DD':Varchar) as $expr1, ToChar(bid.date_time, 'HH:MI':Varchar) as $expr2, bid._row_id] }
-        └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
+    └─StreamProject { exprs: [bid.auction, bid.bidder, bid.price, bid.date_time, ToChar(bid.date_time, 'YYYY-MM-DD':Varchar) as $expr1, ToChar(bid.date_time, 'HH:MI':Varchar) as $expr2, bid._row_id] }
+      └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
   stream_plan: |-
     StreamMaterialize { columns: [auction, bidder, price, date_time, date, time, bid._row_id(hidden)], stream_key: [bid._row_id], pk_columns: [bid._row_id], pk_conflict: NoCheck }
     └─StreamProject { exprs: [bid.auction, bid.bidder, bid.price, bid.date_time, ToChar(bid.date_time, 'YYYY-MM-DD':Varchar) as $expr1, ToChar(bid.date_time, 'HH:MI':Varchar) as $expr2, bid._row_id] }
@@ -1083,10 +1079,9 @@
         └─BatchScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time, bid.extra], distribution: SomeShard }
   sink_plan: |-
     StreamSink { type: append-only, columns: [auction, bidder, price, bidtimetype, date_time, extra, bid._row_id(hidden)] }
-    └─StreamExchange { dist: Single }
-      └─StreamProject { exprs: [bid.auction, bid.bidder, (0.908:Decimal * bid.price::Decimal) as $expr1, Case(((Extract('HOUR':Varchar, bid.date_time) >= 8:Decimal) AND (Extract('HOUR':Varchar, bid.date_time) <= 18:Decimal)), 'dayTime':Varchar, ((Extract('HOUR':Varchar, bid.date_time) <= 6:Decimal) OR (Extract('HOUR':Varchar, bid.date_time) >= 20:Decimal)), 'nightTime':Varchar, 'otherTime':Varchar) as $expr2, bid.date_time, bid.extra, bid._row_id] }
-        └─StreamFilter { predicate: ((0.908:Decimal * bid.price::Decimal) > 1000000:Decimal) AND ((0.908:Decimal * bid.price::Decimal) < 50000000:Decimal) }
-          └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time, bid.extra, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
+    └─StreamProject { exprs: [bid.auction, bid.bidder, (0.908:Decimal * bid.price::Decimal) as $expr1, Case(((Extract('HOUR':Varchar, bid.date_time) >= 8:Decimal) AND (Extract('HOUR':Varchar, bid.date_time) <= 18:Decimal)), 'dayTime':Varchar, ((Extract('HOUR':Varchar, bid.date_time) <= 6:Decimal) OR (Extract('HOUR':Varchar, bid.date_time) >= 20:Decimal)), 'nightTime':Varchar, 'otherTime':Varchar) as $expr2, bid.date_time, bid.extra, bid._row_id] }
+      └─StreamFilter { predicate: ((0.908:Decimal * bid.price::Decimal) > 1000000:Decimal) AND ((0.908:Decimal * bid.price::Decimal) < 50000000:Decimal) }
+        └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time, bid.extra, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
   stream_plan: |-
     StreamMaterialize { columns: [auction, bidder, price, bidtimetype, date_time, extra, bid._row_id(hidden)], stream_key: [bid._row_id], pk_columns: [bid._row_id], pk_conflict: NoCheck }
     └─StreamProject { exprs: [bid.auction, bid.bidder, (0.908:Decimal * bid.price::Decimal) as $expr1, Case(((Extract('HOUR':Varchar, bid.date_time) >= 8:Decimal) AND (Extract('HOUR':Varchar, bid.date_time) <= 18:Decimal)), 'dayTime':Varchar, ((Extract('HOUR':Varchar, bid.date_time) <= 6:Decimal) OR (Extract('HOUR':Varchar, bid.date_time) >= 20:Decimal)), 'nightTime':Varchar, 'otherTime':Varchar) as $expr2, bid.date_time, bid.extra, bid._row_id] }
@@ -1747,9 +1742,8 @@
       └─BatchScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.channel, bid.url], distribution: SomeShard }
   sink_plan: |-
     StreamSink { type: append-only, columns: [auction, bidder, price, channel, dir1, dir2, dir3, bid._row_id(hidden)] }
-    └─StreamExchange { dist: Single }
-      └─StreamProject { exprs: [bid.auction, bid.bidder, bid.price, bid.channel, SplitPart(bid.url, '/':Varchar, 4:Int32) as $expr1, SplitPart(bid.url, '/':Varchar, 5:Int32) as $expr2, SplitPart(bid.url, '/':Varchar, 6:Int32) as $expr3, bid._row_id] }
-        └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.channel, bid.url, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
+    └─StreamProject { exprs: [bid.auction, bid.bidder, bid.price, bid.channel, SplitPart(bid.url, '/':Varchar, 4:Int32) as $expr1, SplitPart(bid.url, '/':Varchar, 5:Int32) as $expr2, SplitPart(bid.url, '/':Varchar, 6:Int32) as $expr3, bid._row_id] }
+      └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.channel, bid.url, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
   stream_plan: |-
     StreamMaterialize { columns: [auction, bidder, price, channel, dir1, dir2, dir3, bid._row_id(hidden)], stream_key: [bid._row_id], pk_columns: [bid._row_id], pk_conflict: NoCheck }
     └─StreamProject { exprs: [bid.auction, bid.bidder, bid.price, bid.channel, SplitPart(bid.url, '/':Varchar, 4:Int32) as $expr1, SplitPart(bid.url, '/':Varchar, 5:Int32) as $expr2, SplitPart(bid.url, '/':Varchar, 6:Int32) as $expr3, bid._row_id] }
diff --git a/src/frontend/planner_test/tests/testdata/output/over_window_function.yaml b/src/frontend/planner_test/tests/testdata/output/over_window_function.yaml
index fbdccf685ca2e..3449606ad97ba 100644
--- a/src/frontend/planner_test/tests/testdata/output/over_window_function.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/over_window_function.yaml
@@ -73,11 +73,11 @@
       └─LogicalProject { exprs: [t.x, t._row_id] }
         └─LogicalScan { table: t, columns: [t.x, t._row_id] }
   batch_error: |-
-    Feature is not yet implemented: Window function with empty PARTITION BY is not supported yet
-    No tracking issue yet. Feel free to submit a feature request at https://github.com/risingwavelabs/risingwave/issues/new?labels=type%2Ffeature&template=feature_request.yml
+    Feature is not yet implemented: Window function with empty PARTITION BY is not supported because of potential bad performance. If you really need this, please workaround with something like `PARTITION BY 1::int`.
+    Tracking issue: https://github.com/risingwavelabs/risingwave/issues/11505
   stream_error: |-
-    Feature is not yet implemented: Window function with empty PARTITION BY is not supported yet
-    No tracking issue yet. Feel free to submit a feature request at https://github.com/risingwavelabs/risingwave/issues/new?labels=type%2Ffeature&template=feature_request.yml
+    Feature is not yet implemented: Window function with empty PARTITION BY is not supported because of potential bad performance. If you really need this, please workaround with something like `PARTITION BY 1::int`.
+    Tracking issue: https://github.com/risingwavelabs/risingwave/issues/11505
 - id: lead with offset argument and empty over clause
   sql: |
     create table t(x int);
@@ -88,11 +88,11 @@
       └─LogicalProject { exprs: [t.x, t._row_id, 2:Int32] }
         └─LogicalScan { table: t, columns: [t.x, t._row_id] }
   batch_error: |-
-    Feature is not yet implemented: Window function with empty PARTITION BY is not supported yet
-    No tracking issue yet. Feel free to submit a feature request at https://github.com/risingwavelabs/risingwave/issues/new?labels=type%2Ffeature&template=feature_request.yml
+    Feature is not yet implemented: Window function with empty PARTITION BY is not supported because of potential bad performance. If you really need this, please workaround with something like `PARTITION BY 1::int`.
+    Tracking issue: https://github.com/risingwavelabs/risingwave/issues/11505
   stream_error: |-
-    Feature is not yet implemented: Window function with empty PARTITION BY is not supported yet
-    No tracking issue yet. Feel free to submit a feature request at https://github.com/risingwavelabs/risingwave/issues/new?labels=type%2Ffeature&template=feature_request.yml
+    Feature is not yet implemented: Window function with empty PARTITION BY is not supported because of potential bad performance. If you really need this, please workaround with something like `PARTITION BY 1::int`.
+    Tracking issue: https://github.com/risingwavelabs/risingwave/issues/11505
 - id: lead with non-const offset argument and empty over clause
   sql: |
     create table t(x int);
@@ -164,7 +164,7 @@
   stream_error: |-
     Not supported: streaming nested-loop join
     HINT: The non-equal join in the query requires a nested-loop join executor, which could be very expensive to run. Consider rewriting the query to use dynamic filter as a substitute if possible.
-    See also: https://github.com/risingwavelabs/rfcs/blob/main/rfcs/0033-dynamic-filter.md
+    See also: https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/
 - id: aggregate with over clause, without ORDER BY and frame definition
   sql: |
     create table t(x int, y int, z int, w int);
@@ -1235,3 +1235,115 @@
     Caused by these errors (recent errors listed first):
       1: Expr error
       2: for frame order column of type `timestamptz`, offset should not have non-zero `month` and `day`
+- sql: |
+    create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval);
+    select
+        first_value(i) over (partition by bi order by i session with gap 10) as col1,
+        first_value(bi) over (partition by i order by bi session with gap 10) as col2,
+        first_value(i) over (partition by bi order by d session with gap 1.5) as col3,
+        first_value(i) over (partition by bi order by f session with gap 1.5) as col4,
+        -- first_value(i) over (partition by bi order by da session with gap '1 day') as col5, -- `date` not supported yet
+        -- first_value(i) over (partition by bi order by t session with gap '1 min') as col6, -- `time` not supported yet
+        first_value(i) over (partition by bi order by ts session with gap '1 day 1 hour') as col7,
+        first_value(i) over (partition by bi order by tstz session with gap '1 min') as col8
+    from t;
+  logical_plan: |-
+    LogicalProject { exprs: [first_value, first_value, first_value, first_value, first_value, first_value] }
+    └─LogicalOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.i ASC SESSION WITH GAP 10), first_value(t.bi) OVER(PARTITION BY t.i ORDER BY t.bi ASC SESSION WITH GAP 10), first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.d ASC SESSION WITH GAP 1.5), first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.f ASC SESSION WITH GAP 1.5), first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.ts ASC SESSION WITH GAP 1 day 01:00:00), first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.tstz ASC SESSION WITH GAP 00:01:00)] }
+      └─LogicalProject { exprs: [t.i, t.bi, t.d, t.f, t.da, t.t, t.ts, t.tstz, t.itv, t._row_id] }
+        └─LogicalScan { table: t, columns: [t.i, t.bi, t.d, t.f, t.da, t.t, t.ts, t.tstz, t.itv, t._row_id] }
+  optimized_logical_plan_for_stream: |-
+    LogicalProject { exprs: [first_value, first_value, first_value, first_value, first_value, first_value] }
+    └─LogicalOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.tstz ASC SESSION WITH GAP 00:01:00)] }
+      └─LogicalProject { exprs: [t.i, t.bi, t.tstz, first_value, first_value, first_value, first_value, first_value] }
+        └─LogicalOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.ts ASC SESSION WITH GAP 1 day 01:00:00)] }
+          └─LogicalProject { exprs: [t.i, t.bi, t.ts, t.tstz, first_value, first_value, first_value, first_value] }
+            └─LogicalOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.f ASC SESSION WITH GAP 1.5)] }
+              └─LogicalProject { exprs: [t.i, t.bi, t.f, t.ts, t.tstz, first_value, first_value, first_value] }
+                └─LogicalOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.d ASC SESSION WITH GAP 1.5)] }
+                  └─LogicalOverWindow { window_functions: [first_value(t.bi) OVER(PARTITION BY t.i ORDER BY t.bi ASC SESSION WITH GAP 10)] }
+                    └─LogicalOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.i ASC SESSION WITH GAP 10)] }
+                      └─LogicalScan { table: t, columns: [t.i, t.bi, t.d, t.f, t.ts, t.tstz] }
+  batch_plan: |-
+    BatchExchange { order: [], dist: Single }
+    └─BatchProject { exprs: [first_value, first_value, first_value, first_value, first_value, first_value] }
+      └─BatchOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.tstz ASC SESSION WITH GAP 00:01:00)] }
+        └─BatchSort { order: [t.bi ASC, t.tstz ASC] }
+          └─BatchProject { exprs: [t.i, t.bi, t.tstz, first_value, first_value, first_value, first_value, first_value] }
+            └─BatchOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.ts ASC SESSION WITH GAP 1 day 01:00:00)] }
+              └─BatchSort { order: [t.bi ASC, t.ts ASC] }
+                └─BatchProject { exprs: [t.i, t.bi, t.ts, t.tstz, first_value, first_value, first_value, first_value] }
+                  └─BatchOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.f ASC SESSION WITH GAP 1.5)] }
+                    └─BatchSort { order: [t.bi ASC, t.f ASC] }
+                      └─BatchProject { exprs: [t.i, t.bi, t.f, t.ts, t.tstz, first_value, first_value, first_value] }
+                        └─BatchOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.d ASC SESSION WITH GAP 1.5)] }
+                          └─BatchExchange { order: [t.bi ASC, t.d ASC], dist: HashShard(t.bi) }
+                            └─BatchSort { order: [t.bi ASC, t.d ASC] }
+                              └─BatchOverWindow { window_functions: [first_value(t.bi) OVER(PARTITION BY t.i ORDER BY t.bi ASC SESSION WITH GAP 10)] }
+                                └─BatchExchange { order: [t.i ASC, t.bi ASC], dist: HashShard(t.i) }
+                                  └─BatchSort { order: [t.i ASC, t.bi ASC] }
+                                    └─BatchOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.i ASC SESSION WITH GAP 10)] }
+                                      └─BatchExchange { order: [t.bi ASC, t.i ASC], dist: HashShard(t.bi) }
+                                        └─BatchSort { order: [t.bi ASC, t.i ASC] }
+                                          └─BatchScan { table: t, columns: [t.i, t.bi, t.d, t.f, t.ts, t.tstz], distribution: SomeShard }
+  stream_error: |-
+    Feature is not yet implemented: Session frame is not yet supported in general streaming mode. Please consider using Emit-On-Window-Close mode.
+    No tracking issue yet. Feel free to submit a feature request at https://github.com/risingwavelabs/risingwave/issues/new?labels=type%2Ffeature&template=feature_request.yml
+- sql: |
+    create table t (i int, bi bigint, ts timestamp, watermark for ts as ts - interval '1 minute') append only;
+    select
+        first_value(i) over (partition by bi order by ts session with gap '10 minutes') as window_start,
+        last_value(i) over (partition by bi order by ts session with gap '10 minutes') as window_end
+    from t;
+  logical_plan: |-
+    LogicalProject { exprs: [first_value, last_value] }
+    └─LogicalOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.ts ASC SESSION WITH GAP 00:10:00), last_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.ts ASC SESSION WITH GAP 00:10:00)] }
+      └─LogicalProject { exprs: [t.i, t.bi, t.ts, t._row_id] }
+        └─LogicalScan { table: t, columns: [t.i, t.bi, t.ts, t._row_id] }
+  batch_plan: |-
+    BatchExchange { order: [], dist: Single }
+    └─BatchProject { exprs: [first_value, last_value] }
+      └─BatchOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.ts ASC SESSION WITH GAP 00:10:00), last_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.ts ASC SESSION WITH GAP 00:10:00)] }
+        └─BatchExchange { order: [t.bi ASC, t.ts ASC], dist: HashShard(t.bi) }
+          └─BatchSort { order: [t.bi ASC, t.ts ASC] }
+            └─BatchScan { table: t, columns: [t.i, t.bi, t.ts], distribution: SomeShard }
+  eowc_stream_plan: |-
+    StreamMaterialize { columns: [window_start, window_end, t._row_id(hidden), t.bi(hidden)], stream_key: [t._row_id, t.bi], pk_columns: [t._row_id, t.bi], pk_conflict: NoCheck }
+    └─StreamProject { exprs: [first_value, last_value, t._row_id, t.bi] }
+      └─StreamEowcOverWindow { window_functions: [first_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.ts ASC SESSION WITH GAP 00:10:00), last_value(t.i) OVER(PARTITION BY t.bi ORDER BY t.ts ASC SESSION WITH GAP 00:10:00)] }
+        └─StreamEowcSort { sort_column: t.ts }
+          └─StreamExchange { dist: HashShard(t.bi) }
+            └─StreamTableScan { table: t, columns: [t.i, t.bi, t.ts, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+- sql: |
+    create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval);
+    select
+        count(*) over (partition by 1::int order by da session with gap '1 day') -- `date` not supported yet
+    from t;
+  binder_error: |
+    Failed to bind expression: count(*) OVER (PARTITION BY CAST(1 AS INT) ORDER BY da SESSION WITH GAP '1 day')
+
+    Caused by:
+      Feature is not yet implemented: `SESSION` frame with offset of type `date` is not implemented yet, please manually cast the `ORDER BY` column to `timestamp`
+    No tracking issue yet. Feel free to submit a feature request at https://github.com/risingwavelabs/risingwave/issues/new?labels=type%2Ffeature&template=feature_request.yml
+- sql: |
+    create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval);
+    select
+        count(*) over (partition by 1::int order by t session with gap '1 min') -- `time` not supported yet
+    from t;
+  binder_error: |
+    Failed to bind expression: count(*) OVER (PARTITION BY CAST(1 AS INT) ORDER BY t SESSION WITH GAP '1 min')
+
+    Caused by:
+      Feature is not yet implemented: `SESSION` frame with offset of type `time without time zone` is not implemented yet, please manually cast the `ORDER BY` column to `timestamp`
+    No tracking issue yet. Feel free to submit a feature request at https://github.com/risingwavelabs/risingwave/issues/new?labels=type%2Ffeature&template=feature_request.yml
+- sql: |
+    create table t (i int, bi bigint, d decimal, f float, da date, t time, ts timestamp, tstz timestamptz, itv interval);
+    select
+        count(*) over (partition by 1::int order by tstz session with gap '1 day 1 hour') -- `timestamptz` +/- 'x month x day' not supported yet
+    from t;
+  binder_error: |
+    Failed to bind expression: count(*) OVER (PARTITION BY CAST(1 AS INT) ORDER BY tstz SESSION WITH GAP '1 day 1 hour')
+
+    Caused by these errors (recent errors listed first):
+      1: Expr error
+      2: for session order column of type `timestamptz`, gap should not have non-zero `month` and `day`
diff --git a/src/frontend/planner_test/tests/testdata/output/pk_derive.yaml b/src/frontend/planner_test/tests/testdata/output/pk_derive.yaml
index 5c8e87aea559f..7392c88bde3cc 100644
--- a/src/frontend/planner_test/tests/testdata/output/pk_derive.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/pk_derive.yaml
@@ -77,7 +77,7 @@
     └─LogicalScan { table: t, columns: [t.v1, t.v2, t.v3] }
   stream_plan: |-
     StreamMaterialize { columns: [v1, v2, v3], stream_key: [v1, v2, v3], pk_columns: [v1, v2, v3], pk_conflict: NoCheck }
-    └─StreamProject { exprs: [t.v1, t.v2, t.v3] }
+    └─StreamProject { exprs: [t.v1, t.v2, t.v3], noop_update_hint: true }
       └─StreamHashAgg { group_key: [t.v1, t.v2, t.v3], aggs: [count] }
         └─StreamExchange { dist: HashShard(t.v1, t.v2, t.v3) }
           └─StreamTableScan { table: t, columns: [t.v1, t.v2, t.v3, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
diff --git a/src/frontend/planner_test/tests/testdata/output/project_set.yaml b/src/frontend/planner_test/tests/testdata/output/project_set.yaml
index 890480493af92..8373ea5187fca 100644
--- a/src/frontend/planner_test/tests/testdata/output/project_set.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/project_set.yaml
@@ -118,7 +118,7 @@
           └─BatchScan { table: t, columns: [t.x], distribution: SomeShard }
   stream_plan: |-
     StreamMaterialize { columns: [unnest], stream_key: [unnest], pk_columns: [unnest], pk_conflict: NoCheck }
-    └─StreamProject { exprs: [Unnest($0)] }
+    └─StreamProject { exprs: [Unnest($0)], noop_update_hint: true }
       └─StreamHashAgg { group_key: [Unnest($0)], aggs: [count] }
         └─StreamExchange { dist: HashShard(Unnest($0)) }
           └─StreamProjectSet { select_list: [Unnest($0), $1] }
diff --git a/src/frontend/planner_test/tests/testdata/output/range_scan.yaml b/src/frontend/planner_test/tests/testdata/output/range_scan.yaml
index b54dc13b5470b..95a4315481ef9 100644
--- a/src/frontend/planner_test/tests/testdata/output/range_scan.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/range_scan.yaml
@@ -35,8 +35,9 @@
   batch_error: |
     Expr error
 
-    Caused by:
-      Division by zero
+    Caused by these errors (recent errors listed first):
+      1: error while evaluating expression `general_div('1', '0')`
+      2: Division by zero
 - before:
   - create_table_and_mv
   sql: |
@@ -44,8 +45,9 @@
   batch_error: |
     Expr error
 
-    Caused by:
-      Numeric out of range
+    Caused by these errors (recent errors listed first):
+      1: error while evaluating expression `general_add('2147483647', '1')`
+      2: Numeric out of range
 - before:
   - create_table_and_mv
   sql: |
@@ -53,8 +55,9 @@
   batch_error: |
     Expr error
 
-    Caused by:
-      Parse error: bigint invalid digit found in string
+    Caused by these errors (recent errors listed first):
+      1: error while evaluating expression `str_parse('a')`
+      2: Parse error: bigint invalid digit found in string
 - before:
   - create_table_and_mv
   sql: |
@@ -62,8 +65,9 @@
   batch_error: |
     Expr error
 
-    Caused by:
-      Parse error: bigint invalid digit found in string
+    Caused by these errors (recent errors listed first):
+      1: error while evaluating expression `str_parse('a')`
+      2: Parse error: bigint invalid digit found in string
 - before:
   - create_table_and_mv
   sql: |
@@ -147,8 +151,9 @@
   batch_error: |
     Expr error
 
-    Caused by:
-      Parse error: bigint invalid digit found in string
+    Caused by these errors (recent errors listed first):
+      1: error while evaluating expression `str_parse('43.0')`
+      2: Parse error: bigint invalid digit found in string
 - before:
   - create_table_and_mv
   sql: |
diff --git a/src/frontend/planner_test/tests/testdata/output/recursive_cte.yaml b/src/frontend/planner_test/tests/testdata/output/recursive_cte.yaml
index 18e0e18726d38..93872d5848bd6 100644
--- a/src/frontend/planner_test/tests/testdata/output/recursive_cte.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/recursive_cte.yaml
@@ -1,14 +1,33 @@
 # This file is automatically generated. See `src/frontend/planner_test/README.md` for more information.
 - name: basic
   sql: WITH RECURSIVE t1 AS (SELECT 1 AS a UNION ALL SELECT a + 1 FROM t1 WHERE a < 10) SELECT * FROM t1;
-  planner_error: |-
-    Feature is not yet implemented: recursive CTE is not supported
-    Tracking issue: https://github.com/risingwavelabs/risingwave/issues/15135
+  logical_plan: |-
+    LogicalProject { exprs: [$expr1] }
+    └─LogicalRecursiveUnion { id: 4 }
+      ├─LogicalProject { exprs: [1:Int32] }
+      │ └─LogicalValues { rows: [[]], schema: Schema { fields: [] } }
+      └─LogicalProject { exprs: [(1:Int32 + 1:Int32) as $expr1] }
+        └─LogicalFilter { predicate: (1:Int32 < 10:Int32) }
+          └─LogicalCteRef { share_id: 0 }
 - name: output column follows lhs
   sql: WITH RECURSIVE t1 AS (SELECT 1 AS a UNION ALL SELECT a + 1 FROM t1 WHERE a < 10) SELECT a FROM t1;
-  planner_error: |-
-    Feature is not yet implemented: recursive CTE is not supported
-    Tracking issue: https://github.com/risingwavelabs/risingwave/issues/15135
+  logical_plan: |-
+    LogicalProject { exprs: [$expr1] }
+    └─LogicalRecursiveUnion { id: 4 }
+      ├─LogicalProject { exprs: [1:Int32] }
+      │ └─LogicalValues { rows: [[]], schema: Schema { fields: [] } }
+      └─LogicalProject { exprs: [(1:Int32 + 1:Int32) as $expr1] }
+        └─LogicalFilter { predicate: (1:Int32 < 10:Int32) }
+          └─LogicalCteRef { share_id: 0 }
+- name: with normal column
+  sql: WITH RECURSIVE t(a) AS (VALUES(1) UNION ALL SELECT a + 1 FROM t WHERE a < 100) SELECT * FROM t;
+  logical_plan: |-
+    LogicalProject { exprs: [$expr1] }
+    └─LogicalRecursiveUnion { id: 3 }
+      ├─LogicalValues { rows: [[1:Int32]], schema: Schema { fields: [*VALUES*_0.column_0:Int32] } }
+      └─LogicalProject { exprs: [(*VALUES*_0.column_0 + 1:Int32) as $expr1] }
+        └─LogicalFilter { predicate: (*VALUES*_0.column_0 < 100:Int32) }
+          └─LogicalCteRef { share_id: 0 }
 - name: name a is leaked outside
   sql: WITH RECURSIVE t1 AS (SELECT 1 AS a UNION ALL SELECT a + 1 FROM t1 WHERE a < 10) SELECT a;
   binder_error: |
diff --git a/src/frontend/planner_test/tests/testdata/output/share.yaml b/src/frontend/planner_test/tests/testdata/output/share.yaml
index cfe5d841cb2c1..2cf3aee9fe043 100644
--- a/src/frontend/planner_test/tests/testdata/output/share.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/share.yaml
@@ -222,12 +222,12 @@
           └─StreamStatelessSimpleAgg { aggs: [count] }
             └─StreamHashJoin { type: Inner, predicate: t.a = t.a, output: all }
               ├─StreamShare { id: 4 }
-              │ └─StreamProject { exprs: [t.a] }
+              │ └─StreamProject { exprs: [t.a], noop_update_hint: true }
               │   └─StreamHashAgg { group_key: [t.a], aggs: [count] }
               │     └─StreamExchange { dist: HashShard(t.a) }
               │       └─StreamTableScan { table: t, columns: [t.a, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
               └─StreamShare { id: 4 }
-                └─StreamProject { exprs: [t.a] }
+                └─StreamProject { exprs: [t.a], noop_update_hint: true }
                   └─StreamHashAgg { group_key: [t.a], aggs: [count] }
                     └─StreamExchange { dist: HashShard(t.a) }
                       └─StreamTableScan { table: t, columns: [t.a, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
@@ -251,7 +251,7 @@
         └── StreamExchange NoShuffle from 4
 
     Fragment 2
-    StreamProject { exprs: [t.a] }
+    StreamProject { exprs: [t.a], noop_update_hint: true }
     └── StreamHashAgg { group_key: [t.a], aggs: [count] } { tables: [ HashAggState: 5 ] }
         └── StreamExchange Hash([0]) from 3
 
diff --git a/src/frontend/planner_test/tests/testdata/output/shared_source.yml b/src/frontend/planner_test/tests/testdata/output/shared_source.yml
index 5bf3739f28411..39fb6b799fb83 100644
--- a/src/frontend/planner_test/tests/testdata/output/shared_source.yml
+++ b/src/frontend/planner_test/tests/testdata/output/shared_source.yml
@@ -73,7 +73,7 @@
     StreamMaterialize { columns: [x, y, _row_id(hidden)], stream_key: [_row_id], pk_columns: [_row_id], pk_conflict: NoCheck }
     └─StreamProject { exprs: [x, y, _row_id] }
       └─StreamRowIdGen { row_id_index: 3 }
-        └─StreamSource { source: s, columns: [x, y, _rw_kafka_timestamp, _row_id, _rw_kafka_partition, _rw_kafka_offset] }
+        └─StreamSourceScan { columns: [x, y, _rw_kafka_timestamp, _row_id, _rw_kafka_partition, _rw_kafka_offset] }
   with_config_map:
     rw_enable_shared_source: 'true'
 - before:
diff --git a/src/frontend/planner_test/tests/testdata/output/short_circuit.yaml b/src/frontend/planner_test/tests/testdata/output/short_circuit.yaml
index f870d5d09e9b6..47f1c73897a25 100644
--- a/src/frontend/planner_test/tests/testdata/output/short_circuit.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/short_circuit.yaml
@@ -44,3 +44,10 @@
     BatchExchange { order: [], dist: Single }
     └─BatchProject { exprs: [false:Boolean] }
       └─BatchScan { table: t1, columns: [t1.c1], distribution: SomeShard }
+- id: short_circuit_optimize_time
+  sql: |
+    select true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true and true;
+  logical_plan: |-
+    LogicalProject { exprs: [(((((((((((((((((((((((((true:Boolean AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) AND true:Boolean) as $expr1] }
+    └─LogicalValues { rows: [[]], schema: Schema { fields: [] } }
+  batch_plan: 'BatchValues { rows: [[true:Boolean]] }'
diff --git a/src/frontend/planner_test/tests/testdata/output/subquery.yaml b/src/frontend/planner_test/tests/testdata/output/subquery.yaml
index e17f2f3cf699a..e113a0aca4d1d 100644
--- a/src/frontend/planner_test/tests/testdata/output/subquery.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/subquery.yaml
@@ -241,7 +241,7 @@
         │       │ │ │ │ │ │ └─LogicalProject { exprs: [rw_system_tables.id, rw_system_tables.name, 'system table':Varchar, rw_system_tables.schema_id, rw_system_tables.owner, rw_system_tables.definition, rw_system_tables.acl] }
         │       │ │ │ │ │ │   └─LogicalSysScan { table: rw_system_tables, columns: [rw_system_tables.id, rw_system_tables.name, rw_system_tables.schema_id, rw_system_tables.owner, rw_system_tables.definition, rw_system_tables.acl] }
         │       │ │ │ │ │ └─LogicalProject { exprs: [rw_sources.id, rw_sources.name, 'source':Varchar, rw_sources.schema_id, rw_sources.owner, rw_sources.definition, rw_sources.acl] }
-        │       │ │ │ │ │   └─LogicalSysScan { table: rw_sources, columns: [rw_sources.id, rw_sources.name, rw_sources.schema_id, rw_sources.owner, rw_sources.connector, rw_sources.columns, rw_sources.format, rw_sources.row_encode, rw_sources.append_only, rw_sources.connection_id, rw_sources.definition, rw_sources.acl, rw_sources.initialized_at, rw_sources.created_at, rw_sources.initialized_at_cluster_version, rw_sources.created_at_cluster_version] }
+        │       │ │ │ │ │   └─LogicalSysScan { table: rw_sources, columns: [rw_sources.id, rw_sources.name, rw_sources.schema_id, rw_sources.owner, rw_sources.connector, rw_sources.columns, rw_sources.format, rw_sources.row_encode, rw_sources.append_only, rw_sources.associated_table_id, rw_sources.connection_id, rw_sources.definition, rw_sources.acl, rw_sources.initialized_at, rw_sources.created_at, rw_sources.initialized_at_cluster_version, rw_sources.created_at_cluster_version] }
         │       │ │ │ │ └─LogicalProject { exprs: [rw_indexes.id, rw_indexes.name, 'index':Varchar, rw_indexes.schema_id, rw_indexes.owner, rw_indexes.definition, rw_indexes.acl] }
         │       │ │ │ │   └─LogicalSysScan { table: rw_indexes, columns: [rw_indexes.id, rw_indexes.name, rw_indexes.primary_table_id, rw_indexes.key_columns, rw_indexes.include_columns, rw_indexes.schema_id, rw_indexes.owner, rw_indexes.definition, rw_indexes.acl, rw_indexes.initialized_at, rw_indexes.created_at, rw_indexes.initialized_at_cluster_version, rw_indexes.created_at_cluster_version] }
         │       │ │ │ └─LogicalProject { exprs: [rw_sinks.id, rw_sinks.name, 'sink':Varchar, rw_sinks.schema_id, rw_sinks.owner, rw_sinks.definition, rw_sinks.acl] }
@@ -331,9 +331,9 @@
         │   └─StreamHopWindow { time_col: auction.date_time, slide: 00:00:01, size: 01:00:00, output: [auction.date_time, window_start, window_end, auction._row_id] }
         │     └─StreamFilter { predicate: IsNotNull(auction.date_time) }
         │       └─StreamTableScan { table: auction, columns: [auction.date_time, auction._row_id], stream_scan_type: ArrangementBackfill, stream_key: [auction._row_id], pk: [_row_id], dist: UpstreamHashShard(auction._row_id) }
-        └─StreamProject { exprs: [auction.date_time] }
+        └─StreamProject { exprs: [auction.date_time], noop_update_hint: true }
           └─StreamHashAgg { group_key: [auction.date_time], aggs: [count] }
-            └─StreamProject { exprs: [auction.date_time] }
+            └─StreamProject { exprs: [auction.date_time], noop_update_hint: true }
               └─StreamHashAgg { group_key: [auction.date_time], aggs: [count] }
                 └─StreamExchange { dist: HashShard(auction.date_time) }
                   └─StreamShare { id: 3 }
@@ -439,7 +439,7 @@
   stream_error: |-
     Not supported: streaming nested-loop join
     HINT: The non-equal join in the query requires a nested-loop join executor, which could be very expensive to run. Consider rewriting the query to use dynamic filter as a substitute if possible.
-    See also: https://github.com/risingwavelabs/rfcs/blob/main/rfcs/0033-dynamic-filter.md
+    See also: https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/
 - sql: |
     SELECT 1 a, (SELECT regexp_matches('barbeque barbeque', '(bar)(beque)', 'g')) b
   optimized_logical_plan_for_batch: |-
@@ -529,13 +529,13 @@
         └─StreamProject { exprs: [t.x, sum(Unnest($0))] }
           └─StreamHashAgg { group_key: [t.x], aggs: [sum(Unnest($0)), count] }
             └─StreamHashJoin { type: LeftOuter, predicate: t.x IS NOT DISTINCT FROM t.x, output: [t.x, Unnest($0), t.x, projected_row_id] }
-              ├─StreamProject { exprs: [t.x] }
+              ├─StreamProject { exprs: [t.x], noop_update_hint: true }
               │ └─StreamHashAgg { group_key: [t.x], aggs: [count] }
               │   └─StreamExchange { dist: HashShard(t.x) }
               │     └─StreamTableScan { table: t, columns: [t.x, t.k], stream_scan_type: ArrangementBackfill, stream_key: [t.k], pk: [k], dist: UpstreamHashShard(t.k) }
               └─StreamProject { exprs: [t.x, Unnest($0), projected_row_id] }
                 └─StreamProjectSet { select_list: [$0, Unnest($0)] }
-                  └─StreamProject { exprs: [t.x] }
+                  └─StreamProject { exprs: [t.x], noop_update_hint: true }
                     └─StreamHashAgg { group_key: [t.x], aggs: [count] }
                       └─StreamExchange { dist: HashShard(t.x) }
                         └─StreamTableScan { table: t, columns: [t.x, t.k], stream_scan_type: ArrangementBackfill, stream_key: [t.k], pk: [k], dist: UpstreamHashShard(t.k) }
@@ -658,7 +658,7 @@
   stream_error: |-
     Not supported: streaming nested-loop join
     HINT: The non-equal join in the query requires a nested-loop join executor, which could be very expensive to run. Consider rewriting the query to use dynamic filter as a substitute if possible.
-    See also: https://github.com/risingwavelabs/rfcs/blob/main/rfcs/0033-dynamic-filter.md
+    See also: https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/
 - name: test over window subquery 3
   sql: |
     CREATE TABLE integers(i INTEGER);
@@ -715,7 +715,7 @@
   stream_error: |-
     Not supported: streaming nested-loop join
     HINT: The non-equal join in the query requires a nested-loop join executor, which could be very expensive to run. Consider rewriting the query to use dynamic filter as a substitute if possible.
-    See also: https://github.com/risingwavelabs/rfcs/blob/main/rfcs/0033-dynamic-filter.md
+    See also: https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/
 - name: test over window subquery 5
   sql: |
     CREATE TABLE integers(i INTEGER, correlated_col int);
@@ -817,7 +817,7 @@
         └─StreamProject { exprs: [integers.correlated_col, (count(distinct rows.k) + count(distinct rows.v)) as $expr1] }
           └─StreamHashAgg { group_key: [integers.correlated_col], aggs: [count(distinct rows.k), count(distinct rows.v), count] }
             └─StreamHashJoin { type: LeftOuter, predicate: integers.correlated_col IS NOT DISTINCT FROM rows.correlated_col, output: [integers.correlated_col, rows.k, rows.v, rows._row_id] }
-              ├─StreamProject { exprs: [integers.correlated_col] }
+              ├─StreamProject { exprs: [integers.correlated_col], noop_update_hint: true }
               │ └─StreamHashAgg { group_key: [integers.correlated_col], aggs: [count] }
               │   └─StreamExchange { dist: HashShard(integers.correlated_col) }
               │     └─StreamTableScan { table: integers, columns: [integers.correlated_col, integers._row_id], stream_scan_type: ArrangementBackfill, stream_key: [integers._row_id], pk: [_row_id], dist: UpstreamHashShard(integers._row_id) }
diff --git a/src/frontend/planner_test/tests/testdata/output/subquery_expr.yaml b/src/frontend/planner_test/tests/testdata/output/subquery_expr.yaml
index e7c8940b5a8cc..9dc9ead95f17f 100644
--- a/src/frontend/planner_test/tests/testdata/output/subquery_expr.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/subquery_expr.yaml
@@ -316,7 +316,8 @@
     LogicalProject { exprs: [b.b1, ] }
     └─LogicalApply { type: Inner, on: true, correlated_id: 1 }
       ├─LogicalScan { table: b, columns: [b.b1, b._row_id] }
-      └─LogicalValues { rows: [[Repeat(CorrelatedInputRef { index: 0, correlated_id: 1 }, 2:Int32)]], schema: Schema { fields: [:Varchar] } }
+      └─LogicalProject { exprs: [] }
+        └─LogicalValues { rows: [[Repeat(CorrelatedInputRef { index: 0, correlated_id: 1 }, 2:Int32)]], schema: Schema { fields: [:Varchar] } }
   batch_plan: |-
     BatchExchange { order: [], dist: Single }
     └─BatchProject { exprs: [b.b1, Repeat(b.b1, 2:Int32) as $expr1] }
diff --git a/src/frontend/planner_test/tests/testdata/output/subquery_expr_correlated.yaml b/src/frontend/planner_test/tests/testdata/output/subquery_expr_correlated.yaml
index 762f4bea8fe5b..09905835f2009 100644
--- a/src/frontend/planner_test/tests/testdata/output/subquery_expr_correlated.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/subquery_expr_correlated.yaml
@@ -1096,7 +1096,7 @@
           └─StreamProject { exprs: [t.b, count(1:Int32)] }
             └─StreamHashAgg { group_key: [t.b], aggs: [count(1:Int32), count] }
               └─StreamHashJoin { type: LeftOuter, predicate: t.b IS NOT DISTINCT FROM t2.d, output: [t.b, 1:Int32, t2._row_id] }
-                ├─StreamProject { exprs: [t.b] }
+                ├─StreamProject { exprs: [t.b], noop_update_hint: true }
                 │ └─StreamHashAgg { group_key: [t.b], aggs: [count] }
                 │   └─StreamExchange { dist: HashShard(t.b) }
                 │     └─StreamTableScan { table: t, columns: [t.b, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
@@ -1132,3 +1132,993 @@
             └─StreamHashAgg { group_key: [t2.d], aggs: [sum(t2.c), count(t2.c), count] }
               └─StreamExchange { dist: HashShard(t2.d) }
                 └─StreamTableScan { table: t2, columns: [t2.c, t2.d, t2._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t2._row_id], pk: [_row_id], dist: UpstreamHashShard(t2._row_id) }
+- name: improve multi scalar subqueries optimization time. issue 16952. case 1.
+  sql: |
+    create table t1(a int, b int);
+    create table t2(c int primary key, d int);
+    select
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col1,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col2,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col3,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col4,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col5,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col6,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col7,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col8,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col9,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col10,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col11,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col12,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col13,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col14,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col15,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col16,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col17,
+    COALESCE((SELECT b FROM t2 WHERE t1.a = t2.c), 0) col18
+    from t1;
+  batch_plan: |-
+    BatchExchange { order: [], dist: Single }
+    └─BatchProject { exprs: [Coalesce(t1.b, 0:Int32) as $expr1, Coalesce(t1.b, 0:Int32) as $expr2, Coalesce(t1.b, 0:Int32) as $expr3, Coalesce(t1.b, 0:Int32) as $expr4, Coalesce(t1.b, 0:Int32) as $expr5, Coalesce(t1.b, 0:Int32) as $expr6, Coalesce(t1.b, 0:Int32) as $expr7, Coalesce(t1.b, 0:Int32) as $expr8, Coalesce(t1.b, 0:Int32) as $expr9, Coalesce(t1.b, 0:Int32) as $expr10, Coalesce(t1.b, 0:Int32) as $expr11, Coalesce(t1.b, 0:Int32) as $expr12, Coalesce(t1.b, 0:Int32) as $expr13, Coalesce(t1.b, 0:Int32) as $expr14, Coalesce(t1.b, 0:Int32) as $expr15, Coalesce(t1.b, 0:Int32) as $expr16, Coalesce(t1.b, 0:Int32) as $expr17, Coalesce(t1.b, 0:Int32) as $expr18] }
+      └─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │ │ │ │ │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ │ │ │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │ │ │ │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │ │ │ │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ │ │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │ │ │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │ │ │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │ │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │ │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │ │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │   └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+        │     └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+        │       └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │         └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │           └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │             └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+          └─BatchProject { exprs: [t1.a, t1.b, t1.b] }
+            └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: [t1.a, t1.b], lookup table: t2 }
+              └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+                └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+                  └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+                    └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+  stream_plan: |-
+    StreamMaterialize { columns: [col1, col2, col3, col4, col5, col6, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, t1._row_id(hidden), t1.b(hidden), t1.a(hidden), t1.b#1(hidden), t1.b#2(hidden), t1.b#3(hidden), t1.b#4(hidden), t1.b#5(hidden), t1.b#6(hidden), t1.b#7(hidden), t1.b#8(hidden), t1.b#9(hidden), t1.b#10(hidden), t1.b#11(hidden), t1.b#12(hidden), t1.b#13(hidden), t1.b#14(hidden), t1.b#15(hidden), t1.b#16(hidden), t1.b#17(hidden), t1.b#18(hidden)], stream_key: [t1._row_id, t1.b, t1.a, t1.b#1, t1.b#2, t1.b#3, t1.b#4, t1.b#5, t1.b#6, t1.b#7, t1.b#8, t1.b#9, t1.b#10, t1.b#11, t1.b#12, t1.b#13, t1.b#14, t1.b#15, t1.b#16, t1.b#17, t1.b#18], pk_columns: [t1._row_id, t1.b, t1.a, t1.b#1, t1.b#2, t1.b#3, t1.b#4, t1.b#5, t1.b#6, t1.b#7, t1.b#8, t1.b#9, t1.b#10, t1.b#11, t1.b#12, t1.b#13, t1.b#14, t1.b#15, t1.b#16, t1.b#17, t1.b#18], pk_conflict: NoCheck }
+    └─StreamProject { exprs: [Coalesce(t1.b, 0:Int32) as $expr1, Coalesce(t1.b, 0:Int32) as $expr2, Coalesce(t1.b, 0:Int32) as $expr3, Coalesce(t1.b, 0:Int32) as $expr4, Coalesce(t1.b, 0:Int32) as $expr5, Coalesce(t1.b, 0:Int32) as $expr6, Coalesce(t1.b, 0:Int32) as $expr7, Coalesce(t1.b, 0:Int32) as $expr8, Coalesce(t1.b, 0:Int32) as $expr9, Coalesce(t1.b, 0:Int32) as $expr10, Coalesce(t1.b, 0:Int32) as $expr11, Coalesce(t1.b, 0:Int32) as $expr12, Coalesce(t1.b, 0:Int32) as $expr13, Coalesce(t1.b, 0:Int32) as $expr14, Coalesce(t1.b, 0:Int32) as $expr15, Coalesce(t1.b, 0:Int32) as $expr16, Coalesce(t1.b, 0:Int32) as $expr17, Coalesce(t1.b, 0:Int32) as $expr18, t1._row_id, t1.b, t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b] }
+      └─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a, t1.b, t1.a] }
+        ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ │ │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ │ │ │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ │ │ │ │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t1.b, t1._row_id, t1.a] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │ │ │ │ │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │ │ │ │ │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │ │ │ │ │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │ │ │ │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │ │ │ │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │ │ │ │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │ │ │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │ │ │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │ │ │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │ │ │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │ │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │ │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │ │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │ │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │ │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │     └─StreamExchange { dist: HashShard(t2.c) }
+        │ │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+        │   └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │     ├─StreamExchange { dist: HashShard(t1.a) }
+        │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │     └─StreamExchange { dist: HashShard(t2.c) }
+        │       └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        └─StreamProject { exprs: [t1.a, t1.b, t1.b] }
+          └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+            ├─StreamExchange { dist: HashShard(t1.a) }
+            │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+            │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+            │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+            │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+            └─StreamExchange { dist: HashShard(t2.c) }
+              └─StreamTableScan { table: t2, columns: [t2.c], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+- name: improve multi scalar subqueries optimization time. issue 16952. case 2.
+  sql: |
+    create table t1(a int, b int);
+    create table t2(c int primary key, d int);
+    create table t3(e int, f int);
+    create table t4(g int, h int);
+    create table t5(i int, j int);
+    create table t6(k int, l int);
+    select
+    COALESCE((SELECT sum(d) FROM t2 left join t3 on e = a and f = c left join t4 on g = a and h = c left join t5 on i = a and j = c WHERE t1.a = t2.c and t1.a = t2.c and j in (select k from t6 where b = l) ), 0) col1,
+    COALESCE((SELECT sum(d) FROM t2 left join t3 on e = a and f = c left join t4 on g = a and h = c left join t5 on i = a and j = c WHERE t1.a = t2.c and t1.a = t2.c and j in (select k from t6 where b = l) ), 0) col2,
+    COALESCE((SELECT sum(d) FROM t2 left join t3 on e = a and f = c left join t4 on g = a and h = c left join t5 on i = a and j = c WHERE t1.a = t2.c and t1.a = t2.c and j in (select k from t6 where b = l) ), 0) col3,
+    COALESCE((SELECT sum(d) FROM t2 left join t3 on e = a and f = c left join t4 on g = a and h = c left join t5 on i = a and j = c WHERE t1.a = t2.c and t1.a = t2.c and j in (select k from t6 where b = l) ), 0) col4,
+    COALESCE((SELECT sum(d) FROM t2 left join t3 on e = a and f = c left join t4 on g = a and h = c left join t5 on i = a and j = c WHERE t1.a = t2.c and t1.a = t2.c and j in (select k from t6 where b = l) ), 0) col5,
+    COALESCE((SELECT sum(d) FROM t2 left join t3 on e = a and f = c left join t4 on g = a and h = c left join t5 on i = a and j = c WHERE t1.a = t2.c and t1.a = t2.c and j in (select k from t6 where b = l) ), 0) col6
+    from t1;
+  batch_plan: |-
+    BatchExchange { order: [], dist: Single }
+    └─BatchProject { exprs: [Coalesce(sum(t2.d), 0:Int64) as $expr1, Coalesce(sum(t2.d), 0:Int64) as $expr2, Coalesce(sum(t2.d), 0:Int64) as $expr3, Coalesce(sum(t2.d), 0:Int64) as $expr4, Coalesce(sum(t2.d), 0:Int64) as $expr5, Coalesce(sum(t2.d), 0:Int64) as $expr6] }
+      └─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [sum(t2.d), sum(t2.d), sum(t2.d), sum(t2.d), sum(t2.d), sum(t2.d)] }
+        ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, sum(t2.d), sum(t2.d), sum(t2.d), sum(t2.d), sum(t2.d)] }
+        │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, sum(t2.d), sum(t2.d), sum(t2.d), sum(t2.d)] }
+        │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, sum(t2.d), sum(t2.d), sum(t2.d)] }
+        │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, sum(t2.d), sum(t2.d)] }
+        │ │ │ │ ├─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, sum(t2.d)] }
+        │ │ │ │ │ ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │ │ │ │ │ └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [sum(t2.d)] }
+        │ │ │ │ │   └─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d] }
+        │ │ │ │ │     ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │ │ │ │     │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │     │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │     │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │     └─BatchHashJoin { type: LeftSemi, predicate: t5.j = t6.k AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d] }
+        │ │ │ │ │       ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │ │ │ │       │ └─BatchHashJoin { type: LeftOuter, predicate: t2.c = t5.j AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t5.j] }
+        │ │ │ │ │       │   ├─BatchHashJoin { type: LeftOuter, predicate: t2.c = t4.h AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d] }
+        │ │ │ │ │       │   │ ├─BatchHashJoin { type: LeftOuter, predicate: t2.c = t3.f AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d] }
+        │ │ │ │ │       │   │ │ ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │ │ │ │       │   │ │ │ └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: all, lookup table: t2 }
+        │ │ │ │ │       │   │ │ │   └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │ │       │   │ │ │     └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │       │   │ │ │       └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │       │   │ │ │         └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │       │   │ │ └─BatchHashJoin { type: Inner, predicate: t1.a = t3.e AND t1.a = t3.f, output: [t1.a, t1.b, t3.f] }
+        │ │ │ │ │       │   │ │   ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │ │ │ │       │   │ │   │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │       │   │ │   │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │       │   │ │   │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │       │   │ │   └─BatchExchange { order: [], dist: HashShard(t3.e, t3.f) }
+        │ │ │ │ │       │   │ │     └─BatchScan { table: t3, columns: [t3.e, t3.f], distribution: SomeShard }
+        │ │ │ │ │       │   │ └─BatchHashJoin { type: Inner, predicate: t1.a = t4.g AND t1.a = t4.h, output: [t1.a, t1.b, t4.h] }
+        │ │ │ │ │       │   │   ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │ │ │ │       │   │   │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │       │   │   │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │       │   │   │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │       │   │   └─BatchExchange { order: [], dist: HashShard(t4.g, t4.h) }
+        │ │ │ │ │       │   │     └─BatchScan { table: t4, columns: [t4.g, t4.h], distribution: SomeShard }
+        │ │ │ │ │       │   └─BatchHashJoin { type: Inner, predicate: t1.a = t5.i AND t1.a = t5.j, output: [t1.a, t1.b, t5.j] }
+        │ │ │ │ │       │     ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │ │ │ │       │     │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │       │     │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │       │     │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │       │     └─BatchExchange { order: [], dist: HashShard(t5.i, t5.j) }
+        │ │ │ │ │       │       └─BatchScan { table: t5, columns: [t5.i, t5.j], distribution: SomeShard }
+        │ │ │ │ │       └─BatchHashJoin { type: Inner, predicate: t1.b = t6.l, output: [t1.a, t1.b, t6.k] }
+        │ │ │ │ │         ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │ │ │ │         │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │ │         │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │         │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │ │         └─BatchExchange { order: [], dist: HashShard(t6.l) }
+        │ │ │ │ │           └─BatchScan { table: t6, columns: [t6.k, t6.l], distribution: SomeShard }
+        │ │ │ │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [sum(t2.d)] }
+        │ │ │ │   └─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d] }
+        │ │ │ │     ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │ │ │     │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │     │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │     │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │     └─BatchHashJoin { type: LeftSemi, predicate: t5.j = t6.k AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d] }
+        │ │ │ │       ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │ │ │       │ └─BatchHashJoin { type: LeftOuter, predicate: t2.c = t5.j AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t5.j] }
+        │ │ │ │       │   ├─BatchHashJoin { type: LeftOuter, predicate: t2.c = t4.h AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d] }
+        │ │ │ │       │   │ ├─BatchHashJoin { type: LeftOuter, predicate: t2.c = t3.f AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d] }
+        │ │ │ │       │   │ │ ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │ │ │       │   │ │ │ └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: all, lookup table: t2 }
+        │ │ │ │       │   │ │ │   └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │ │       │   │ │ │     └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │       │   │ │ │       └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │       │   │ │ │         └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │       │   │ │ └─BatchHashJoin { type: Inner, predicate: t1.a = t3.e AND t1.a = t3.f, output: [t1.a, t1.b, t3.f] }
+        │ │ │ │       │   │ │   ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │ │ │       │   │ │   │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │       │   │ │   │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │       │   │ │   │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │       │   │ │   └─BatchExchange { order: [], dist: HashShard(t3.e, t3.f) }
+        │ │ │ │       │   │ │     └─BatchScan { table: t3, columns: [t3.e, t3.f], distribution: SomeShard }
+        │ │ │ │       │   │ └─BatchHashJoin { type: Inner, predicate: t1.a = t4.g AND t1.a = t4.h, output: [t1.a, t1.b, t4.h] }
+        │ │ │ │       │   │   ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │ │ │       │   │   │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │       │   │   │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │       │   │   │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │       │   │   └─BatchExchange { order: [], dist: HashShard(t4.g, t4.h) }
+        │ │ │ │       │   │     └─BatchScan { table: t4, columns: [t4.g, t4.h], distribution: SomeShard }
+        │ │ │ │       │   └─BatchHashJoin { type: Inner, predicate: t1.a = t5.i AND t1.a = t5.j, output: [t1.a, t1.b, t5.j] }
+        │ │ │ │       │     ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │ │ │       │     │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │       │     │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │       │     │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │       │     └─BatchExchange { order: [], dist: HashShard(t5.i, t5.j) }
+        │ │ │ │       │       └─BatchScan { table: t5, columns: [t5.i, t5.j], distribution: SomeShard }
+        │ │ │ │       └─BatchHashJoin { type: Inner, predicate: t1.b = t6.l, output: [t1.a, t1.b, t6.k] }
+        │ │ │ │         ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │ │ │         │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │ │         │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │ │         │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │ │         └─BatchExchange { order: [], dist: HashShard(t6.l) }
+        │ │ │ │           └─BatchScan { table: t6, columns: [t6.k, t6.l], distribution: SomeShard }
+        │ │ │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [sum(t2.d)] }
+        │ │ │   └─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d] }
+        │ │ │     ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │ │     │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │     │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │     │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │     └─BatchHashJoin { type: LeftSemi, predicate: t5.j = t6.k AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d] }
+        │ │ │       ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │ │       │ └─BatchHashJoin { type: LeftOuter, predicate: t2.c = t5.j AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t5.j] }
+        │ │ │       │   ├─BatchHashJoin { type: LeftOuter, predicate: t2.c = t4.h AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d] }
+        │ │ │       │   │ ├─BatchHashJoin { type: LeftOuter, predicate: t2.c = t3.f AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d] }
+        │ │ │       │   │ │ ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │ │       │   │ │ │ └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: all, lookup table: t2 }
+        │ │ │       │   │ │ │   └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │ │       │   │ │ │     └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │       │   │ │ │       └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │       │   │ │ │         └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │       │   │ │ └─BatchHashJoin { type: Inner, predicate: t1.a = t3.e AND t1.a = t3.f, output: [t1.a, t1.b, t3.f] }
+        │ │ │       │   │ │   ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │ │       │   │ │   │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │       │   │ │   │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │       │   │ │   │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │       │   │ │   └─BatchExchange { order: [], dist: HashShard(t3.e, t3.f) }
+        │ │ │       │   │ │     └─BatchScan { table: t3, columns: [t3.e, t3.f], distribution: SomeShard }
+        │ │ │       │   │ └─BatchHashJoin { type: Inner, predicate: t1.a = t4.g AND t1.a = t4.h, output: [t1.a, t1.b, t4.h] }
+        │ │ │       │   │   ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │ │       │   │   │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │       │   │   │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │       │   │   │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │       │   │   └─BatchExchange { order: [], dist: HashShard(t4.g, t4.h) }
+        │ │ │       │   │     └─BatchScan { table: t4, columns: [t4.g, t4.h], distribution: SomeShard }
+        │ │ │       │   └─BatchHashJoin { type: Inner, predicate: t1.a = t5.i AND t1.a = t5.j, output: [t1.a, t1.b, t5.j] }
+        │ │ │       │     ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │ │       │     │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │       │     │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │       │     │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │       │     └─BatchExchange { order: [], dist: HashShard(t5.i, t5.j) }
+        │ │ │       │       └─BatchScan { table: t5, columns: [t5.i, t5.j], distribution: SomeShard }
+        │ │ │       └─BatchHashJoin { type: Inner, predicate: t1.b = t6.l, output: [t1.a, t1.b, t6.k] }
+        │ │ │         ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │ │         │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │ │         │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │ │         │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │ │         └─BatchExchange { order: [], dist: HashShard(t6.l) }
+        │ │ │           └─BatchScan { table: t6, columns: [t6.k, t6.l], distribution: SomeShard }
+        │ │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [sum(t2.d)] }
+        │ │   └─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d] }
+        │ │     ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │     │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │     │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │     │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │     └─BatchHashJoin { type: LeftSemi, predicate: t5.j = t6.k AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d] }
+        │ │       ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │       │ └─BatchHashJoin { type: LeftOuter, predicate: t2.c = t5.j AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t5.j] }
+        │ │       │   ├─BatchHashJoin { type: LeftOuter, predicate: t2.c = t4.h AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d] }
+        │ │       │   │ ├─BatchHashJoin { type: LeftOuter, predicate: t2.c = t3.f AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d] }
+        │ │       │   │ │ ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │       │   │ │ │ └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: all, lookup table: t2 }
+        │ │       │   │ │ │   └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │ │       │   │ │ │     └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │       │   │ │ │       └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │       │   │ │ │         └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │       │   │ │ └─BatchHashJoin { type: Inner, predicate: t1.a = t3.e AND t1.a = t3.f, output: [t1.a, t1.b, t3.f] }
+        │ │       │   │ │   ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │       │   │ │   │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │       │   │ │   │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │       │   │ │   │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │       │   │ │   └─BatchExchange { order: [], dist: HashShard(t3.e, t3.f) }
+        │ │       │   │ │     └─BatchScan { table: t3, columns: [t3.e, t3.f], distribution: SomeShard }
+        │ │       │   │ └─BatchHashJoin { type: Inner, predicate: t1.a = t4.g AND t1.a = t4.h, output: [t1.a, t1.b, t4.h] }
+        │ │       │   │   ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │       │   │   │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │       │   │   │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │       │   │   │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │       │   │   └─BatchExchange { order: [], dist: HashShard(t4.g, t4.h) }
+        │ │       │   │     └─BatchScan { table: t4, columns: [t4.g, t4.h], distribution: SomeShard }
+        │ │       │   └─BatchHashJoin { type: Inner, predicate: t1.a = t5.i AND t1.a = t5.j, output: [t1.a, t1.b, t5.j] }
+        │ │       │     ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │ │       │     │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │       │     │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │       │     │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │       │     └─BatchExchange { order: [], dist: HashShard(t5.i, t5.j) }
+        │ │       │       └─BatchScan { table: t5, columns: [t5.i, t5.j], distribution: SomeShard }
+        │ │       └─BatchHashJoin { type: Inner, predicate: t1.b = t6.l, output: [t1.a, t1.b, t6.k] }
+        │ │         ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │ │         │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │ │         │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │ │         │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │ │         └─BatchExchange { order: [], dist: HashShard(t6.l) }
+        │ │           └─BatchScan { table: t6, columns: [t6.k, t6.l], distribution: SomeShard }
+        │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [sum(t2.d)] }
+        │   └─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d] }
+        │     ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │     │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │     │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │     │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │     └─BatchHashJoin { type: LeftSemi, predicate: t5.j = t6.k AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d] }
+        │       ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │       │ └─BatchHashJoin { type: LeftOuter, predicate: t2.c = t5.j AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t5.j] }
+        │       │   ├─BatchHashJoin { type: LeftOuter, predicate: t2.c = t4.h AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d] }
+        │       │   │ ├─BatchHashJoin { type: LeftOuter, predicate: t2.c = t3.f AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d] }
+        │       │   │ │ ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │       │   │ │ │ └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: all, lookup table: t2 }
+        │       │   │ │ │   └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+        │       │   │ │ │     └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │       │   │ │ │       └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │       │   │ │ │         └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │       │   │ │ └─BatchHashJoin { type: Inner, predicate: t1.a = t3.e AND t1.a = t3.f, output: [t1.a, t1.b, t3.f] }
+        │       │   │ │   ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │       │   │ │   │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │       │   │ │   │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │       │   │ │   │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │       │   │ │   └─BatchExchange { order: [], dist: HashShard(t3.e, t3.f) }
+        │       │   │ │     └─BatchScan { table: t3, columns: [t3.e, t3.f], distribution: SomeShard }
+        │       │   │ └─BatchHashJoin { type: Inner, predicate: t1.a = t4.g AND t1.a = t4.h, output: [t1.a, t1.b, t4.h] }
+        │       │   │   ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │       │   │   │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │       │   │   │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │       │   │   │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │       │   │   └─BatchExchange { order: [], dist: HashShard(t4.g, t4.h) }
+        │       │   │     └─BatchScan { table: t4, columns: [t4.g, t4.h], distribution: SomeShard }
+        │       │   └─BatchHashJoin { type: Inner, predicate: t1.a = t5.i AND t1.a = t5.j, output: [t1.a, t1.b, t5.j] }
+        │       │     ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+        │       │     │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │       │     │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │       │     │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │       │     └─BatchExchange { order: [], dist: HashShard(t5.i, t5.j) }
+        │       │       └─BatchScan { table: t5, columns: [t5.i, t5.j], distribution: SomeShard }
+        │       └─BatchHashJoin { type: Inner, predicate: t1.b = t6.l, output: [t1.a, t1.b, t6.k] }
+        │         ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+        │         │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+        │         │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+        │         │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+        │         └─BatchExchange { order: [], dist: HashShard(t6.l) }
+        │           └─BatchScan { table: t6, columns: [t6.k, t6.l], distribution: SomeShard }
+        └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [sum(t2.d)] }
+          └─BatchHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d] }
+            ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+            │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+            │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+            │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+            └─BatchHashJoin { type: LeftSemi, predicate: t5.j = t6.k AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d] }
+              ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+              │ └─BatchHashJoin { type: LeftOuter, predicate: t2.c = t5.j AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t5.j] }
+              │   ├─BatchHashJoin { type: LeftOuter, predicate: t2.c = t4.h AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d] }
+              │   │ ├─BatchHashJoin { type: LeftOuter, predicate: t2.c = t3.f AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d] }
+              │   │ │ ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+              │   │ │ │ └─BatchLookupJoin { type: Inner, predicate: t1.a = t2.c, output: all, lookup table: t2 }
+              │   │ │ │   └─BatchExchange { order: [], dist: UpstreamHashShard(t1.a) }
+              │   │ │ │     └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+              │   │ │ │       └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+              │   │ │ │         └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+              │   │ │ └─BatchHashJoin { type: Inner, predicate: t1.a = t3.e AND t1.a = t3.f, output: [t1.a, t1.b, t3.f] }
+              │   │ │   ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+              │   │ │   │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+              │   │ │   │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+              │   │ │   │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+              │   │ │   └─BatchExchange { order: [], dist: HashShard(t3.e, t3.f) }
+              │   │ │     └─BatchScan { table: t3, columns: [t3.e, t3.f], distribution: SomeShard }
+              │   │ └─BatchHashJoin { type: Inner, predicate: t1.a = t4.g AND t1.a = t4.h, output: [t1.a, t1.b, t4.h] }
+              │   │   ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+              │   │   │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+              │   │   │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+              │   │   │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+              │   │   └─BatchExchange { order: [], dist: HashShard(t4.g, t4.h) }
+              │   │     └─BatchScan { table: t4, columns: [t4.g, t4.h], distribution: SomeShard }
+              │   └─BatchHashJoin { type: Inner, predicate: t1.a = t5.i AND t1.a = t5.j, output: [t1.a, t1.b, t5.j] }
+              │     ├─BatchExchange { order: [], dist: HashShard(t1.a, t1.a) }
+              │     │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+              │     │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+              │     │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+              │     └─BatchExchange { order: [], dist: HashShard(t5.i, t5.j) }
+              │       └─BatchScan { table: t5, columns: [t5.i, t5.j], distribution: SomeShard }
+              └─BatchHashJoin { type: Inner, predicate: t1.b = t6.l, output: [t1.a, t1.b, t6.k] }
+                ├─BatchExchange { order: [], dist: HashShard(t1.b) }
+                │ └─BatchHashAgg { group_key: [t1.a, t1.b], aggs: [] }
+                │   └─BatchExchange { order: [], dist: HashShard(t1.a, t1.b) }
+                │     └─BatchScan { table: t1, columns: [t1.a, t1.b], distribution: SomeShard }
+                └─BatchExchange { order: [], dist: HashShard(t6.l) }
+                  └─BatchScan { table: t6, columns: [t6.k, t6.l], distribution: SomeShard }
+  stream_plan: |-
+    StreamMaterialize { columns: [col1, col2, col3, col4, col5, col6, t1._row_id(hidden), t1.a(hidden), t1.b(hidden)], stream_key: [t1._row_id, t1.a, t1.b], pk_columns: [t1._row_id, t1.a, t1.b], pk_conflict: NoCheck }
+    └─StreamProject { exprs: [Coalesce(sum(t2.d), 0:Int64) as $expr1, Coalesce(sum(t2.d), 0:Int64) as $expr2, Coalesce(sum(t2.d), 0:Int64) as $expr3, Coalesce(sum(t2.d), 0:Int64) as $expr4, Coalesce(sum(t2.d), 0:Int64) as $expr5, Coalesce(sum(t2.d), 0:Int64) as $expr6, t1._row_id, t1.a, t1.b] }
+      └─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [sum(t2.d), sum(t2.d), sum(t2.d), sum(t2.d), sum(t2.d), sum(t2.d), t1._row_id, t1.a, t1.b, t1.a, t1.b] }
+        ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, sum(t2.d), sum(t2.d), sum(t2.d), sum(t2.d), sum(t2.d), t1._row_id, t1.a, t1.b] }
+        │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, sum(t2.d), sum(t2.d), sum(t2.d), sum(t2.d), t1._row_id, t1.a, t1.b] }
+        │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, sum(t2.d), sum(t2.d), sum(t2.d), t1._row_id, t1.a, t1.b] }
+        │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, sum(t2.d), sum(t2.d), t1._row_id, t1.a, t1.b] }
+        │ │ │ │ ├─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, sum(t2.d), t1._row_id, t1.a, t1.b] }
+        │ │ │ │ │ ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │ │ │ │ │ └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, sum(t2.d)] }
+        │ │ │ │ │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [sum(t2.d), count] }
+        │ │ │ │ │     └─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t1.a, t1.b, t3._row_id, t2.c, t4._row_id, t5._row_id, t5.j] }
+        │ │ │ │ │       ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │ │ │ │       │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │       │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │       │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │       │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │       └─StreamHashJoin { type: LeftSemi, predicate: t5.j = t6.k AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t3._row_id, t2.c, t4._row_id, t5._row_id, t5.j] }
+        │ │ │ │ │         ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │ │ │ │         │ └─StreamHashJoin { type: LeftOuter, predicate: t2.c = t5.j AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t5.j, t3._row_id, t2.c, t4._row_id, t1.a, t1.b, t5._row_id] }
+        │ │ │ │ │         │   ├─StreamHashJoin { type: LeftOuter, predicate: t2.c = t4.h AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d, t3._row_id, t1.a, t1.b, t4._row_id] }
+        │ │ │ │ │         │   │ ├─StreamHashJoin { type: LeftOuter, predicate: t2.c = t3.f AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d, t1.a, t1.b, t3._row_id] }
+        │ │ │ │ │         │   │ │ ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │ │ │ │         │   │ │ │ └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │ │         │   │ │ │   ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │ │         │   │ │ │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │         │   │ │ │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │         │   │ │ │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │         │   │ │ │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │         │   │ │ │   └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │ │         │   │ │ │     └─StreamTableScan { table: t2, columns: [t2.c, t2.d], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │ │         │   │ │ └─StreamHashJoin { type: Inner, predicate: t1.a = t3.e AND t1.a = t3.f, output: [t1.a, t1.b, t3.f, t3._row_id] }
+        │ │ │ │ │         │   │ │   ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │ │ │ │         │   │ │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │         │   │ │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │         │   │ │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │         │   │ │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │         │   │ │   └─StreamExchange { dist: HashShard(t3.e, t3.f) }
+        │ │ │ │ │         │   │ │     └─StreamTableScan { table: t3, columns: [t3.e, t3.f, t3._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t3._row_id], pk: [_row_id], dist: UpstreamHashShard(t3._row_id) }
+        │ │ │ │ │         │   │ └─StreamHashJoin { type: Inner, predicate: t1.a = t4.g AND t1.a = t4.h, output: [t1.a, t1.b, t4.h, t4._row_id] }
+        │ │ │ │ │         │   │   ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │ │ │ │         │   │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │         │   │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │         │   │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │         │   │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │         │   │   └─StreamExchange { dist: HashShard(t4.g, t4.h) }
+        │ │ │ │ │         │   │     └─StreamTableScan { table: t4, columns: [t4.g, t4.h, t4._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t4._row_id], pk: [_row_id], dist: UpstreamHashShard(t4._row_id) }
+        │ │ │ │ │         │   └─StreamHashJoin { type: Inner, predicate: t1.a = t5.i AND t1.a = t5.j, output: [t1.a, t1.b, t5.j, t5._row_id] }
+        │ │ │ │ │         │     ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │ │ │ │         │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │         │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │         │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │         │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │         │     └─StreamExchange { dist: HashShard(t5.i, t5.j) }
+        │ │ │ │ │         │       └─StreamTableScan { table: t5, columns: [t5.i, t5.j, t5._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t5._row_id], pk: [_row_id], dist: UpstreamHashShard(t5._row_id) }
+        │ │ │ │ │         └─StreamHashJoin { type: Inner, predicate: t1.b = t6.l, output: [t1.a, t1.b, t6.k, t6._row_id] }
+        │ │ │ │ │           ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │ │ │ │           │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │ │           │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │ │           │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │ │           │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │ │           └─StreamExchange { dist: HashShard(t6.l) }
+        │ │ │ │ │             └─StreamTableScan { table: t6, columns: [t6.k, t6.l, t6._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t6._row_id], pk: [_row_id], dist: UpstreamHashShard(t6._row_id) }
+        │ │ │ │ └─StreamProject { exprs: [t1.a, t1.b, sum(t2.d)] }
+        │ │ │ │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [sum(t2.d), count] }
+        │ │ │ │     └─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t1.a, t1.b, t3._row_id, t2.c, t4._row_id, t5._row_id, t5.j] }
+        │ │ │ │       ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │ │ │       │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │       │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │       │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │       │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │       └─StreamHashJoin { type: LeftSemi, predicate: t5.j = t6.k AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t3._row_id, t2.c, t4._row_id, t5._row_id, t5.j] }
+        │ │ │ │         ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │ │ │         │ └─StreamHashJoin { type: LeftOuter, predicate: t2.c = t5.j AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t5.j, t3._row_id, t2.c, t4._row_id, t1.a, t1.b, t5._row_id] }
+        │ │ │ │         │   ├─StreamHashJoin { type: LeftOuter, predicate: t2.c = t4.h AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d, t3._row_id, t1.a, t1.b, t4._row_id] }
+        │ │ │ │         │   │ ├─StreamHashJoin { type: LeftOuter, predicate: t2.c = t3.f AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d, t1.a, t1.b, t3._row_id] }
+        │ │ │ │         │   │ │ ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │ │ │         │   │ │ │ └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │ │         │   │ │ │   ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │ │         │   │ │ │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │         │   │ │ │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │         │   │ │ │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │         │   │ │ │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │         │   │ │ │   └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │ │         │   │ │ │     └─StreamTableScan { table: t2, columns: [t2.c, t2.d], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │ │         │   │ │ └─StreamHashJoin { type: Inner, predicate: t1.a = t3.e AND t1.a = t3.f, output: [t1.a, t1.b, t3.f, t3._row_id] }
+        │ │ │ │         │   │ │   ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │ │ │         │   │ │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │         │   │ │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │         │   │ │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │         │   │ │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │         │   │ │   └─StreamExchange { dist: HashShard(t3.e, t3.f) }
+        │ │ │ │         │   │ │     └─StreamTableScan { table: t3, columns: [t3.e, t3.f, t3._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t3._row_id], pk: [_row_id], dist: UpstreamHashShard(t3._row_id) }
+        │ │ │ │         │   │ └─StreamHashJoin { type: Inner, predicate: t1.a = t4.g AND t1.a = t4.h, output: [t1.a, t1.b, t4.h, t4._row_id] }
+        │ │ │ │         │   │   ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │ │ │         │   │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │         │   │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │         │   │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │         │   │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │         │   │   └─StreamExchange { dist: HashShard(t4.g, t4.h) }
+        │ │ │ │         │   │     └─StreamTableScan { table: t4, columns: [t4.g, t4.h, t4._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t4._row_id], pk: [_row_id], dist: UpstreamHashShard(t4._row_id) }
+        │ │ │ │         │   └─StreamHashJoin { type: Inner, predicate: t1.a = t5.i AND t1.a = t5.j, output: [t1.a, t1.b, t5.j, t5._row_id] }
+        │ │ │ │         │     ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │ │ │         │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │         │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │         │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │         │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │         │     └─StreamExchange { dist: HashShard(t5.i, t5.j) }
+        │ │ │ │         │       └─StreamTableScan { table: t5, columns: [t5.i, t5.j, t5._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t5._row_id], pk: [_row_id], dist: UpstreamHashShard(t5._row_id) }
+        │ │ │ │         └─StreamHashJoin { type: Inner, predicate: t1.b = t6.l, output: [t1.a, t1.b, t6.k, t6._row_id] }
+        │ │ │ │           ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │ │ │           │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │ │           │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │ │           │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │ │           │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │ │           └─StreamExchange { dist: HashShard(t6.l) }
+        │ │ │ │             └─StreamTableScan { table: t6, columns: [t6.k, t6.l, t6._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t6._row_id], pk: [_row_id], dist: UpstreamHashShard(t6._row_id) }
+        │ │ │ └─StreamProject { exprs: [t1.a, t1.b, sum(t2.d)] }
+        │ │ │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [sum(t2.d), count] }
+        │ │ │     └─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t1.a, t1.b, t3._row_id, t2.c, t4._row_id, t5._row_id, t5.j] }
+        │ │ │       ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │ │       │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │       │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │       │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │       │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │       └─StreamHashJoin { type: LeftSemi, predicate: t5.j = t6.k AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t3._row_id, t2.c, t4._row_id, t5._row_id, t5.j] }
+        │ │ │         ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │ │         │ └─StreamHashJoin { type: LeftOuter, predicate: t2.c = t5.j AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t5.j, t3._row_id, t2.c, t4._row_id, t1.a, t1.b, t5._row_id] }
+        │ │ │         │   ├─StreamHashJoin { type: LeftOuter, predicate: t2.c = t4.h AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d, t3._row_id, t1.a, t1.b, t4._row_id] }
+        │ │ │         │   │ ├─StreamHashJoin { type: LeftOuter, predicate: t2.c = t3.f AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d, t1.a, t1.b, t3._row_id] }
+        │ │ │         │   │ │ ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │ │         │   │ │ │ └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │ │         │   │ │ │   ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │ │         │   │ │ │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │         │   │ │ │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │         │   │ │ │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │         │   │ │ │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │         │   │ │ │   └─StreamExchange { dist: HashShard(t2.c) }
+        │ │ │         │   │ │ │     └─StreamTableScan { table: t2, columns: [t2.c, t2.d], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │ │         │   │ │ └─StreamHashJoin { type: Inner, predicate: t1.a = t3.e AND t1.a = t3.f, output: [t1.a, t1.b, t3.f, t3._row_id] }
+        │ │ │         │   │ │   ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │ │         │   │ │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │         │   │ │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │         │   │ │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │         │   │ │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │         │   │ │   └─StreamExchange { dist: HashShard(t3.e, t3.f) }
+        │ │ │         │   │ │     └─StreamTableScan { table: t3, columns: [t3.e, t3.f, t3._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t3._row_id], pk: [_row_id], dist: UpstreamHashShard(t3._row_id) }
+        │ │ │         │   │ └─StreamHashJoin { type: Inner, predicate: t1.a = t4.g AND t1.a = t4.h, output: [t1.a, t1.b, t4.h, t4._row_id] }
+        │ │ │         │   │   ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │ │         │   │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │         │   │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │         │   │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │         │   │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │         │   │   └─StreamExchange { dist: HashShard(t4.g, t4.h) }
+        │ │ │         │   │     └─StreamTableScan { table: t4, columns: [t4.g, t4.h, t4._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t4._row_id], pk: [_row_id], dist: UpstreamHashShard(t4._row_id) }
+        │ │ │         │   └─StreamHashJoin { type: Inner, predicate: t1.a = t5.i AND t1.a = t5.j, output: [t1.a, t1.b, t5.j, t5._row_id] }
+        │ │ │         │     ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │ │         │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │         │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │         │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │         │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │         │     └─StreamExchange { dist: HashShard(t5.i, t5.j) }
+        │ │ │         │       └─StreamTableScan { table: t5, columns: [t5.i, t5.j, t5._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t5._row_id], pk: [_row_id], dist: UpstreamHashShard(t5._row_id) }
+        │ │ │         └─StreamHashJoin { type: Inner, predicate: t1.b = t6.l, output: [t1.a, t1.b, t6.k, t6._row_id] }
+        │ │ │           ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │ │           │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │ │           │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │ │           │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │ │           │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │ │           └─StreamExchange { dist: HashShard(t6.l) }
+        │ │ │             └─StreamTableScan { table: t6, columns: [t6.k, t6.l, t6._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t6._row_id], pk: [_row_id], dist: UpstreamHashShard(t6._row_id) }
+        │ │ └─StreamProject { exprs: [t1.a, t1.b, sum(t2.d)] }
+        │ │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [sum(t2.d), count] }
+        │ │     └─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t1.a, t1.b, t3._row_id, t2.c, t4._row_id, t5._row_id, t5.j] }
+        │ │       ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │       │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │       │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │       │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │       │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │       └─StreamHashJoin { type: LeftSemi, predicate: t5.j = t6.k AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t3._row_id, t2.c, t4._row_id, t5._row_id, t5.j] }
+        │ │         ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │         │ └─StreamHashJoin { type: LeftOuter, predicate: t2.c = t5.j AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t5.j, t3._row_id, t2.c, t4._row_id, t1.a, t1.b, t5._row_id] }
+        │ │         │   ├─StreamHashJoin { type: LeftOuter, predicate: t2.c = t4.h AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d, t3._row_id, t1.a, t1.b, t4._row_id] }
+        │ │         │   │ ├─StreamHashJoin { type: LeftOuter, predicate: t2.c = t3.f AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d, t1.a, t1.b, t3._row_id] }
+        │ │         │   │ │ ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │         │   │ │ │ └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │ │         │   │ │ │   ├─StreamExchange { dist: HashShard(t1.a) }
+        │ │         │   │ │ │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │         │   │ │ │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │         │   │ │ │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │         │   │ │ │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │         │   │ │ │   └─StreamExchange { dist: HashShard(t2.c) }
+        │ │         │   │ │ │     └─StreamTableScan { table: t2, columns: [t2.c, t2.d], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │ │         │   │ │ └─StreamHashJoin { type: Inner, predicate: t1.a = t3.e AND t1.a = t3.f, output: [t1.a, t1.b, t3.f, t3._row_id] }
+        │ │         │   │ │   ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │         │   │ │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │         │   │ │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │         │   │ │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │         │   │ │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │         │   │ │   └─StreamExchange { dist: HashShard(t3.e, t3.f) }
+        │ │         │   │ │     └─StreamTableScan { table: t3, columns: [t3.e, t3.f, t3._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t3._row_id], pk: [_row_id], dist: UpstreamHashShard(t3._row_id) }
+        │ │         │   │ └─StreamHashJoin { type: Inner, predicate: t1.a = t4.g AND t1.a = t4.h, output: [t1.a, t1.b, t4.h, t4._row_id] }
+        │ │         │   │   ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │         │   │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │         │   │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │         │   │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │         │   │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │         │   │   └─StreamExchange { dist: HashShard(t4.g, t4.h) }
+        │ │         │   │     └─StreamTableScan { table: t4, columns: [t4.g, t4.h, t4._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t4._row_id], pk: [_row_id], dist: UpstreamHashShard(t4._row_id) }
+        │ │         │   └─StreamHashJoin { type: Inner, predicate: t1.a = t5.i AND t1.a = t5.j, output: [t1.a, t1.b, t5.j, t5._row_id] }
+        │ │         │     ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │ │         │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │         │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │         │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │         │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │         │     └─StreamExchange { dist: HashShard(t5.i, t5.j) }
+        │ │         │       └─StreamTableScan { table: t5, columns: [t5.i, t5.j, t5._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t5._row_id], pk: [_row_id], dist: UpstreamHashShard(t5._row_id) }
+        │ │         └─StreamHashJoin { type: Inner, predicate: t1.b = t6.l, output: [t1.a, t1.b, t6.k, t6._row_id] }
+        │ │           ├─StreamExchange { dist: HashShard(t1.b) }
+        │ │           │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │ │           │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │ │           │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │ │           │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │ │           └─StreamExchange { dist: HashShard(t6.l) }
+        │ │             └─StreamTableScan { table: t6, columns: [t6.k, t6.l, t6._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t6._row_id], pk: [_row_id], dist: UpstreamHashShard(t6._row_id) }
+        │ └─StreamProject { exprs: [t1.a, t1.b, sum(t2.d)] }
+        │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [sum(t2.d), count] }
+        │     └─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t1.a, t1.b, t3._row_id, t2.c, t4._row_id, t5._row_id, t5.j] }
+        │       ├─StreamExchange { dist: HashShard(t1.b) }
+        │       │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │       │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │       │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │       │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │       └─StreamHashJoin { type: LeftSemi, predicate: t5.j = t6.k AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t3._row_id, t2.c, t4._row_id, t5._row_id, t5.j] }
+        │         ├─StreamExchange { dist: HashShard(t1.b) }
+        │         │ └─StreamHashJoin { type: LeftOuter, predicate: t2.c = t5.j AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t5.j, t3._row_id, t2.c, t4._row_id, t1.a, t1.b, t5._row_id] }
+        │         │   ├─StreamHashJoin { type: LeftOuter, predicate: t2.c = t4.h AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d, t3._row_id, t1.a, t1.b, t4._row_id] }
+        │         │   │ ├─StreamHashJoin { type: LeftOuter, predicate: t2.c = t3.f AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d, t1.a, t1.b, t3._row_id] }
+        │         │   │ │ ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │         │   │ │ │ └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+        │         │   │ │ │   ├─StreamExchange { dist: HashShard(t1.a) }
+        │         │   │ │ │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │         │   │ │ │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │         │   │ │ │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │         │   │ │ │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │         │   │ │ │   └─StreamExchange { dist: HashShard(t2.c) }
+        │         │   │ │ │     └─StreamTableScan { table: t2, columns: [t2.c, t2.d], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+        │         │   │ │ └─StreamHashJoin { type: Inner, predicate: t1.a = t3.e AND t1.a = t3.f, output: [t1.a, t1.b, t3.f, t3._row_id] }
+        │         │   │ │   ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │         │   │ │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │         │   │ │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │         │   │ │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │         │   │ │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │         │   │ │   └─StreamExchange { dist: HashShard(t3.e, t3.f) }
+        │         │   │ │     └─StreamTableScan { table: t3, columns: [t3.e, t3.f, t3._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t3._row_id], pk: [_row_id], dist: UpstreamHashShard(t3._row_id) }
+        │         │   │ └─StreamHashJoin { type: Inner, predicate: t1.a = t4.g AND t1.a = t4.h, output: [t1.a, t1.b, t4.h, t4._row_id] }
+        │         │   │   ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │         │   │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │         │   │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │         │   │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │         │   │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │         │   │   └─StreamExchange { dist: HashShard(t4.g, t4.h) }
+        │         │   │     └─StreamTableScan { table: t4, columns: [t4.g, t4.h, t4._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t4._row_id], pk: [_row_id], dist: UpstreamHashShard(t4._row_id) }
+        │         │   └─StreamHashJoin { type: Inner, predicate: t1.a = t5.i AND t1.a = t5.j, output: [t1.a, t1.b, t5.j, t5._row_id] }
+        │         │     ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+        │         │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │         │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │         │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │         │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │         │     └─StreamExchange { dist: HashShard(t5.i, t5.j) }
+        │         │       └─StreamTableScan { table: t5, columns: [t5.i, t5.j, t5._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t5._row_id], pk: [_row_id], dist: UpstreamHashShard(t5._row_id) }
+        │         └─StreamHashJoin { type: Inner, predicate: t1.b = t6.l, output: [t1.a, t1.b, t6.k, t6._row_id] }
+        │           ├─StreamExchange { dist: HashShard(t1.b) }
+        │           │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+        │           │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+        │           │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+        │           │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        │           └─StreamExchange { dist: HashShard(t6.l) }
+        │             └─StreamTableScan { table: t6, columns: [t6.k, t6.l, t6._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t6._row_id], pk: [_row_id], dist: UpstreamHashShard(t6._row_id) }
+        └─StreamProject { exprs: [t1.a, t1.b, sum(t2.d)] }
+          └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [sum(t2.d), count] }
+            └─StreamHashJoin { type: LeftOuter, predicate: t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t1.a, t1.b, t3._row_id, t2.c, t4._row_id, t5._row_id, t5.j] }
+              ├─StreamExchange { dist: HashShard(t1.b) }
+              │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+              │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+              │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+              │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+              └─StreamHashJoin { type: LeftSemi, predicate: t5.j = t6.k AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t3._row_id, t2.c, t4._row_id, t5._row_id, t5.j] }
+                ├─StreamExchange { dist: HashShard(t1.b) }
+                │ └─StreamHashJoin { type: LeftOuter, predicate: t2.c = t5.j AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.d, t5.j, t3._row_id, t2.c, t4._row_id, t1.a, t1.b, t5._row_id] }
+                │   ├─StreamHashJoin { type: LeftOuter, predicate: t2.c = t4.h AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d, t3._row_id, t1.a, t1.b, t4._row_id] }
+                │   │ ├─StreamHashJoin { type: LeftOuter, predicate: t2.c = t3.f AND t1.a IS NOT DISTINCT FROM t1.a AND t1.b IS NOT DISTINCT FROM t1.b, output: [t1.a, t1.b, t2.c, t2.d, t1.a, t1.b, t3._row_id] }
+                │   │ │ ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+                │   │ │ │ └─StreamHashJoin { type: Inner, predicate: t1.a = t2.c, output: all }
+                │   │ │ │   ├─StreamExchange { dist: HashShard(t1.a) }
+                │   │ │ │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+                │   │ │ │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+                │   │ │ │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+                │   │ │ │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+                │   │ │ │   └─StreamExchange { dist: HashShard(t2.c) }
+                │   │ │ │     └─StreamTableScan { table: t2, columns: [t2.c, t2.d], stream_scan_type: ArrangementBackfill, stream_key: [t2.c], pk: [c], dist: UpstreamHashShard(t2.c) }
+                │   │ │ └─StreamHashJoin { type: Inner, predicate: t1.a = t3.e AND t1.a = t3.f, output: [t1.a, t1.b, t3.f, t3._row_id] }
+                │   │ │   ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+                │   │ │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+                │   │ │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+                │   │ │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+                │   │ │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+                │   │ │   └─StreamExchange { dist: HashShard(t3.e, t3.f) }
+                │   │ │     └─StreamTableScan { table: t3, columns: [t3.e, t3.f, t3._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t3._row_id], pk: [_row_id], dist: UpstreamHashShard(t3._row_id) }
+                │   │ └─StreamHashJoin { type: Inner, predicate: t1.a = t4.g AND t1.a = t4.h, output: [t1.a, t1.b, t4.h, t4._row_id] }
+                │   │   ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+                │   │   │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+                │   │   │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+                │   │   │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+                │   │   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+                │   │   └─StreamExchange { dist: HashShard(t4.g, t4.h) }
+                │   │     └─StreamTableScan { table: t4, columns: [t4.g, t4.h, t4._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t4._row_id], pk: [_row_id], dist: UpstreamHashShard(t4._row_id) }
+                │   └─StreamHashJoin { type: Inner, predicate: t1.a = t5.i AND t1.a = t5.j, output: [t1.a, t1.b, t5.j, t5._row_id] }
+                │     ├─StreamExchange { dist: HashShard(t1.a, t1.a) }
+                │     │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+                │     │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+                │     │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+                │     │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+                │     └─StreamExchange { dist: HashShard(t5.i, t5.j) }
+                │       └─StreamTableScan { table: t5, columns: [t5.i, t5.j, t5._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t5._row_id], pk: [_row_id], dist: UpstreamHashShard(t5._row_id) }
+                └─StreamHashJoin { type: Inner, predicate: t1.b = t6.l, output: [t1.a, t1.b, t6.k, t6._row_id] }
+                  ├─StreamExchange { dist: HashShard(t1.b) }
+                  │ └─StreamProject { exprs: [t1.a, t1.b], noop_update_hint: true }
+                  │   └─StreamHashAgg { group_key: [t1.a, t1.b], aggs: [count] }
+                  │     └─StreamExchange { dist: HashShard(t1.a, t1.b) }
+                  │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+                  └─StreamExchange { dist: HashShard(t6.l) }
+                    └─StreamTableScan { table: t6, columns: [t6.k, t6.l, t6._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t6._row_id], pk: [_row_id], dist: UpstreamHashShard(t6._row_id) }
diff --git a/src/frontend/planner_test/tests/testdata/output/temporal_join.yaml b/src/frontend/planner_test/tests/testdata/output/temporal_join.yaml
index aa8887e98bef6..5cdfdf6cf45ea 100644
--- a/src/frontend/planner_test/tests/testdata/output/temporal_join.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/temporal_join.yaml
@@ -111,7 +111,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [k, x1, x2, a1, b1, stream._row_id(hidden), version2.k(hidden)], stream_key: [stream._row_id, k], pk_columns: [stream._row_id, k], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.k, stream._row_id) }
-      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.k = version2.k, output: [stream.k, version1.x1, version2.x2, stream.a1, stream.b1, stream._row_id, version2.k] }
+      └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.k = version2.k, output: [stream.k, version1.x1, version2.x2, stream.a1, stream.b1, stream._row_id, version2.k] }
         ├─StreamExchange { dist: HashShard(stream.k) }
         │ └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.k = version1.k, output: [stream.k, stream.a1, stream.b1, version1.x1, stream._row_id, version1.k] }
         │   ├─StreamExchange { dist: HashShard(stream.k) }
@@ -133,7 +133,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, x1, id2, x2, a1, b1, stream._row_id(hidden), version2.id2(hidden)], stream_key: [stream._row_id, id1, id2], pk_columns: [stream._row_id, id1, id2], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.id1, stream.id2, stream._row_id) }
-      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.id2 = version2.id2, output: [stream.id1, version1.x1, stream.id2, version2.x2, stream.a1, stream.b1, stream._row_id, version2.id2] }
+      └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.id2 = version2.id2, output: [stream.id1, version1.x1, stream.id2, version2.x2, stream.a1, stream.b1, stream._row_id, version2.id2] }
         ├─StreamExchange { dist: HashShard(stream.id2) }
         │ └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.id1 = version1.id1, output: [stream.id1, stream.id2, stream.a1, stream.b1, version1.x1, stream._row_id, version1.id1] }
         │   ├─StreamExchange { dist: HashShard(stream.id1) }
@@ -155,7 +155,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, x1, id2, x2, a1, b1, stream._row_id(hidden), version2.id2(hidden)], stream_key: [stream._row_id, id1, id2], pk_columns: [stream._row_id, id1, id2], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.id1, stream.id2, stream._row_id) }
-      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.id2 = version2.id2, output: [stream.id1, version1.x1, stream.id2, version2.x2, stream.a1, stream.b1, stream._row_id, version2.id2] }
+      └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.id2 = version2.id2, output: [stream.id1, version1.x1, stream.id2, version2.x2, stream.a1, stream.b1, stream._row_id, version2.id2] }
         ├─StreamExchange { dist: HashShard(stream.id2) }
         │ └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.id1 = version1.id1, output: [stream.id1, stream.id2, stream.a1, stream.b1, version1.x1, stream._row_id, version1.id1] }
         │   ├─StreamExchange { dist: HashShard(stream.id1) }
diff --git a/src/frontend/planner_test/tests/testdata/output/union.yaml b/src/frontend/planner_test/tests/testdata/output/union.yaml
index c9591f2dee2d4..ffd31dec73da5 100644
--- a/src/frontend/planner_test/tests/testdata/output/union.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/union.yaml
@@ -78,7 +78,7 @@
             └─BatchScan { table: t2, columns: [t2.a, t2.b, t2.c], distribution: SomeShard }
   stream_plan: |-
     StreamMaterialize { columns: [a, b, c], stream_key: [a, b, c], pk_columns: [a, b, c], pk_conflict: NoCheck }
-    └─StreamProject { exprs: [t1.a, t1.b, t1.c] }
+    └─StreamProject { exprs: [t1.a, t1.b, t1.c], noop_update_hint: true }
       └─StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] }
         └─StreamExchange { dist: HashShard(t1.a, t1.b, t1.c) }
           └─StreamUnion { all: true }
@@ -92,7 +92,7 @@
     Fragment 0
     StreamMaterialize { columns: [a, b, c], stream_key: [a, b, c], pk_columns: [a, b, c], pk_conflict: NoCheck }
     ├── tables: [ Materialize: 4294967294 ]
-    └── StreamProject { exprs: [t1.a, t1.b, t1.c] }
+    └── StreamProject { exprs: [t1.a, t1.b, t1.c], noop_update_hint: true }
         └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } { tables: [ HashAggState: 0 ] }
             └── StreamExchange Hash([0, 1, 2]) from 1
 
@@ -155,7 +155,7 @@
             └─BatchScan { table: t2, columns: [t2.a, t2.b, t2.c], distribution: UpstreamHashShard(t2.a) }
   stream_plan: |-
     StreamMaterialize { columns: [a, b, c], stream_key: [a, b, c], pk_columns: [a, b, c], pk_conflict: NoCheck }
-    └─StreamProject { exprs: [t1.a, t1.b, t1.c] }
+    └─StreamProject { exprs: [t1.a, t1.b, t1.c], noop_update_hint: true }
       └─StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] }
         └─StreamExchange { dist: HashShard(t1.a, t1.b, t1.c) }
           └─StreamUnion { all: true }
@@ -169,7 +169,7 @@
     Fragment 0
     StreamMaterialize { columns: [a, b, c], stream_key: [a, b, c], pk_columns: [a, b, c], pk_conflict: NoCheck }
     ├── tables: [ Materialize: 4294967294 ]
-    └── StreamProject { exprs: [t1.a, t1.b, t1.c] }
+    └── StreamProject { exprs: [t1.a, t1.b, t1.c], noop_update_hint: true }
         └── StreamHashAgg { group_key: [t1.a, t1.b, t1.c], aggs: [count] } { tables: [ HashAggState: 0 ] }
             └── StreamExchange Hash([0, 1, 2]) from 1
 
diff --git a/src/frontend/planner_test/tests/testdata/output/with_ordinality.yaml b/src/frontend/planner_test/tests/testdata/output/with_ordinality.yaml
index 8216619881a5a..1b811a4871993 100644
--- a/src/frontend/planner_test/tests/testdata/output/with_ordinality.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/with_ordinality.yaml
@@ -30,7 +30,7 @@
         ├─StreamExchange { dist: HashShard(t.arr) }
         │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
         └─StreamProjectSet { select_list: [$0, Unnest($0)] }
-          └─StreamProject { exprs: [t.arr] }
+          └─StreamProject { exprs: [t.arr], noop_update_hint: true }
             └─StreamHashAgg { group_key: [t.arr], aggs: [count] }
               └─StreamExchange { dist: HashShard(t.arr) }
                 └─StreamTableScan { table: t, columns: [t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
@@ -54,7 +54,7 @@
         ├─StreamExchange { dist: HashShard(t.arr) }
         │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
         └─StreamProjectSet { select_list: [$0, Unnest($0)] }
-          └─StreamProject { exprs: [t.arr] }
+          └─StreamProject { exprs: [t.arr], noop_update_hint: true }
             └─StreamHashAgg { group_key: [t.arr], aggs: [count] }
               └─StreamExchange { dist: HashShard(t.arr) }
                 └─StreamTableScan { table: t, columns: [t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
@@ -78,7 +78,7 @@
         ├─StreamExchange { dist: HashShard(t.arr) }
         │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
         └─StreamProjectSet { select_list: [$0, Unnest($0)] }
-          └─StreamProject { exprs: [t.arr] }
+          └─StreamProject { exprs: [t.arr], noop_update_hint: true }
             └─StreamHashAgg { group_key: [t.arr], aggs: [count] }
               └─StreamExchange { dist: HashShard(t.arr) }
                 └─StreamTableScan { table: t, columns: [t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
@@ -102,7 +102,7 @@
         ├─StreamExchange { dist: HashShard(t.arr) }
         │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
         └─StreamProjectSet { select_list: [$0, Unnest($0)] }
-          └─StreamProject { exprs: [t.arr] }
+          └─StreamProject { exprs: [t.arr], noop_update_hint: true }
             └─StreamHashAgg { group_key: [t.arr], aggs: [count] }
               └─StreamExchange { dist: HashShard(t.arr) }
                 └─StreamTableScan { table: t, columns: [t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
@@ -127,7 +127,7 @@
         ├─StreamExchange { dist: HashShard(t.arr) }
         │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
         └─StreamProjectSet { select_list: [$0, Unnest($0)] }
-          └─StreamProject { exprs: [t.arr] }
+          └─StreamProject { exprs: [t.arr], noop_update_hint: true }
             └─StreamHashAgg { group_key: [t.arr], aggs: [count] }
               └─StreamExchange { dist: HashShard(t.arr) }
                 └─StreamTableScan { table: t, columns: [t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
@@ -151,46 +151,34 @@
         │       └─BatchScan { table: t, columns: [t.arr], distribution: SomeShard }
         └─BatchProjectSet { select_list: [$0, Unnest($0)] }
           └─BatchHashAgg { group_key: [t.arr], aggs: [] }
-            └─BatchHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.arr] }
-              ├─BatchExchange { order: [], dist: HashShard(t.arr) }
-              │ └─BatchScan { table: t, columns: [t.arr], distribution: SomeShard }
-              └─BatchProjectSet { select_list: [$0, Unnest($0)] }
-                └─BatchHashAgg { group_key: [t.arr], aggs: [] }
-                  └─BatchExchange { order: [], dist: HashShard(t.arr) }
-                    └─BatchScan { table: t, columns: [t.arr], distribution: SomeShard }
+            └─BatchExchange { order: [], dist: HashShard(t.arr) }
+              └─BatchScan { table: t, columns: [t.arr], distribution: SomeShard }
   stream_plan: |-
     StreamMaterialize { columns: [x, arr, unnest, ordinality, arr_2, ordinality_2, t._row_id(hidden), projected_row_id(hidden), projected_row_id#1(hidden)], stream_key: [t._row_id, projected_row_id, arr, projected_row_id#1], pk_columns: [t._row_id, projected_row_id, arr, projected_row_id#1], pk_conflict: NoCheck }
-    └─StreamProject { exprs: [t.x, t.arr, Unnest($0), $expr1, Unnest($0), (projected_row_id + 1:Int64) as $expr2, t._row_id, projected_row_id, projected_row_id] }
-      └─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, Unnest($0), $expr1, projected_row_id, t.arr, Unnest($0), t._row_id, projected_row_id] }
-        ├─StreamShare { id: 8 }
-        │ └─StreamProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1, t._row_id, projected_row_id] }
-        │   └─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, projected_row_id, t.arr, Unnest($0), t._row_id] }
-        │     ├─StreamExchange { dist: HashShard(t.arr) }
-        │     │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
-        │     └─StreamProjectSet { select_list: [$0, Unnest($0)] }
-        │       └─StreamProject { exprs: [t.arr] }
-        │         └─StreamHashAgg { group_key: [t.arr], aggs: [count] }
-        │           └─StreamExchange { dist: HashShard(t.arr) }
-        │             └─StreamTableScan { table: t, columns: [t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+    └─StreamProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1, Unnest($0), (projected_row_id + 1:Int64) as $expr2, t._row_id, projected_row_id, projected_row_id] }
+      └─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, projected_row_id, t.arr, Unnest($0), projected_row_id, t.arr, Unnest($0), t._row_id] }
+        ├─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, projected_row_id, t.arr, Unnest($0), t._row_id] }
+        │ ├─StreamExchange { dist: HashShard(t.arr) }
+        │ │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+        │ └─StreamProjectSet { select_list: [$0, Unnest($0)] }
+        │   └─StreamProject { exprs: [t.arr], noop_update_hint: true }
+        │     └─StreamHashAgg { group_key: [t.arr], aggs: [count] }
+        │       └─StreamExchange { dist: HashShard(t.arr) }
+        │         └─StreamTableScan { table: t, columns: [t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
         └─StreamProjectSet { select_list: [$0, Unnest($0)] }
-          └─StreamProject { exprs: [t.arr] }
+          └─StreamProject { exprs: [t.arr], noop_update_hint: true }
             └─StreamHashAgg { group_key: [t.arr], aggs: [count] }
-              └─StreamShare { id: 8 }
-                └─StreamProject { exprs: [t.x, t.arr, Unnest($0), (projected_row_id + 1:Int64) as $expr1, t._row_id, projected_row_id] }
-                  └─StreamHashJoin { type: Inner, predicate: t.arr IS NOT DISTINCT FROM t.arr, output: [t.x, t.arr, projected_row_id, t.arr, Unnest($0), t._row_id] }
-                    ├─StreamExchange { dist: HashShard(t.arr) }
-                    │ └─StreamTableScan { table: t, columns: [t.x, t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
-                    └─StreamProjectSet { select_list: [$0, Unnest($0)] }
-                      └─StreamProject { exprs: [t.arr] }
-                        └─StreamHashAgg { group_key: [t.arr], aggs: [count] }
-                          └─StreamExchange { dist: HashShard(t.arr) }
-                            └─StreamTableScan { table: t, columns: [t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
+              └─StreamExchange { dist: HashShard(t.arr) }
+                └─StreamTableScan { table: t, columns: [t.arr, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
 - sql: |
     select * from abs(1) WITH ORDINALITY;
-  batch_plan: 'BatchValues { rows: [[1:Int32, 1:Int64]] }'
+  batch_plan: |-
+    BatchProject { exprs: [, 1:Int64] }
+    └─BatchValues { rows: [[1:Int32]] }
   stream_plan: |-
     StreamMaterialize { columns: [abs, ordinality, _row_id(hidden)], stream_key: [_row_id], pk_columns: [_row_id], pk_conflict: NoCheck }
-    └─StreamValues { rows: [[1:Int32, 1:Int64, 0:Int64]] }
+    └─StreamProject { exprs: [, 1:Int64, _row_id] }
+      └─StreamValues { rows: [[1:Int32, 0:Int64]] }
 - sql: |
     create table t(x int , arr int[]);
     select * from t, abs(x) WITH ORDINALITY;
@@ -210,4 +198,4 @@
   stream_error: |-
     Not supported: streaming nested-loop join
     HINT: The non-equal join in the query requires a nested-loop join executor, which could be very expensive to run. Consider rewriting the query to use dynamic filter as a substitute if possible.
-    See also: https://github.com/risingwavelabs/rfcs/blob/main/rfcs/0033-dynamic-filter.md
+    See also: https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/
diff --git a/src/frontend/src/binder/bind_context.rs b/src/frontend/src/binder/bind_context.rs
index 7a52de7decec4..2876ec4aa9447 100644
--- a/src/frontend/src/binder/bind_context.rs
+++ b/src/frontend/src/binder/bind_context.rs
@@ -81,12 +81,12 @@ pub struct LateralBindContext {
 /// WITH RECURSIVE t(n) AS (
 /// # -------------^ => Init
 ///     VALUES (1)
-/// # ----------^ => BaseResolved (after binding the base term)
 ///   UNION ALL
-///     SELECT n+1 FROM t WHERE n < 100
-/// # ------------------^ => Bound (we know exactly what the entire cte looks like)
+///     SELECT n + 1 FROM t WHERE n < 100
+/// # --------------------^ => BaseResolved (after binding the base term, this relation will be bound to `Relation::BackCteRef`)
 /// )
 /// SELECT sum(n) FROM t;
+/// # -----------------^ => Bound (we know exactly what the entire `RecursiveUnion` looks like, and this relation will be bound to `Relation::Share`)
 /// ```
 #[derive(Default, Debug, Clone)]
 pub enum BindingCteState {
@@ -94,7 +94,7 @@ pub enum BindingCteState {
     #[default]
     Init,
     /// We know the schema form after the base term resolved.
-    BaseResolved { schema: Schema },
+    BaseResolved { base: BoundSetExpr },
     /// We get the whole bound result of the (recursive) CTE.
     Bound {
         query: Either<BoundQuery, RecursiveUnion>,
@@ -107,6 +107,7 @@ pub enum BindingCteState {
 pub struct RecursiveUnion {
     /// currently this *must* be true,
     /// otherwise binding will fail.
+    #[allow(dead_code)]
     pub all: bool,
     /// lhs part of the `UNION ALL` operator
     pub base: Box<BoundSetExpr>,
diff --git a/src/frontend/src/binder/expr/function.rs b/src/frontend/src/binder/expr/function.rs
index 2134e08fe8c66..2a4812c2e1d1f 100644
--- a/src/frontend/src/binder/expr/function.rs
+++ b/src/frontend/src/binder/expr/function.rs
@@ -23,25 +23,26 @@ use risingwave_common::array::ListValue;
 use risingwave_common::catalog::{INFORMATION_SCHEMA_SCHEMA_NAME, PG_CATALOG_SCHEMA_NAME};
 use risingwave_common::session_config::USER_NAME_WILD_CARD;
 use risingwave_common::types::{data_types, DataType, ScalarImpl, Timestamptz};
-use risingwave_common::{bail_not_implemented, current_cluster_version, no_function};
+use risingwave_common::{bail_not_implemented, current_cluster_version, must_match, no_function};
 use risingwave_expr::aggregate::{agg_kinds, AggKind};
 use risingwave_expr::window_function::{
     Frame, FrameBound, FrameBounds, FrameExclusion, RangeFrameBounds, RangeFrameOffset,
-    RowsFrameBounds, WindowFuncKind,
+    RowsFrameBounds, SessionFrameBounds, SessionFrameGap, WindowFuncKind,
 };
 use risingwave_sqlparser::ast::{
-    self, Function, FunctionArg, FunctionArgExpr, Ident, WindowFrameBound, WindowFrameExclusion,
-    WindowFrameUnits, WindowSpec,
+    self, Function, FunctionArg, FunctionArgExpr, Ident, WindowFrameBound, WindowFrameBounds,
+    WindowFrameExclusion, WindowFrameUnits, WindowSpec,
 };
 use risingwave_sqlparser::parser::ParserError;
 use thiserror_ext::AsReport;
 
 use crate::binder::bind_context::Clause;
 use crate::binder::{Binder, UdfContext};
+use crate::catalog::function_catalog::FunctionCatalog;
 use crate::error::{ErrorCode, Result, RwError};
 use crate::expr::{
-    AggCall, Expr, ExprImpl, ExprType, FunctionCall, FunctionCallWithLambda, Literal, Now, OrderBy,
-    TableFunction, TableFunctionType, UserDefinedFunction, WindowFunction,
+    AggCall, CastContext, Expr, ExprImpl, ExprType, FunctionCall, FunctionCallWithLambda, Literal,
+    Now, OrderBy, TableFunction, TableFunctionType, UserDefinedFunction, WindowFunction,
 };
 use crate::utils::Condition;
 
@@ -95,21 +96,6 @@ impl Binder {
             _ => bail_not_implemented!(issue = 112, "qualified function {}", f.name),
         };
 
-        // agg calls
-        if f.over.is_none()
-            && let Ok(kind) = function_name.parse()
-        {
-            return self.bind_agg(f, kind);
-        }
-
-        if f.distinct || !f.order_by.is_empty() || f.filter.is_some() {
-            return Err(ErrorCode::InvalidInputSyntax(format!(
-                    "DISTINCT, ORDER BY or FILTER is only allowed in aggregation functions, but `{}` is not an aggregation function", function_name
-                )
-                )
-                .into());
-        }
-
         // FIXME: This is a hack to support [Bytebase queries](https://github.com/TennyZhuang/bytebase/blob/4a26f7c62b80e86e58ad2f77063138dc2f420623/backend/plugin/db/pg/sync.go#L549).
         // Bytebase widely used the pattern like `obj_description(format('%s.%s',
         // quote_ident(idx.schemaname), quote_ident(idx.indexname))::regclass) AS comment` to
@@ -119,47 +105,18 @@ impl Binder {
         if function_name == "obj_description" || function_name == "col_description" {
             return Ok(ExprImpl::literal_varchar("".to_string()));
         }
-
         if function_name == "array_transform" {
             // For type inference, we need to bind the array type first.
             return self.bind_array_transform(f);
         }
 
-        // Used later in sql udf expression evaluation
-        let args = f.args.clone();
-
-        let mut inputs = f
+        let mut inputs: Vec<_> = f
             .args
-            .into_iter()
-            .map(|arg| self.bind_function_arg(arg))
+            .iter()
+            .map(|arg| self.bind_function_arg(arg.clone()))
             .flatten_ok()
             .try_collect()?;
 
-        // window function
-        let window_func_kind = WindowFuncKind::from_str(function_name.as_str());
-        if let Ok(kind) = window_func_kind {
-            if let Some(window_spec) = f.over {
-                return self.bind_window_function(kind, inputs, window_spec);
-            }
-            return Err(ErrorCode::InvalidInputSyntax(format!(
-                "Window function `{}` must have OVER clause",
-                function_name
-            ))
-            .into());
-        } else if f.over.is_some() {
-            bail_not_implemented!(
-                issue = 8961,
-                "Unrecognized window function: {}",
-                function_name
-            );
-        }
-
-        // table function
-        if let Ok(function_type) = TableFunctionType::from_str(function_name.as_str()) {
-            self.ensure_table_function_allowed()?;
-            return Ok(TableFunction::new(function_type, inputs)?.into());
-        }
-
         // user defined function
         // TODO: resolve schema name https://github.com/risingwavelabs/risingwave/issues/12422
         if let Ok(schema) = self.first_valid_schema()
@@ -200,7 +157,7 @@ impl Binder {
 
                 // The actual inline logic for sql udf
                 // Note that we will always create new udf context for each sql udf
-                let Ok(context) = UdfContext::create_udf_context(&args, &Arc::clone(func)) else {
+                let Ok(context) = UdfContext::create_udf_context(&f.args, &Arc::clone(func)) else {
                     return Err(ErrorCode::InvalidInputSyntax(
                         "failed to create the `udf_context`, please recheck your function definition and syntax".to_string()
                     )
@@ -268,11 +225,53 @@ impl Binder {
                         self.ensure_table_function_allowed()?;
                         return Ok(TableFunction::new_user_defined(func.clone(), inputs).into());
                     }
-                    Aggregate => todo!("support UDAF"),
+                    Aggregate => {
+                        return self.bind_agg(f, AggKind::UserDefined, Some(func.clone()));
+                    }
                 }
             }
         }
 
+        // agg calls
+        if f.over.is_none()
+            && let Ok(kind) = function_name.parse()
+        {
+            return self.bind_agg(f, kind, None);
+        }
+
+        if f.distinct || !f.order_by.is_empty() || f.filter.is_some() {
+            return Err(ErrorCode::InvalidInputSyntax(format!(
+                    "DISTINCT, ORDER BY or FILTER is only allowed in aggregation functions, but `{}` is not an aggregation function", function_name
+                )
+                )
+                .into());
+        }
+
+        // window function
+        let window_func_kind = WindowFuncKind::from_str(function_name.as_str());
+        if let Ok(kind) = window_func_kind {
+            if let Some(window_spec) = f.over {
+                return self.bind_window_function(kind, inputs, window_spec);
+            }
+            return Err(ErrorCode::InvalidInputSyntax(format!(
+                "Window function `{}` must have OVER clause",
+                function_name
+            ))
+            .into());
+        } else if f.over.is_some() {
+            bail_not_implemented!(
+                issue = 8961,
+                "Unrecognized window function: {}",
+                function_name
+            );
+        }
+
+        // table function
+        if let Ok(function_type) = TableFunctionType::from_str(function_name.as_str()) {
+            self.ensure_table_function_allowed()?;
+            return Ok(TableFunction::new(function_type, inputs)?.into());
+        }
+
         self.bind_builtin_scalar_function(function_name.as_str(), inputs, f.variadic)
     }
 
@@ -351,7 +350,12 @@ impl Binder {
         Ok(body)
     }
 
-    pub(super) fn bind_agg(&mut self, f: Function, kind: AggKind) -> Result<ExprImpl> {
+    pub(super) fn bind_agg(
+        &mut self,
+        f: Function,
+        kind: AggKind,
+        user_defined: Option<Arc<FunctionCatalog>>,
+    ) -> Result<ExprImpl> {
         self.ensure_aggregate_allowed()?;
 
         let distinct = f.distinct;
@@ -385,14 +389,26 @@ impl Binder {
             None => Condition::true_cond(),
         };
 
-        Ok(ExprImpl::AggCall(Box::new(AggCall::new(
-            kind,
-            args,
-            distinct,
-            order_by,
-            filter,
-            direct_args,
-        )?)))
+        if let Some(user_defined) = user_defined {
+            Ok(AggCall::new_user_defined(
+                args,
+                distinct,
+                order_by,
+                filter,
+                direct_args,
+                user_defined,
+            )?
+            .into())
+        } else {
+            Ok(ExprImpl::AggCall(Box::new(AggCall::new(
+                kind,
+                args,
+                distinct,
+                order_by,
+                filter,
+                direct_args,
+            )?)))
+        }
     }
 
     fn bind_ordered_set_agg(
@@ -614,34 +630,35 @@ impl Binder {
             };
             let bounds = match frame.units {
                 WindowFrameUnits::Rows => {
-                    let (start, end) =
-                        self.bind_window_frame_usize_bounds(frame.start_bound, frame.end_bound)?;
+                    let (start, end) = must_match!(frame.bounds, WindowFrameBounds::Bounds { start, end } => (start, end));
+                    let (start, end) = self.bind_window_frame_usize_bounds(start, end)?;
                     FrameBounds::Rows(RowsFrameBounds { start, end })
                 }
-                WindowFrameUnits::Range => {
+                unit @ (WindowFrameUnits::Range | WindowFrameUnits::Session) => {
                     let order_by_expr = order_by
                         .sort_exprs
                         .iter()
-                        // for `RANGE` frame, there should be exactly one `ORDER BY` column
+                        // for `RANGE | SESSION` frame, there should be exactly one `ORDER BY` column
                         .exactly_one()
                         .map_err(|_| {
-                            ErrorCode::InvalidInputSyntax(
-                                "there should be exactly one ordering column for `RANGE` frame"
-                                    .to_string(),
-                            )
+                            ErrorCode::InvalidInputSyntax(format!(
+                                "there should be exactly one ordering column for `{}` frame",
+                                unit
+                            ))
                         })?;
                     let order_data_type = order_by_expr.expr.return_type();
                     let order_type = order_by_expr.order_type;
 
                     let offset_data_type = match &order_data_type {
-                        // for numeric ordering columns, `offset` should be the same type
+                        // for numeric ordering columns, `offset`/`gap` should be the same type
                         // NOTE: actually in PG it can be a larger type, but we don't support this here
                         t @ data_types::range_frame_numeric!() => t.clone(),
-                        // for datetime ordering columns, `offset` should be interval
+                        // for datetime ordering columns, `offset`/`gap` should be interval
                         t @ data_types::range_frame_datetime!() => {
                             if matches!(t, DataType::Date | DataType::Time) {
                                 bail_not_implemented!(
-                                    "`RANGE` frame with offset of type `{}` is not implemented yet, please manually cast the `ORDER BY` column to `timestamp`",
+                                    "`{}` frame with offset of type `{}` is not implemented yet, please manually cast the `ORDER BY` column to `timestamp`",
+                                    unit,
                                     t
                                 );
                             }
@@ -651,8 +668,8 @@ impl Binder {
                         t => {
                             return Err(ErrorCode::NotSupported(
                                 format!(
-                                    "`RANGE` frame with offset of type `{}` is not supported",
-                                    t
+                                    "`{}` frame with offset of type `{}` is not supported",
+                                    unit, t
                                 ),
                                 "Please re-consider the `ORDER BY` column".to_string(),
                             )
@@ -660,18 +677,31 @@ impl Binder {
                         }
                     };
 
-                    let (start, end) = self.bind_window_frame_scalar_impl_bounds(
-                        frame.start_bound,
-                        frame.end_bound,
-                        &offset_data_type,
-                    )?;
-                    FrameBounds::Range(RangeFrameBounds {
-                        order_data_type,
-                        order_type,
-                        offset_data_type,
-                        start: start.map(RangeFrameOffset::new),
-                        end: end.map(RangeFrameOffset::new),
-                    })
+                    if unit == WindowFrameUnits::Range {
+                        let (start, end) = must_match!(frame.bounds, WindowFrameBounds::Bounds { start, end } => (start, end));
+                        let (start, end) = self.bind_window_frame_scalar_impl_bounds(
+                            start,
+                            end,
+                            &offset_data_type,
+                        )?;
+                        FrameBounds::Range(RangeFrameBounds {
+                            order_data_type,
+                            order_type,
+                            offset_data_type,
+                            start: start.map(RangeFrameOffset::new),
+                            end: end.map(RangeFrameOffset::new),
+                        })
+                    } else {
+                        let gap = must_match!(frame.bounds, WindowFrameBounds::Gap(gap) => gap);
+                        let gap_value =
+                            self.bind_window_frame_bound_offset(*gap, offset_data_type.clone())?;
+                        FrameBounds::Session(SessionFrameBounds {
+                            order_data_type,
+                            order_type,
+                            gap_data_type: offset_data_type,
+                            gap: SessionFrameGap::new(gap_value),
+                        })
+                    }
                 }
                 WindowFrameUnits::Groups => {
                     bail_not_implemented!(
@@ -766,13 +796,13 @@ impl Binder {
         let mut offset = self.bind_expr(offset)?;
         if !offset.is_const() {
             return Err(ErrorCode::InvalidInputSyntax(
-                "offset in window frame bounds must be constant".to_string(),
+                "offset/gap in window frame bounds must be constant".to_string(),
             )
             .into());
         }
         if offset.cast_implicit_mut(cast_to.clone()).is_err() {
             return Err(ErrorCode::InvalidInputSyntax(format!(
-                "offset in window frame bounds must be castable to {}",
+                "offset/gap in window frame bounds must be castable to {}",
                 cast_to
             ))
             .into());
@@ -780,7 +810,7 @@ impl Binder {
         let offset = offset.fold_const()?;
         let Some(offset) = offset else {
             return Err(ErrorCode::InvalidInputSyntax(
-                "offset in window frame bounds must not be NULL".to_string(),
+                "offset/gap in window frame bounds must not be NULL".to_string(),
             )
             .into());
         };
@@ -1012,6 +1042,22 @@ impl Binder {
                 ("to_ascii", raw_call(ExprType::ToAscii)),
                 ("to_hex", raw_call(ExprType::ToHex)),
                 ("quote_ident", raw_call(ExprType::QuoteIdent)),
+                ("quote_literal", guard_by_len(1, raw(|_binder, mut inputs| {
+                    if inputs[0].return_type() != DataType::Varchar {
+                        // Support `quote_literal(any)` by converting it to `quote_literal(any::text)`
+                        // Ref. https://github.com/postgres/postgres/blob/REL_16_1/src/include/catalog/pg_proc.dat#L4641
+                        FunctionCall::cast_mut(&mut inputs[0], DataType::Varchar, CastContext::Explicit)?;
+                    }
+                    Ok(FunctionCall::new_unchecked(ExprType::QuoteLiteral, inputs, DataType::Varchar).into())
+                }))),
+                ("quote_nullable", guard_by_len(1, raw(|_binder, mut inputs| {
+                    if inputs[0].return_type() != DataType::Varchar {
+                        // Support `quote_nullable(any)` by converting it to `quote_nullable(any::text)`
+                        // Ref. https://github.com/postgres/postgres/blob/REL_16_1/src/include/catalog/pg_proc.dat#L4650
+                        FunctionCall::cast_mut(&mut inputs[0], DataType::Varchar, CastContext::Explicit)?;
+                    }
+                    Ok(FunctionCall::new_unchecked(ExprType::QuoteNullable, inputs, DataType::Varchar).into())
+                }))),
                 ("string_to_array", raw_call(ExprType::StringToArray)),
                 ("encode", raw_call(ExprType::Encode)),
                 ("decode", raw_call(ExprType::Decode)),
@@ -1128,6 +1174,7 @@ impl Binder {
                 ("jsonb_path_exists", raw_call(ExprType::JsonbPathExists)),
                 ("jsonb_path_query_array", raw_call(ExprType::JsonbPathQueryArray)),
                 ("jsonb_path_query_first", raw_call(ExprType::JsonbPathQueryFirst)),
+                ("jsonb_set", raw_call(ExprType::JsonbSet)),
                 // Functions that return a constant value
                 ("pi", pi()),
                 // greatest and least
@@ -1303,6 +1350,51 @@ impl Binder {
                 ("pg_get_partkeydef", raw_literal(ExprImpl::literal_null(DataType::Varchar))),
                 ("pg_encoding_to_char", raw_literal(ExprImpl::literal_varchar("UTF8".into()))),
                 ("has_database_privilege", raw_literal(ExprImpl::literal_bool(true))),
+                ("has_table_privilege", raw(|binder, mut inputs|{
+                    if inputs.len() == 2 {
+                        inputs.insert(0, ExprImpl::literal_varchar(binder.auth_context.user_name.clone()));
+                    }
+                    if inputs.len() == 3 {
+                        if inputs[1].return_type() == DataType::Varchar {
+                            inputs[1].cast_to_regclass_mut()?;
+                        }
+                        Ok(FunctionCall::new(ExprType::HasTablePrivilege, inputs)?.into())
+                    } else {
+                        Err(ErrorCode::ExprError(
+                            "Too many/few arguments for pg_catalog.has_table_privilege()".into(),
+                        )
+                        .into())
+                    }
+                })),
+                ("has_any_column_privilege", raw(|binder, mut inputs|{
+                    if inputs.len() == 2 {
+                        inputs.insert(0, ExprImpl::literal_varchar(binder.auth_context.user_name.clone()));
+                    }
+                    if inputs.len() == 3 {
+                        if inputs[1].return_type() == DataType::Varchar {
+                            inputs[1].cast_to_regclass_mut()?;
+                        }
+                        Ok(FunctionCall::new(ExprType::HasAnyColumnPrivilege, inputs)?.into())
+                    } else {
+                        Err(ErrorCode::ExprError(
+                            "Too many/few arguments for pg_catalog.has_any_column_privilege()".into(),
+                        )
+                        .into())
+                    }
+                })),
+                ("has_schema_privilege", raw(|binder, mut inputs|{
+                    if inputs.len() == 2 {
+                        inputs.insert(0, ExprImpl::literal_varchar(binder.auth_context.user_name.clone()));
+                    }
+                    if inputs.len() == 3 {
+                        Ok(FunctionCall::new(ExprType::HasSchemaPrivilege, inputs)?.into())
+                    } else {
+                        Err(ErrorCode::ExprError(
+                            "Too many/few arguments for pg_catalog.has_schema_privilege()".into(),
+                        )
+                        .into())
+                    }
+                })),
                 ("pg_stat_get_numscans", raw_literal(ExprImpl::literal_bigint(0))),
                 ("pg_backend_pid", raw(|binder, _inputs| {
                     // FIXME: the session id is not global unique in multi-frontend env.
diff --git a/src/frontend/src/binder/expr/value.rs b/src/frontend/src/binder/expr/value.rs
index 0b6eace27083f..a0758a15d4440 100644
--- a/src/frontend/src/binder/expr/value.rs
+++ b/src/frontend/src/binder/expr/value.rs
@@ -221,13 +221,12 @@ impl Binder {
 #[cfg(test)]
 mod tests {
     use risingwave_common::types::test_utils::IntervalTestExt;
-    use risingwave_common::types::DataType;
     use risingwave_expr::expr::build_from_prost;
     use risingwave_sqlparser::ast::Value::Number;
 
     use super::*;
     use crate::binder::test_utils::mock_binder;
-    use crate::expr::{Expr, ExprImpl, ExprType, FunctionCall};
+    use crate::expr::Expr;
 
     #[tokio::test]
     async fn test_bind_value() {
diff --git a/src/frontend/src/binder/insert.rs b/src/frontend/src/binder/insert.rs
index 1cc719d066420..e0b5ce422e75a 100644
--- a/src/frontend/src/binder/insert.rs
+++ b/src/frontend/src/binder/insert.rs
@@ -175,14 +175,14 @@ impl Binder {
         // is given and it is NOT equivalent to assignment cast over potential implicit cast inside.
         // For example, the following is valid:
         //
-        // ```
+        // ```sql
         //   create table t (v1 time);
         //   insert into t values (timestamp '2020-01-01 01:02:03'), (time '03:04:05');
         // ```
         //
         // But the followings are not:
         //
-        // ```
+        // ```sql
         //   values (timestamp '2020-01-01 01:02:03'), (time '03:04:05');
         //   insert into t values (timestamp '2020-01-01 01:02:03'), (time '03:04:05') limit 1;
         // ```
diff --git a/src/frontend/src/binder/mod.rs b/src/frontend/src/binder/mod.rs
index 3648f53d50271..cf12417334612 100644
--- a/src/frontend/src/binder/mod.rs
+++ b/src/frontend/src/binder/mod.rs
@@ -49,8 +49,9 @@ pub use insert::BoundInsert;
 use pgwire::pg_server::{Session, SessionId};
 pub use query::BoundQuery;
 pub use relation::{
-    BoundBaseTable, BoundJoin, BoundShare, BoundSource, BoundSystemTable, BoundWatermark,
-    BoundWindowTableFunction, Relation, ResolveQualifiedNameError, WindowTableFunctionKind,
+    BoundBackCteRef, BoundBaseTable, BoundJoin, BoundShare, BoundSource, BoundSystemTable,
+    BoundWatermark, BoundWindowTableFunction, Relation, ResolveQualifiedNameError,
+    WindowTableFunctionKind,
 };
 pub use select::{BoundDistinct, BoundSelect};
 pub use set_expr::*;
@@ -260,17 +261,17 @@ impl UdfContext {
 /// following the rules:
 /// 1. At the beginning, it contains the user specified parameters type.
 /// 2. When the binder encounters a parameter, it will record it as unknown(call `record_new_param`)
-/// if it didn't exist in `ParameterTypes`.
+///    if it didn't exist in `ParameterTypes`.
 /// 3. When the binder encounters a cast on parameter, if it's a unknown type, the cast function
-/// will record the target type as infer type for that parameter(call `record_infer_type`). If the
-/// parameter has been inferred, the cast function will act as a normal cast.
+///    will record the target type as infer type for that parameter(call `record_infer_type`). If the
+///    parameter has been inferred, the cast function will act as a normal cast.
 /// 4. After bind finished:
 ///     (a) parameter not in `ParameterTypes` means that the user didn't specify it and it didn't
-/// occur in the query. `export` will return error if there is a kind of
-/// parameter. This rule is compatible with PostgreSQL
+///    occur in the query. `export` will return error if there is a kind of
+///    parameter. This rule is compatible with PostgreSQL
 ///     (b) parameter is None means that it's a unknown type. The user didn't specify it
-/// and we can't infer it in the query. We will treat it as VARCHAR type finally. This rule is
-/// compatible with PostgreSQL.
+///    and we can't infer it in the query. We will treat it as VARCHAR type finally. This rule is
+///    compatible with PostgreSQL.
 ///     (c) parameter is Some means that it's a known type.
 #[derive(Clone, Debug)]
 pub struct ParameterTypes(Arc<RwLock<HashMap<u64, Option<DataType>>>>);
@@ -649,6 +650,41 @@ mod tests {
                                                             BackCteRef(
                                                                 BoundBackCteRef {
                                                                     share_id: 0,
+                                                                    base: Select(
+                                                                        BoundSelect {
+                                                                            distinct: All,
+                                                                            select_items: [
+                                                                                Literal(
+                                                                                    Literal {
+                                                                                        data: Some(
+                                                                                            Int32(
+                                                                                                1,
+                                                                                            ),
+                                                                                        ),
+                                                                                        data_type: Some(
+                                                                                            Int32,
+                                                                                        ),
+                                                                                    },
+                                                                                ),
+                                                                            ],
+                                                                            aliases: [
+                                                                                Some(
+                                                                                    "a",
+                                                                                ),
+                                                                            ],
+                                                                            from: None,
+                                                                            where_clause: None,
+                                                                            group_by: GroupKey(
+                                                                                [],
+                                                                            ),
+                                                                            having: None,
+                                                                            schema: Schema {
+                                                                                fields: [
+                                                                                    a:Int32,
+                                                                                ],
+                                                                            },
+                                                                        },
+                                                                    ),
                                                                 },
                                                             ),
                                                         ),
diff --git a/src/frontend/src/binder/query.rs b/src/frontend/src/binder/query.rs
index 39de526662976..2aa1d066953c7 100644
--- a/src/frontend/src/binder/query.rs
+++ b/src/frontend/src/binder/query.rs
@@ -284,56 +284,28 @@ impl Binder {
 
     fn bind_with(&mut self, with: With) -> Result<()> {
         for cte_table in with.cte_tables {
+            // note that the new `share_id` for the rcte is generated here
             let share_id = self.next_share_id();
             let Cte { alias, query, .. } = cte_table;
             let table_name = alias.name.real_value();
 
             if with.recursive {
-                let Query {
+                let (
+                    SetExpr::SetOperation {
+                        op: SetOperator::Union,
+                        all,
+                        left,
+                        right,
+                    },
                     with,
-                    body,
-                    order_by,
-                    limit,
-                    offset,
-                    fetch,
-                } = query;
-
-                /// the input clause should not be supported.
-                fn should_be_empty<T>(v: Option<T>, clause: &str) -> Result<()> {
-                    if v.is_some() {
-                        return Err(ErrorCode::BindError(format!(
-                            "`{clause}` is not supported in recursive CTE"
-                        ))
-                        .into());
-                    }
-                    Ok(())
-                }
-
-                should_be_empty(order_by.first(), "ORDER BY")?;
-                should_be_empty(limit, "LIMIT")?;
-                should_be_empty(offset, "OFFSET")?;
-                should_be_empty(fetch, "FETCH")?;
-
-                let SetExpr::SetOperation {
-                    op: SetOperator::Union,
-                    all,
-                    left,
-                    right,
-                } = body
+                ) = Self::validate_rcte(query)?
                 else {
                     return Err(ErrorCode::BindError(
-                        "`UNION` is required in recursive CTE".to_string(),
+                        "expect `SetOperation` as the return type of validation".into(),
                     )
                     .into());
                 };
 
-                if !all {
-                    return Err(ErrorCode::BindError(
-                        "only `UNION ALL` is supported in recursive CTE now".to_string(),
-                    )
-                    .into());
-                }
-
                 let entry = self
                     .context
                     .cte_to_relation
@@ -346,47 +318,7 @@ impl Binder {
                     .get()
                     .clone();
 
-                self.push_context();
-                if let Some(with) = with {
-                    self.bind_with(with)?;
-                }
-
-                // We assume `left` is the base term, otherwise the implementation may be very hard.
-                // The behavior is the same as PostgreSQL's.
-                // reference: <https://www.postgresql.org/docs/16/sql-select.html#:~:text=the%20recursive%20self%2Dreference%20must%20appear%20on%20the%20right%2Dhand%20side%20of%20the%20UNION>
-                let mut base = self.bind_set_expr(*left)?;
-
-                entry.borrow_mut().state = BindingCteState::BaseResolved {
-                    schema: base.schema().clone(),
-                };
-
-                // Reset context for right side, but keep `cte_to_relation`.
-                let new_context = std::mem::take(&mut self.context);
-                self.context
-                    .cte_to_relation
-                    .clone_from(&new_context.cte_to_relation);
-                // bind the rest of the recursive cte
-                let mut recursive = self.bind_set_expr(*right)?;
-                // Reset context for the set operation.
-                self.context = Default::default();
-                self.context.cte_to_relation = new_context.cte_to_relation;
-
-                Self::align_schema(&mut base, &mut recursive, SetOperator::Union)?;
-                let schema = base.schema().clone();
-
-                let recursive_union = RecursiveUnion {
-                    all,
-                    base: Box::new(base),
-                    recursive: Box::new(recursive),
-                    schema,
-                };
-
-                entry.borrow_mut().state = BindingCteState::Bound {
-                    query: either::Either::Right(recursive_union),
-                };
-                // TODO: This does not execute during early return by `?`
-                // We shall extract it similar to `bind_query` and `bind_query_inner`.
-                self.pop_context()?;
+                self.bind_rcte(with, entry, *left, *right, all)?;
             } else {
                 let bound_query = self.bind_query(query)?;
                 self.context.cte_to_relation.insert(
@@ -403,6 +335,124 @@ impl Binder {
         }
         Ok(())
     }
+
+    /// syntactically validate the recursive cte ast with the current support features in rw.
+    fn validate_rcte(query: Query) -> Result<(SetExpr, Option<With>)> {
+        let Query {
+            with,
+            body,
+            order_by,
+            limit,
+            offset,
+            fetch,
+        } = query;
+
+        /// the input clause should not be supported.
+        fn should_be_empty<T>(v: Option<T>, clause: &str) -> Result<()> {
+            if v.is_some() {
+                return Err(ErrorCode::BindError(format!(
+                    "`{clause}` is not supported in recursive CTE"
+                ))
+                .into());
+            }
+            Ok(())
+        }
+
+        should_be_empty(order_by.first(), "ORDER BY")?;
+        should_be_empty(limit, "LIMIT")?;
+        should_be_empty(offset, "OFFSET")?;
+        should_be_empty(fetch, "FETCH")?;
+
+        let SetExpr::SetOperation {
+            op: SetOperator::Union,
+            all,
+            left,
+            right,
+        } = body
+        else {
+            return Err(
+                ErrorCode::BindError("`UNION` is required in recursive CTE".to_string()).into(),
+            );
+        };
+
+        if !all {
+            return Err(ErrorCode::BindError(
+                "only `UNION ALL` is supported in recursive CTE now".to_string(),
+            )
+            .into());
+        }
+
+        Ok((
+            SetExpr::SetOperation {
+                op: SetOperator::Union,
+                all,
+                left,
+                right,
+            },
+            with,
+        ))
+    }
+
+    fn bind_rcte(
+        &mut self,
+        with: Option<With>,
+        entry: Rc<RefCell<BindingCte>>,
+        left: SetExpr,
+        right: SetExpr,
+        all: bool,
+    ) -> Result<()> {
+        self.push_context();
+        let result = self.bind_rcte_inner(with, entry, left, right, all);
+        self.pop_context()?;
+        result
+    }
+
+    fn bind_rcte_inner(
+        &mut self,
+        with: Option<With>,
+        entry: Rc<RefCell<BindingCte>>,
+        left: SetExpr,
+        right: SetExpr,
+        all: bool,
+    ) -> Result<()> {
+        if let Some(with) = with {
+            self.bind_with(with)?;
+        }
+
+        // We assume `left` is the base term, otherwise the implementation may be very hard.
+        // The behavior is the same as PostgreSQL's.
+        // reference: <https://www.postgresql.org/docs/16/sql-select.html#:~:text=the%20recursive%20self%2Dreference%20must%20appear%20on%20the%20right%2Dhand%20side%20of%20the%20UNION>
+        let mut base = self.bind_set_expr(left)?;
+
+        entry.borrow_mut().state = BindingCteState::BaseResolved { base: base.clone() };
+
+        // Reset context for right side, but keep `cte_to_relation`.
+        let new_context = std::mem::take(&mut self.context);
+        self.context
+            .cte_to_relation
+            .clone_from(&new_context.cte_to_relation);
+        // bind the rest of the recursive cte
+        let mut recursive = self.bind_set_expr(right)?;
+        // Reset context for the set operation.
+        self.context = Default::default();
+        self.context.cte_to_relation = new_context.cte_to_relation;
+
+        Self::align_schema(&mut base, &mut recursive, SetOperator::Union)?;
+        let schema = base.schema().clone();
+
+        let recursive_union = RecursiveUnion {
+            all,
+            base: Box::new(base),
+            recursive: Box::new(recursive),
+            schema,
+        };
+
+        entry.borrow_mut().state = BindingCteState::Bound {
+            query: either::Either::Right(recursive_union),
+        };
+
+        Ok(())
+    }
 }
 
 // TODO: Make clause a const generic param after <https://github.com/rust-lang/rust/issues/95174>.
diff --git a/src/frontend/src/binder/relation/cte_ref.rs b/src/frontend/src/binder/relation/cte_ref.rs
index 86c40cc096dd1..87e87fb1823ae 100644
--- a/src/frontend/src/binder/relation/cte_ref.rs
+++ b/src/frontend/src/binder/relation/cte_ref.rs
@@ -13,14 +13,14 @@
 // limitations under the License.
 
 use crate::binder::statement::RewriteExprsRecursive;
-use crate::binder::ShareId;
+use crate::binder::{BoundSetExpr, ShareId};
 
 /// A CTE reference, currently only used in the back reference of recursive CTE.
 /// For the non-recursive one, see [`BoundShare`](super::BoundShare).
 #[derive(Debug, Clone)]
 pub struct BoundBackCteRef {
-    #[expect(dead_code)]
     pub(crate) share_id: ShareId,
+    pub(crate) base: BoundSetExpr,
 }
 
 impl RewriteExprsRecursive for BoundBackCteRef {
diff --git a/src/frontend/src/binder/relation/join.rs b/src/frontend/src/binder/relation/join.rs
index c2f820aff7744..d13b683be08b0 100644
--- a/src/frontend/src/binder/relation/join.rs
+++ b/src/frontend/src/binder/relation/join.rs
@@ -164,10 +164,10 @@ impl Binder {
                     JoinConstraint::Using(cols) => {
                         // sanity check
                         for col in &cols {
-                            if old_context.indices_of.get(&col.real_value()).is_none() {
+                            if !old_context.indices_of.contains_key(&col.real_value()) {
                                 return Err(ErrorCode::ItemNotFound(format!("column \"{}\" specified in USING clause does not exist in left table", col.real_value())).into());
                             }
-                            if self.context.indices_of.get(&col.real_value()).is_none() {
+                            if !self.context.indices_of.contains_key(&col.real_value()) {
                                 return Err(ErrorCode::ItemNotFound(format!("column \"{}\" specified in USING clause does not exist in right table", col.real_value())).into());
                             }
                         }
diff --git a/src/frontend/src/binder/relation/mod.rs b/src/frontend/src/binder/relation/mod.rs
index b7c288551191c..4de9c3cfaa9aa 100644
--- a/src/frontend/src/binder/relation/mod.rs
+++ b/src/frontend/src/binder/relation/mod.rs
@@ -26,7 +26,6 @@ use risingwave_sqlparser::ast::{
 use thiserror::Error;
 use thiserror_ext::AsReport;
 
-use self::cte_ref::BoundBackCteRef;
 use super::bind_context::ColumnBinding;
 use super::statement::RewriteExprsRecursive;
 use crate::binder::bind_context::{BindingCte, BindingCteState};
@@ -43,6 +42,7 @@ mod table_or_source;
 mod watermark;
 mod window_table_function;
 
+pub use cte_ref::BoundBackCteRef;
 pub use join::BoundJoin;
 pub use share::BoundShare;
 pub use subquery::BoundSubquery;
@@ -374,13 +374,13 @@ impl Binder {
                 BindingCteState::Init => {
                     Err(ErrorCode::BindError("Base term of recursive CTE not found, consider writing it to left side of the `UNION ALL` operator".to_string()).into())
                 }
-                BindingCteState::BaseResolved { schema } => {
+                BindingCteState::BaseResolved { base } => {
                     self.bind_table_to_context(
-                        schema.fields.iter().map(|f| (false, f.clone())),
+                        base.schema().fields.iter().map(|f| (false, f.clone())),
                         table_name.clone(),
                         Some(original_alias),
                     )?;
-                    Ok(Relation::BackCteRef(Box::new(BoundBackCteRef { share_id })))
+                    Ok(Relation::BackCteRef(Box::new(BoundBackCteRef { share_id, base })))
                 }
                 BindingCteState::Bound { query } => {
                     let schema = match &query {
diff --git a/src/frontend/src/binder/relation/table_function.rs b/src/frontend/src/binder/relation/table_function.rs
index 9189e176e0d55..1a609f87670f3 100644
--- a/src/frontend/src/binder/relation/table_function.rs
+++ b/src/frontend/src/binder/relation/table_function.rs
@@ -16,18 +16,13 @@ use std::str::FromStr;
 
 use itertools::Itertools;
 use risingwave_common::bail_not_implemented;
-use risingwave_common::catalog::{
-    Field, Schema, PG_CATALOG_SCHEMA_NAME, RW_INTERNAL_TABLE_FUNCTION_NAME,
-};
+use risingwave_common::catalog::{Field, Schema, RW_INTERNAL_TABLE_FUNCTION_NAME};
 use risingwave_common::types::DataType;
 use risingwave_sqlparser::ast::{Function, FunctionArg, ObjectName, TableAlias};
 
 use super::watermark::is_watermark_func;
 use super::{Binder, Relation, Result, WindowTableFunctionKind};
 use crate::binder::bind_context::Clause;
-use crate::catalog::system_catalog::pg_catalog::{
-    PG_GET_KEYWORDS_FUNC_NAME, PG_KEYWORDS_TABLE_NAME,
-};
 use crate::error::ErrorCode;
 use crate::expr::{Expr, ExprImpl};
 
@@ -57,24 +52,6 @@ impl Binder {
                 }
                 return self.bind_internal_table(args, alias);
             }
-            if func_name.eq_ignore_ascii_case(PG_GET_KEYWORDS_FUNC_NAME)
-                || name.real_value().eq_ignore_ascii_case(
-                    format!("{}.{}", PG_CATALOG_SCHEMA_NAME, PG_GET_KEYWORDS_FUNC_NAME).as_str(),
-                )
-            {
-                if with_ordinality {
-                    bail_not_implemented!(
-                        "WITH ORDINALITY for internal/system table function {}",
-                        func_name
-                    );
-                }
-                return self.bind_relation_by_name_inner(
-                    Some(PG_CATALOG_SCHEMA_NAME),
-                    PG_KEYWORDS_TABLE_NAME,
-                    alias,
-                    None,
-                );
-            }
         }
         // window table functions (tumble/hop)
         if let Ok(kind) = WindowTableFunctionKind::from_str(func_name) {
@@ -143,7 +120,7 @@ impl Binder {
             // Note: named return value should take precedence over table alias.
             // But we don't support it yet.
             // e.g.,
-            // ```
+            // ```sql
             // > create function foo(ret out int) language sql as 'select 1';
             // > select t.ret from foo() as t;
             // ```
diff --git a/src/frontend/src/binder/values.rs b/src/frontend/src/binder/values.rs
index 93f9e3f4f8a18..9e190c3cf4dc5 100644
--- a/src/frontend/src/binder/values.rs
+++ b/src/frontend/src/binder/values.rs
@@ -154,7 +154,7 @@ impl Binder {
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::util::iter_util::{zip_eq_fast, ZipEqFast};
+    use risingwave_common::util::iter_util::zip_eq_fast;
     use risingwave_sqlparser::ast::{Expr, Value};
 
     use super::*;
diff --git a/src/frontend/src/catalog/catalog_service.rs b/src/frontend/src/catalog/catalog_service.rs
index 7994fca811a35..f2bcdd2b62e12 100644
--- a/src/frontend/src/catalog/catalog_service.rs
+++ b/src/frontend/src/catalog/catalog_service.rs
@@ -34,7 +34,7 @@ use risingwave_rpc_client::MetaClient;
 use tokio::sync::watch::Receiver;
 
 use super::root_catalog::Catalog;
-use super::{DatabaseId, TableId};
+use super::{DatabaseId, SecretId, TableId};
 use crate::error::Result;
 use crate::user::UserId;
 
@@ -43,6 +43,7 @@ pub type CatalogReadGuard = ArcRwLockReadGuard<RawRwLock, Catalog>;
 /// [`CatalogReader`] can read catalog from local catalog and force the holder can not modify it.
 #[derive(Clone)]
 pub struct CatalogReader(Arc<RwLock<Catalog>>);
+
 impl CatalogReader {
     pub fn new(inner: Arc<RwLock<Catalog>>) -> Self {
         CatalogReader(inner)
@@ -117,11 +118,7 @@ pub trait CatalogWriter: Send + Sync {
         affected_table_change: Option<PbReplaceTablePlan>,
     ) -> Result<()>;
 
-    async fn create_subscription(
-        &self,
-        subscription: PbSubscription,
-        graph: StreamFragmentGraph,
-    ) -> Result<()>;
+    async fn create_subscription(&self, subscription: PbSubscription) -> Result<()>;
 
     async fn create_function(&self, function: PbFunction) -> Result<()>;
 
@@ -134,6 +131,15 @@ pub trait CatalogWriter: Send + Sync {
         connection: create_connection_request::Payload,
     ) -> Result<()>;
 
+    async fn create_secret(
+        &self,
+        secret_name: String,
+        database_id: u32,
+        schema_id: u32,
+        owner_id: u32,
+        payload: Vec<u8>,
+    ) -> Result<()>;
+
     async fn comment_on(&self, comment: PbComment) -> Result<()>;
 
     async fn drop_table(
@@ -168,6 +174,8 @@ pub trait CatalogWriter: Send + Sync {
 
     async fn drop_connection(&self, connection_id: u32) -> Result<()>;
 
+    async fn drop_secret(&self, secret_id: SecretId) -> Result<()>;
+
     async fn alter_table_name(&self, table_id: u32, table_name: &str) -> Result<()>;
 
     async fn alter_view_name(&self, view_id: u32, view_name: &str) -> Result<()>;
@@ -204,6 +212,13 @@ pub trait CatalogWriter: Send + Sync {
         object: alter_set_schema_request::Object,
         new_schema_id: u32,
     ) -> Result<()>;
+
+    async fn list_change_log_epochs(
+        &self,
+        table_id: u32,
+        min_epoch: u64,
+        max_count: u32,
+    ) -> Result<Vec<u64>>;
 }
 
 #[derive(Clone)]
@@ -339,15 +354,8 @@ impl CatalogWriter for CatalogWriterImpl {
         self.wait_version(version).await
     }
 
-    async fn create_subscription(
-        &self,
-        subscription: PbSubscription,
-        graph: StreamFragmentGraph,
-    ) -> Result<()> {
-        let version = self
-            .meta_client
-            .create_subscription(subscription, graph)
-            .await?;
+    async fn create_subscription(&self, subscription: PbSubscription) -> Result<()> {
+        let version = self.meta_client.create_subscription(subscription).await?;
         self.wait_version(version).await
     }
 
@@ -377,6 +385,21 @@ impl CatalogWriter for CatalogWriterImpl {
         self.wait_version(version).await
     }
 
+    async fn create_secret(
+        &self,
+        secret_name: String,
+        database_id: u32,
+        schema_id: u32,
+        owner_id: u32,
+        payload: Vec<u8>,
+    ) -> Result<()> {
+        let version = self
+            .meta_client
+            .create_secret(secret_name, database_id, schema_id, owner_id, payload)
+            .await?;
+        self.wait_version(version).await
+    }
+
     async fn comment_on(&self, comment: PbComment) -> Result<()> {
         let version = self.meta_client.comment_on(comment).await?;
         self.wait_version(version).await
@@ -459,6 +482,11 @@ impl CatalogWriter for CatalogWriterImpl {
         self.wait_version(version).await
     }
 
+    async fn drop_secret(&self, secret_id: SecretId) -> Result<()> {
+        let version = self.meta_client.drop_secret(secret_id).await?;
+        self.wait_version(version).await
+    }
+
     async fn alter_table_name(&self, table_id: u32, table_name: &str) -> Result<()> {
         let version = self
             .meta_client
@@ -568,6 +596,18 @@ impl CatalogWriter for CatalogWriterImpl {
 
         Ok(())
     }
+
+    async fn list_change_log_epochs(
+        &self,
+        table_id: u32,
+        min_epoch: u64,
+        max_count: u32,
+    ) -> Result<Vec<u64>> {
+        Ok(self
+            .meta_client
+            .list_change_log_epochs(table_id, min_epoch, max_count)
+            .await?)
+    }
 }
 
 impl CatalogWriterImpl {
diff --git a/src/frontend/src/catalog/database_catalog.rs b/src/frontend/src/catalog/database_catalog.rs
index ec040e309eba8..c562e5fc77a86 100644
--- a/src/frontend/src/catalog/database_catalog.rs
+++ b/src/frontend/src/catalog/database_catalog.rs
@@ -17,6 +17,7 @@ use std::collections::HashMap;
 use itertools::Itertools;
 use risingwave_common::catalog::PG_CATALOG_SCHEMA_NAME;
 use risingwave_pb::catalog::{PbDatabase, PbSchema};
+use risingwave_pb::user::grant_privilege::Object;
 
 use super::OwnedByUserCatalog;
 use crate::catalog::schema_catalog::SchemaCatalog;
@@ -99,6 +100,16 @@ impl DatabaseCatalog {
             .find(|schema| schema.get_table_by_id(table_id).is_some())
     }
 
+    pub fn get_grant_object_by_oid(&self, oid: u32) -> Option<Object> {
+        for schema in self.schema_by_name.values() {
+            let object = schema.get_grant_object_by_oid(oid);
+            if object.is_some() {
+                return object;
+            }
+        }
+        None
+    }
+
     pub fn update_schema(&mut self, prost: &PbSchema) {
         let id = prost.id;
         let name = prost.name.clone();
diff --git a/src/frontend/src/catalog/function_catalog.rs b/src/frontend/src/catalog/function_catalog.rs
index 1a0e01db1b92a..e042f33f377a8 100644
--- a/src/frontend/src/catalog/function_catalog.rs
+++ b/src/frontend/src/catalog/function_catalog.rs
@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use enum_as_inner::EnumAsInner;
 use parse_display::Display;
 use risingwave_common::catalog::FunctionId;
 use risingwave_common::types::DataType;
 use risingwave_pb::catalog::function::PbKind;
 use risingwave_pb::catalog::PbFunction;
+use risingwave_pb::expr::PbUserDefinedFunctionMetadata;
 
 use crate::catalog::OwnedByUserCatalog;
 
@@ -39,7 +41,7 @@ pub struct FunctionCatalog {
     pub runtime: Option<String>,
 }
 
-#[derive(Clone, Display, PartialEq, Eq, Hash, Debug)]
+#[derive(Clone, Display, PartialEq, Eq, Hash, Debug, EnumAsInner)]
 #[display(style = "UPPERCASE")]
 pub enum FunctionKind {
     Scalar,
@@ -80,6 +82,22 @@ impl From<&PbFunction> for FunctionCatalog {
     }
 }
 
+impl From<&FunctionCatalog> for PbUserDefinedFunctionMetadata {
+    fn from(c: &FunctionCatalog) -> Self {
+        PbUserDefinedFunctionMetadata {
+            arg_names: c.arg_names.clone(),
+            arg_types: c.arg_types.iter().map(|t| t.to_protobuf()).collect(),
+            language: c.language.clone(),
+            link: c.link.clone(),
+            identifier: c.identifier.clone(),
+            body: c.body.clone(),
+            compressed_binary: c.compressed_binary.clone(),
+            function_type: c.function_type.clone(),
+            runtime: c.runtime.clone(),
+        }
+    }
+}
+
 impl OwnedByUserCatalog for FunctionCatalog {
     fn owner(&self) -> u32 {
         self.owner
diff --git a/src/frontend/src/catalog/mod.rs b/src/frontend/src/catalog/mod.rs
index 687fa5ac350e7..64c3f525dd8f6 100644
--- a/src/frontend/src/catalog/mod.rs
+++ b/src/frontend/src/catalog/mod.rs
@@ -39,6 +39,8 @@ pub(crate) mod system_catalog;
 pub(crate) mod table_catalog;
 pub(crate) mod view_catalog;
 
+pub(crate) mod secret_catalog;
+
 pub(crate) use catalog_service::CatalogReader;
 pub use index_catalog::IndexCatalog;
 pub use table_catalog::TableCatalog;
@@ -55,6 +57,7 @@ pub(crate) type SchemaId = u32;
 pub(crate) type TableId = risingwave_common::catalog::TableId;
 pub(crate) type ColumnId = risingwave_common::catalog::ColumnId;
 pub(crate) type FragmentId = u32;
+pub(crate) type SecretId = risingwave_common::catalog::SecretId;
 
 /// Check if the column name does not conflict with the internally reserved column name.
 pub fn check_valid_column_name(column_name: &str) -> Result<()> {
diff --git a/src/frontend/src/catalog/root_catalog.rs b/src/frontend/src/catalog/root_catalog.rs
index 88e75f4ad90d6..5e6431065ce79 100644
--- a/src/frontend/src/catalog/root_catalog.rs
+++ b/src/frontend/src/catalog/root_catalog.rs
@@ -21,8 +21,8 @@ use risingwave_common::session_config::{SearchPath, USER_NAME_WILD_CARD};
 use risingwave_common::types::DataType;
 use risingwave_connector::sink::catalog::SinkCatalog;
 use risingwave_pb::catalog::{
-    PbConnection, PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbSubscription,
-    PbTable, PbView,
+    PbConnection, PbDatabase, PbFunction, PbIndex, PbSchema, PbSecret, PbSink, PbSource,
+    PbSubscription, PbTable, PbView,
 };
 use risingwave_pb::hummock::HummockVersionStats;
 
@@ -30,10 +30,13 @@ use super::function_catalog::FunctionCatalog;
 use super::source_catalog::SourceCatalog;
 use super::subscription_catalog::SubscriptionCatalog;
 use super::view_catalog::ViewCatalog;
-use super::{CatalogError, CatalogResult, ConnectionId, SinkId, SourceId, SubscriptionId, ViewId};
+use super::{
+    CatalogError, CatalogResult, ConnectionId, SecretId, SinkId, SourceId, SubscriptionId, ViewId,
+};
 use crate::catalog::connection_catalog::ConnectionCatalog;
 use crate::catalog::database_catalog::DatabaseCatalog;
 use crate::catalog::schema_catalog::SchemaCatalog;
+use crate::catalog::secret_catalog::SecretCatalog;
 use crate::catalog::system_catalog::{
     get_sys_tables_in_schema, get_sys_views_in_schema, SystemTableCatalog,
 };
@@ -201,6 +204,14 @@ impl Catalog {
             .create_subscription(proto);
     }
 
+    pub fn create_secret(&mut self, proto: &PbSecret) {
+        self.get_database_mut(proto.database_id)
+            .unwrap()
+            .get_schema_mut(proto.schema_id)
+            .unwrap()
+            .create_secret(proto);
+    }
+
     pub fn create_view(&mut self, proto: &PbView) {
         self.get_database_mut(proto.database_id)
             .unwrap()
@@ -257,6 +268,25 @@ impl Catalog {
         }
     }
 
+    pub fn update_secret(&mut self, proto: &PbSecret) {
+        let database = self.get_database_mut(proto.database_id).unwrap();
+        let schema = database.get_schema_mut(proto.schema_id).unwrap();
+        let secret_id = SecretId::new(proto.id);
+        if schema.get_secret_by_id(&secret_id).is_some() {
+            schema.update_secret(proto);
+        } else {
+            // Enter this branch when schema is changed by `ALTER ... SET SCHEMA ...` statement.
+            schema.create_secret(proto);
+            database
+                .iter_schemas_mut()
+                .find(|schema| {
+                    schema.id() != proto.schema_id && schema.get_secret_by_id(&secret_id).is_some()
+                })
+                .unwrap()
+                .drop_secret(secret_id);
+        }
+    }
+
     pub fn drop_database(&mut self, db_id: DatabaseId) {
         let name = self.db_name_by_id.remove(&db_id).unwrap();
         let database = self.database_by_name.remove(&name).unwrap();
@@ -377,6 +407,14 @@ impl Catalog {
             .drop_sink(sink_id);
     }
 
+    pub fn drop_secret(&mut self, db_id: DatabaseId, schema_id: SchemaId, secret_id: SecretId) {
+        self.get_database_mut(db_id)
+            .unwrap()
+            .get_schema_mut(schema_id)
+            .unwrap()
+            .drop_secret(secret_id);
+    }
+
     pub fn update_sink(&mut self, proto: &PbSink) {
         let database = self.get_database_mut(proto.database_id).unwrap();
         let schema = database.get_schema_mut(proto.schema_id).unwrap();
@@ -597,6 +635,7 @@ impl Catalog {
         ))
     }
 
+    /// Used to get `TableCatalog` for Materialized Views, Tables and Indexes.
     pub fn get_table_by_name<'a>(
         &self,
         db_name: &str,
@@ -792,6 +831,21 @@ impl Catalog {
         Err(CatalogError::NotFound("view", view_id.to_string()))
     }
 
+    pub fn get_secret_by_name<'a>(
+        &self,
+        db_name: &str,
+        schema_path: SchemaPath<'a>,
+        secret_name: &str,
+    ) -> CatalogResult<(&Arc<SecretCatalog>, &'a str)> {
+        schema_path
+            .try_find(|schema_name| {
+                Ok(self
+                    .get_schema_by_name(db_name, schema_name)?
+                    .get_secret_by_name(secret_name))
+            })?
+            .ok_or_else(|| CatalogError::NotFound("secret", secret_name.to_string()))
+    }
+
     pub fn get_connection_by_name<'a>(
         &self,
         db_name: &str,
@@ -957,6 +1011,21 @@ impl Catalog {
         }
     }
 
+    pub fn check_secret_name_duplicated(
+        &self,
+        db_name: &str,
+        schema_name: &str,
+        secret_name: &str,
+    ) -> CatalogResult<()> {
+        let schema = self.get_schema_by_name(db_name, schema_name)?;
+
+        if schema.get_secret_by_name(secret_name).is_some() {
+            Err(CatalogError::Duplicated("secret", secret_name.to_string()))
+        } else {
+            Ok(())
+        }
+    }
+
     /// Get the catalog cache's catalog version.
     pub fn version(&self) -> u64 {
         self.version
diff --git a/src/frontend/src/catalog/schema_catalog.rs b/src/frontend/src/catalog/schema_catalog.rs
index 56de1d743da41..fffb171c4c8bc 100644
--- a/src/frontend/src/catalog/schema_catalog.rs
+++ b/src/frontend/src/catalog/schema_catalog.rs
@@ -22,19 +22,22 @@ use risingwave_common::types::DataType;
 use risingwave_connector::sink::catalog::SinkCatalog;
 pub use risingwave_expr::sig::*;
 use risingwave_pb::catalog::{
-    PbConnection, PbFunction, PbIndex, PbSchema, PbSink, PbSource, PbSubscription, PbTable, PbView,
+    PbConnection, PbFunction, PbIndex, PbSchema, PbSecret, PbSink, PbSource, PbSubscription,
+    PbTable, PbView,
 };
+use risingwave_pb::user::grant_privilege::Object;
 
 use super::subscription_catalog::SubscriptionCatalog;
 use super::{OwnedByUserCatalog, SubscriptionId};
 use crate::catalog::connection_catalog::ConnectionCatalog;
 use crate::catalog::function_catalog::FunctionCatalog;
 use crate::catalog::index_catalog::IndexCatalog;
+use crate::catalog::secret_catalog::SecretCatalog;
 use crate::catalog::source_catalog::SourceCatalog;
 use crate::catalog::system_catalog::SystemTableCatalog;
 use crate::catalog::table_catalog::TableCatalog;
 use crate::catalog::view_catalog::ViewCatalog;
-use crate::catalog::{ConnectionId, DatabaseId, SchemaId, SinkId, SourceId, ViewId};
+use crate::catalog::{ConnectionId, DatabaseId, SchemaId, SecretId, SinkId, SourceId, ViewId};
 use crate::expr::{infer_type_name, infer_type_with_sigmap, Expr, ExprImpl};
 use crate::user::UserId;
 
@@ -61,6 +64,11 @@ pub struct SchemaCatalog {
     function_by_id: HashMap<FunctionId, Arc<FunctionCatalog>>,
     connection_by_name: HashMap<String, Arc<ConnectionCatalog>>,
     connection_by_id: HashMap<ConnectionId, Arc<ConnectionCatalog>>,
+    secret_by_name: HashMap<String, Arc<SecretCatalog>>,
+    secret_by_id: HashMap<SecretId, Arc<SecretCatalog>>,
+
+    _secret_source_ref: HashMap<SecretId, Vec<SourceId>>,
+    _secret_sink_ref: HashMap<SecretId, Vec<SinkId>>,
 
     // This field is currently used only for `show connections`
     connection_source_ref: HashMap<ConnectionId, Vec<SourceId>>,
@@ -483,6 +491,46 @@ impl SchemaCatalog {
             .expect("connection not found by name");
     }
 
+    pub fn create_secret(&mut self, prost: &PbSecret) {
+        let name = prost.name.clone();
+        let id = SecretId::new(prost.id);
+        let secret = SecretCatalog::from(prost);
+        let secret_ref = Arc::new(secret);
+
+        self.secret_by_id
+            .try_insert(id, secret_ref.clone())
+            .unwrap();
+        self.secret_by_name
+            .try_insert(name, secret_ref.clone())
+            .unwrap();
+    }
+
+    pub fn update_secret(&mut self, prost: &PbSecret) {
+        let name = prost.name.clone();
+        let id = SecretId::new(prost.id);
+        let secret = SecretCatalog::from(prost);
+        let secret_ref = Arc::new(secret);
+
+        let old_secret = self.secret_by_id.get(&id).unwrap();
+        // check if secret name get updated.
+        if old_secret.name != name {
+            self.secret_by_name.remove(&old_secret.name);
+        }
+
+        self.secret_by_name.insert(name, secret_ref.clone());
+        self.secret_by_id.insert(id, secret_ref);
+    }
+
+    pub fn drop_secret(&mut self, secret_id: SecretId) {
+        let secret_ref = self
+            .secret_by_id
+            .remove(&secret_id)
+            .expect("secret not found by id");
+        self.secret_by_name
+            .remove(&secret_ref.name)
+            .expect("secret not found by name");
+    }
+
     pub fn iter_all(&self) -> impl Iterator<Item = &Arc<TableCatalog>> {
         self.table_by_name.values()
     }
@@ -545,6 +593,10 @@ impl SchemaCatalog {
         self.connection_by_name.values()
     }
 
+    pub fn iter_secret(&self) -> impl Iterator<Item = &Arc<SecretCatalog>> {
+        self.secret_by_name.values()
+    }
+
     pub fn iter_system_tables(&self) -> impl Iterator<Item = &Arc<SystemTableCatalog>> {
         self.system_table_by_name.values()
     }
@@ -686,6 +738,14 @@ impl SchemaCatalog {
         self.connection_by_name.get(connection_name)
     }
 
+    pub fn get_secret_by_name(&self, secret_name: &str) -> Option<&Arc<SecretCatalog>> {
+        self.secret_by_name.get(secret_name)
+    }
+
+    pub fn get_secret_by_id(&self, secret_id: &SecretId) -> Option<&Arc<SecretCatalog>> {
+        self.secret_by_id.get(secret_id)
+    }
+
     /// get all sources referencing the connection
     pub fn get_source_ids_by_connection(
         &self,
@@ -703,6 +763,23 @@ impl SchemaCatalog {
             .map(|s| s.to_owned())
     }
 
+    pub fn get_grant_object_by_oid(&self, oid: u32) -> Option<Object> {
+        #[allow(clippy::manual_map)]
+        if self.get_table_by_id(&TableId::new(oid)).is_some()
+            || self.get_index_by_id(&IndexId::new(oid)).is_some()
+        {
+            Some(Object::TableId(oid))
+        } else if self.get_source_by_id(&oid).is_some() {
+            Some(Object::SourceId(oid))
+        } else if self.get_sink_by_id(&oid).is_some() {
+            Some(Object::SinkId(oid))
+        } else if self.get_view_by_id(&oid).is_some() {
+            Some(Object::ViewId(oid))
+        } else {
+            None
+        }
+    }
+
     pub fn id(&self) -> SchemaId {
         self.id
     }
@@ -746,6 +823,10 @@ impl From<&PbSchema> for SchemaCatalog {
             function_by_id: HashMap::new(),
             connection_by_name: HashMap::new(),
             connection_by_id: HashMap::new(),
+            secret_by_name: HashMap::new(),
+            secret_by_id: HashMap::new(),
+            _secret_source_ref: HashMap::new(),
+            _secret_sink_ref: HashMap::new(),
             connection_source_ref: HashMap::new(),
             connection_sink_ref: HashMap::new(),
             subscription_by_name: HashMap::new(),
diff --git a/src/frontend/src/catalog/secret_catalog.rs b/src/frontend/src/catalog/secret_catalog.rs
new file mode 100644
index 0000000000000..5e9aaae7dec99
--- /dev/null
+++ b/src/frontend/src/catalog/secret_catalog.rs
@@ -0,0 +1,45 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_pb::catalog::PbSecret;
+
+use crate::catalog::{DatabaseId, OwnedByUserCatalog, SecretId};
+use crate::user::UserId;
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct SecretCatalog {
+    pub secret_id: SecretId,
+    pub name: String,
+    pub database_id: DatabaseId,
+    pub value: Vec<u8>,
+    pub owner: UserId,
+}
+
+impl From<&PbSecret> for SecretCatalog {
+    fn from(value: &PbSecret) -> Self {
+        Self {
+            secret_id: SecretId::new(value.id),
+            database_id: value.database_id,
+            owner: value.owner,
+            name: value.name.clone(),
+            value: value.value.clone(),
+        }
+    }
+}
+
+impl OwnedByUserCatalog for SecretCatalog {
+    fn owner(&self) -> UserId {
+        self.owner
+    }
+}
diff --git a/src/frontend/src/catalog/subscription_catalog.rs b/src/frontend/src/catalog/subscription_catalog.rs
index 1409948e07bd9..36a5a71a0e9be 100644
--- a/src/frontend/src/catalog/subscription_catalog.rs
+++ b/src/frontend/src/catalog/subscription_catalog.rs
@@ -13,18 +13,17 @@
 // limitations under the License.
 
 use core::str::FromStr;
-use std::collections::{BTreeMap, HashSet};
 
-use itertools::Itertools;
-use risingwave_common::catalog::{ColumnCatalog, TableId, UserId, OBJECT_ID_PLACEHOLDER};
+use risingwave_common::catalog::{TableId, UserId, OBJECT_ID_PLACEHOLDER};
 use risingwave_common::types::Interval;
 use risingwave_common::util::epoch::Epoch;
-use risingwave_common::util::sort_util::ColumnOrder;
-use risingwave_pb::catalog::{PbStreamJobStatus, PbSubscription};
+use risingwave_pb::catalog::subscription::PbSubscriptionState;
+use risingwave_pb::catalog::PbSubscription;
 use thiserror_ext::AsReport;
 
 use super::OwnedByUserCatalog;
 use crate::error::{ErrorCode, Result};
+use crate::WithOptions;
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 #[cfg_attr(test, derive(Default))]
@@ -38,21 +37,8 @@ pub struct SubscriptionCatalog {
     /// Full SQL definition of the subscription. For debug now.
     pub definition: String,
 
-    /// All columns of the subscription. Note that this is NOT sorted by columnId in the vector.
-    pub columns: Vec<ColumnCatalog>,
-
-    /// Primiary keys of the subscription. Derived by the frontend.
-    pub plan_pk: Vec<ColumnOrder>,
-
-    /// Distribution key indices of the subscription. For example, if `distribution_key = [1, 2]`, then the
-    /// distribution keys will be `columns[1]` and `columns[2]`.
-    pub distribution_key: Vec<usize>,
-
-    /// The properties of the subscription, only `retention`.
-    pub properties: BTreeMap<String, String>,
-
-    /// The upstream table name on which the subscription depends
-    pub subscription_from_name: String,
+    /// The retention seconds of the subscription.
+    pub retention_seconds: u64,
 
     /// The database id
     pub database_id: u32,
@@ -61,7 +47,7 @@ pub struct SubscriptionCatalog {
     pub schema_id: u32,
 
     /// The subscription depends on the upstream list
-    pub dependent_relations: Vec<TableId>,
+    pub dependent_table_id: TableId,
 
     /// The user id
     pub owner: UserId,
@@ -71,7 +57,6 @@ pub struct SubscriptionCatalog {
 
     pub created_at_cluster_version: Option<String>,
     pub initialized_at_cluster_version: Option<String>,
-    pub subscription_internal_table_name: Option<String>,
 }
 
 #[derive(Clone, Copy, Debug, Default, Hash, PartialOrd, PartialEq, Eq, Ord)]
@@ -97,14 +82,8 @@ impl SubscriptionId {
 }
 
 impl SubscriptionCatalog {
-    pub fn add_dependent_relations(mut self, mut dependent_relations: HashSet<TableId>) -> Self {
-        dependent_relations.extend(self.dependent_relations);
-        self.dependent_relations = dependent_relations.into_iter().collect();
-        self
-    }
-
-    pub fn get_retention_seconds(&self) -> Result<u64> {
-        let retention_seconds_str = self.properties.get("retention").ok_or_else(|| {
+    pub fn set_retention_seconds(&mut self, properties: &WithOptions) -> Result<()> {
+        let retention_seconds_str = properties.get("retention").ok_or_else(|| {
             ErrorCode::InternalError("Subscription retention time not set.".to_string())
         })?;
         let retention_seconds = (Interval::from_str(retention_seconds_str)
@@ -116,47 +95,29 @@ impl SubscriptionCatalog {
             })?
             .epoch_in_micros()
             / 1000000) as u64;
-
-        Ok(retention_seconds)
+        self.retention_seconds = retention_seconds;
+        Ok(())
     }
 
     pub fn create_sql(&self) -> String {
         self.definition.clone()
     }
 
-    pub fn get_log_store_name(&self) -> String {
-        self.subscription_internal_table_name.clone().unwrap()
-    }
-
     pub fn to_proto(&self) -> PbSubscription {
-        assert!(!self.dependent_relations.is_empty());
         PbSubscription {
             id: self.id.subscription_id,
             name: self.name.clone(),
             definition: self.definition.clone(),
-            column_catalogs: self
-                .columns
-                .iter()
-                .map(|column| column.to_protobuf())
-                .collect_vec(),
-            plan_pk: self.plan_pk.iter().map(|k| k.to_protobuf()).collect_vec(),
-            distribution_key: self.distribution_key.iter().map(|k| *k as _).collect_vec(),
-            subscription_from_name: self.subscription_from_name.clone(),
-            properties: self.properties.clone().into_iter().collect(),
+            retention_seconds: self.retention_seconds,
             database_id: self.database_id,
             schema_id: self.schema_id,
-            dependent_relations: self
-                .dependent_relations
-                .iter()
-                .map(|k| k.table_id)
-                .collect_vec(),
             initialized_at_epoch: self.initialized_at_epoch.map(|e| e.0),
             created_at_epoch: self.created_at_epoch.map(|e| e.0),
             owner: self.owner.into(),
-            stream_job_status: PbStreamJobStatus::Creating.into(),
             initialized_at_cluster_version: self.initialized_at_cluster_version.clone(),
             created_at_cluster_version: self.created_at_cluster_version.clone(),
-            subscription_internal_table_name: self.subscription_internal_table_name.clone(),
+            dependent_table_id: self.dependent_table_id.table_id,
+            subscription_state: PbSubscriptionState::Init.into(),
         }
     }
 }
@@ -167,32 +128,15 @@ impl From<&PbSubscription> for SubscriptionCatalog {
             id: SubscriptionId::new(prost.id),
             name: prost.name.clone(),
             definition: prost.definition.clone(),
-            columns: prost
-                .column_catalogs
-                .iter()
-                .map(|c| ColumnCatalog::from(c.clone()))
-                .collect_vec(),
-            plan_pk: prost
-                .plan_pk
-                .iter()
-                .map(ColumnOrder::from_protobuf)
-                .collect_vec(),
-            distribution_key: prost.distribution_key.iter().map(|k| *k as _).collect_vec(),
-            subscription_from_name: prost.subscription_from_name.clone(),
-            properties: prost.properties.clone().into_iter().collect(),
+            retention_seconds: prost.retention_seconds,
             database_id: prost.database_id,
             schema_id: prost.schema_id,
-            dependent_relations: prost
-                .dependent_relations
-                .iter()
-                .map(|k| TableId::new(*k))
-                .collect_vec(),
+            dependent_table_id: TableId::new(prost.dependent_table_id),
             owner: prost.owner.into(),
             created_at_epoch: prost.created_at_epoch.map(Epoch::from),
             initialized_at_epoch: prost.initialized_at_epoch.map(Epoch::from),
             created_at_cluster_version: prost.created_at_cluster_version.clone(),
             initialized_at_cluster_version: prost.initialized_at_cluster_version.clone(),
-            subscription_internal_table_name: prost.subscription_internal_table_name.clone(),
         }
     }
 }
diff --git a/src/frontend/src/catalog/system_catalog/information_schema/columns.rs b/src/frontend/src/catalog/system_catalog/information_schema/columns.rs
index 531b8b3012c8f..b7e5889d16c27 100644
--- a/src/frontend/src/catalog/system_catalog/information_schema/columns.rs
+++ b/src/frontend/src/catalog/system_catalog/information_schema/columns.rs
@@ -31,6 +31,7 @@ use risingwave_frontend_macro::system_catalog;
         NULL AS column_default,
         NULL::integer AS character_maximum_length,
         NULL::integer AS numeric_precision,
+        NULL::integer AS numeric_precision_radix,
         NULL::integer AS numeric_scale,
         NULL::integer AS datetime_precision,
         c.position AS ordinal_position,
@@ -81,6 +82,7 @@ struct Column {
     column_default: String,
     character_maximum_length: i32,
     numeric_precision: i32,
+    numeric_precision_radix: i32,
     numeric_scale: i32,
     datetime_precision: i32,
     ordinal_position: i32,
diff --git a/src/frontend/src/catalog/system_catalog/pg_catalog/mod.rs b/src/frontend/src/catalog/system_catalog/pg_catalog/mod.rs
index 202646b55a603..3d1c8f2a56f74 100644
--- a/src/frontend/src/catalog/system_catalog/pg_catalog/mod.rs
+++ b/src/frontend/src/catalog/system_catalog/pg_catalog/mod.rs
@@ -29,7 +29,6 @@ mod pg_extension;
 mod pg_index;
 mod pg_indexes;
 mod pg_inherits;
-mod pg_keywords;
 mod pg_language;
 mod pg_locks;
 mod pg_matviews;
@@ -52,5 +51,3 @@ mod pg_trigger;
 mod pg_type;
 mod pg_user;
 mod pg_views;
-
-pub use pg_keywords::*;
diff --git a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_keywords.rs b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_keywords.rs
deleted file mode 100644
index c3bb43ef0a842..0000000000000
--- a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_keywords.rs
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use risingwave_common::types::Fields;
-use risingwave_frontend_macro::system_catalog;
-
-pub const PG_KEYWORDS_TABLE_NAME: &str = "pg_keywords";
-pub const PG_GET_KEYWORDS_FUNC_NAME: &str = "pg_get_keywords";
-
-/// The catalog `pg_keywords` stores keywords. `pg_get_keywords` returns the content of this table.
-/// Ref: [`https://www.postgresql.org/docs/15/functions-info.html`]
-// TODO: change to read reserved keywords here
-#[system_catalog(view, "pg_catalog.pg_keywords")]
-#[derive(Fields)]
-struct PgKeywords {
-    word: String,
-    catcode: String,
-    catdesc: String,
-}
diff --git a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_tables.rs b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_tables.rs
index 07b3a15e09b1b..27b4c214f0265 100644
--- a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_tables.rs
+++ b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_tables.rs
@@ -34,7 +34,8 @@ use risingwave_frontend_macro::system_catalog;
                 schema_id,
                 owner
                 FROM rw_catalog.rw_system_tables) AS t
-        JOIN rw_catalog.rw_schemas s ON t.schema_id = s.id"
+        JOIN rw_catalog.rw_schemas s ON t.schema_id = s.id
+        AND s.name <> 'rw_catalog'"
 )]
 #[derive(Fields)]
 struct PgTable {
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/mod.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/mod.rs
index 502c41561b65f..4e0e165a6d524 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/mod.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/mod.rs
@@ -56,3 +56,6 @@ mod rw_user_secrets;
 mod rw_users;
 mod rw_views;
 mod rw_worker_nodes;
+
+mod rw_actor_id_to_ddl;
+mod rw_fragment_id_to_ddl;
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_actor_id_to_ddl.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_actor_id_to_ddl.rs
new file mode 100644
index 0000000000000..95e269f0d5d5c
--- /dev/null
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_actor_id_to_ddl.rs
@@ -0,0 +1,42 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::types::Fields;
+use risingwave_frontend_macro::system_catalog;
+
+/// Provides a mapping from `actor_id` to its ddl info.
+#[system_catalog(
+view,
+"rw_catalog.rw_actor_id_to_ddl",
+"with
+   actor_to_job_id as (select actor_id, a.fragment_id, table_id from rw_fragments f join rw_actors a on f.fragment_id = a.fragment_id),
+   job_id_to_mv as (select actor_id, fragment_id, d.id as job_id, schema_id, 'mv' as ddl_type, name from rw_materialized_views d join actor_to_job_id a on d.id = a.table_id),
+   job_id_to_sink as (select actor_id, fragment_id, d.id as job_id, schema_id, 'sink' as ddl_type, name from rw_sinks d join actor_to_job_id a on d.id = a.table_id),
+   job_id_to_source as (select actor_id, fragment_id, d.id as job_id, schema_id, 'source' as ddl_type, name from rw_sources d join actor_to_job_id a on d.id = a.table_id),
+   job_id_to_table as (select actor_id, fragment_id, d.id as job_id, schema_id, 'table' as ddl_type, name from rw_tables d join actor_to_job_id a on d.id = a.table_id)
+   select * from job_id_to_mv
+     union all select * from job_id_to_sink
+       union all select * from job_id_to_source
+         union all select * from job_id_to_table"
+)]
+#[derive(Fields)]
+struct RwActorIdToDdl {
+    #[primary_key]
+    actor_id: i32,
+    fragment_id: i32,
+    job_id: i32,
+    schema_id: i32,
+    ddl_type: String,
+    name: String,
+}
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_fragment_id_to_ddl.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_fragment_id_to_ddl.rs
new file mode 100644
index 0000000000000..094e85903a31a
--- /dev/null
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_fragment_id_to_ddl.rs
@@ -0,0 +1,40 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::types::Fields;
+use risingwave_frontend_macro::system_catalog;
+
+/// Provides a mapping from `actor_id` to its ddl info.
+#[system_catalog(
+view,
+"rw_catalog.rw_fragment_id_to_ddl",
+"with
+   job_id_to_mv as (select fragment_id, d.id as job_id, schema_id, 'mv' as ddl_type, name from rw_materialized_views d join rw_fragments f on d.id = f.table_id),
+   job_id_to_sink as (select fragment_id, d.id as job_id, schema_id, 'sink' as ddl_type, name from rw_sinks d join rw_fragments f on d.id = f.table_id),
+   job_id_to_source as (select fragment_id, d.id as job_id, schema_id, 'source' as ddl_type, name from rw_sources d join rw_fragments f on d.id = f.table_id),
+   job_id_to_table as (select fragment_id, d.id as job_id, schema_id, 'table' as ddl_type, name from rw_tables d join rw_fragments f on d.id = f.table_id)
+   select * from job_id_to_mv
+     union all select * from job_id_to_sink
+       union all select * from job_id_to_source
+         union all select * from job_id_to_table"
+)]
+#[derive(Fields)]
+struct RwFragmentIdToDdl {
+    #[primary_key]
+    fragment_id: i32,
+    job_id: i32,
+    schema_id: i32,
+    ddl_type: String,
+    name: String,
+}
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_branched_objects.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_branched_objects.rs
index 443fa255f4398..43f62aaee2105 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_branched_objects.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_branched_objects.rs
@@ -19,11 +19,11 @@ use crate::catalog::system_catalog::SysCatalogReaderImpl;
 use crate::error::Result;
 
 #[derive(Fields)]
-#[primary_key(object_id, sst_id)] // TODO: is this correct?
+#[primary_key(object_id, compaction_group_id)]
 struct RwHummockBranchedObject {
     object_id: i64,
-    sst_id: i64,
     compaction_group_id: i64,
+    sst_id: Vec<i64>,
 }
 
 #[system_catalog(table, "rw_catalog.rw_hummock_branched_objects")]
@@ -33,7 +33,7 @@ async fn read(reader: &SysCatalogReaderImpl) -> Result<Vec<RwHummockBranchedObje
         .into_iter()
         .map(|o| RwHummockBranchedObject {
             object_id: o.object_id as _,
-            sst_id: o.sst_id as _,
+            sst_id: o.sst_id.into_iter().map(|id| id as _).collect(),
             compaction_group_id: o.compaction_group_id as _,
         })
         .collect();
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs
index e113f5f695f60..f7d265485f706 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_hummock_version.rs
@@ -106,7 +106,7 @@ fn version_to_compaction_group_rows(version: &HummockVersion) -> Vec<RwHummockVe
         .map(|cg| RwHummockVersion {
             version_id: version.id as _,
             max_committed_epoch: version.max_committed_epoch as _,
-            safe_epoch: version.safe_epoch as _,
+            safe_epoch: version.visible_table_safe_epoch() as _,
             compaction_group: json!(cg).into(),
         })
         .collect()
@@ -204,3 +204,28 @@ async fn read_hummock_table_watermarks(
         })
         .collect())
 }
+
+#[derive(Fields)]
+struct RwHummockSnapshot {
+    #[primary_key]
+    table_id: i32,
+    safe_epoch: i64,
+    committed_epoch: i64,
+}
+
+#[system_catalog(table, "rw_catalog.rw_hummock_snapshot")]
+async fn read_hummock_snapshot_groups(
+    reader: &SysCatalogReaderImpl,
+) -> Result<Vec<RwHummockSnapshot>> {
+    let version = reader.meta_client.get_hummock_current_version().await?;
+    Ok(version
+        .state_table_info
+        .info()
+        .iter()
+        .map(|(table_id, info)| RwHummockSnapshot {
+            table_id: table_id.table_id as _,
+            committed_epoch: info.committed_epoch as _,
+            safe_epoch: info.safe_epoch as _,
+        })
+        .collect())
+}
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_files.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_files.rs
index 7afe2561e1775..2bb7648920895 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_files.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_files.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
 use std::ops::Deref;
 
 use anyhow::anyhow;
@@ -44,7 +43,7 @@ struct RwIcebergFiles {
     /// Total file size in bytes
     pub file_size_in_bytes: i64,
     /// Field ids used to determine row equality in equality delete files.
-    /// Required when content is EqualityDeletes and should be null
+    /// Required when content is `EqualityDeletes` and should be null
     /// otherwise. Fields with ids listed in this column must be present
     /// in the delete file
     pub equality_ids: Option<Vec<i32>>,
@@ -78,8 +77,7 @@ async fn read(reader: &SysCatalogReaderImpl) -> Result<Vec<RwIcebergFiles>> {
 
     let mut result = vec![];
     for (schema_name, source) in iceberg_sources {
-        let source_props: HashMap<String, String> =
-            HashMap::from_iter(source.with_properties.clone());
+        let source_props = source.with_properties.clone();
         let config = ConnectorProperties::extract(source_props, false)?;
         if let ConnectorProperties::Iceberg(iceberg_properties) = config {
             let iceberg_config: IcebergConfig = iceberg_properties.to_iceberg_config();
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_snapshots.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_snapshots.rs
index 671cbcee89529..6e32f48eec844 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_snapshots.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_iceberg_snapshots.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
 use std::ops::Deref;
 
 use icelake::Table;
@@ -58,8 +57,7 @@ async fn read(reader: &SysCatalogReaderImpl) -> Result<Vec<RwIcebergSnapshots>>
 
     let mut result = vec![];
     for (schema_name, source) in iceberg_sources {
-        let source_props: HashMap<String, String> =
-            HashMap::from_iter(source.with_properties.clone());
+        let source_props = source.with_properties.clone();
         let config = ConnectorProperties::extract(source_props, false)?;
         if let ConnectorProperties::Iceberg(iceberg_properties) = config {
             let iceberg_config: IcebergConfig = iceberg_properties.to_iceberg_config();
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_meta_snapshot.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_meta_snapshot.rs
index f31b1f7c67c5c..51df244f9539c 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_meta_snapshot.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_meta_snapshot.rs
@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use risingwave_common::types::{Fields, Timestamp};
-use risingwave_common::util::epoch::Epoch;
+use risingwave_common::types::{Fields, JsonbVal};
 use risingwave_frontend_macro::system_catalog;
+use serde_json::json;
 
 use crate::catalog::system_catalog::SysCatalogReaderImpl;
 use crate::error::Result;
@@ -24,30 +24,13 @@ struct RwMetaSnapshot {
     #[primary_key]
     meta_snapshot_id: i64,
     hummock_version_id: i64,
-    // the smallest epoch this meta snapshot includes
-    safe_epoch: i64,
-    // human-readable timestamp of safe_epoch
-    safe_epoch_ts: Option<Timestamp>,
-    // the largest epoch this meta snapshot includes
-    max_committed_epoch: i64,
-    // human-readable timestamp of max_committed_epoch
-    max_committed_epoch_ts: Option<Timestamp>,
     remarks: Option<String>,
+    state_table_info: Option<JsonbVal>,
+    rw_version: Option<String>,
 }
 
 #[system_catalog(table, "rw_catalog.rw_meta_snapshot")]
 async fn read_meta_snapshot(reader: &SysCatalogReaderImpl) -> Result<Vec<RwMetaSnapshot>> {
-    let try_get_date_time = |epoch: u64| {
-        if epoch == 0 {
-            return None;
-        }
-        let time_millis = Epoch::from(epoch).as_unix_millis();
-        Timestamp::with_secs_nsecs(
-            (time_millis / 1000) as i64,
-            (time_millis % 1000 * 1_000_000) as u32,
-        )
-        .ok()
-    };
     let meta_snapshots = reader
         .meta_client
         .list_meta_snapshots()
@@ -56,11 +39,9 @@ async fn read_meta_snapshot(reader: &SysCatalogReaderImpl) -> Result<Vec<RwMetaS
         .map(|s| RwMetaSnapshot {
             meta_snapshot_id: s.id as _,
             hummock_version_id: s.hummock_version_id as _,
-            safe_epoch: s.safe_epoch as _,
-            safe_epoch_ts: try_get_date_time(s.safe_epoch),
-            max_committed_epoch: s.max_committed_epoch as _,
-            max_committed_epoch_ts: try_get_date_time(s.max_committed_epoch),
             remarks: s.remarks,
+            state_table_info: Some(json!(s.state_table_info).into()),
+            rw_version: s.rw_version.clone(),
         })
         .collect();
     Ok(meta_snapshots)
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_sources.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_sources.rs
index 5a05f69750caf..441ac99b48cda 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_sources.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_sources.rs
@@ -32,6 +32,7 @@ struct RwSource {
     format: Option<String>,
     row_encode: Option<String>,
     append_only: bool,
+    associated_table_id: Option<i32>,
     connection_id: Option<i32>,
     definition: String,
     acl: String,
@@ -51,40 +52,38 @@ fn read_rw_sources_info(reader: &SysCatalogReaderImpl) -> Result<Vec<RwSource>>
 
     Ok(schemas
         .flat_map(|schema| {
-            schema
-                .iter_source()
-                .filter(|s| s.associated_table_id.is_none())
-                .map(|source| RwSource {
-                    id: source.id as i32,
-                    name: source.name.clone(),
-                    schema_id: schema.id() as i32,
-                    owner: source.owner as i32,
-                    connector: source
-                        .with_properties
-                        .get(UPSTREAM_SOURCE_KEY)
-                        .cloned()
-                        .unwrap_or("".to_string())
-                        .to_uppercase(),
-                    columns: source.columns.iter().map(|c| c.name().into()).collect(),
-                    format: source
-                        .info
-                        .get_format()
-                        .ok()
-                        .map(|format| format.as_str_name().into()),
-                    row_encode: source
-                        .info
-                        .get_row_encode()
-                        .ok()
-                        .map(|row_encode| row_encode.as_str_name().into()),
-                    append_only: source.append_only,
-                    connection_id: source.connection_id.map(|id| id as i32),
-                    definition: source.create_sql(),
-                    acl: get_acl_items(&Object::SourceId(source.id), false, &users, username_map),
-                    initialized_at: source.initialized_at_epoch.map(|e| e.as_timestamptz()),
-                    created_at: source.created_at_epoch.map(|e| e.as_timestamptz()),
-                    initialized_at_cluster_version: source.initialized_at_cluster_version.clone(),
-                    created_at_cluster_version: source.created_at_cluster_version.clone(),
-                })
+            schema.iter_source().map(|source| RwSource {
+                id: source.id as i32,
+                name: source.name.clone(),
+                schema_id: schema.id() as i32,
+                owner: source.owner as i32,
+                connector: source
+                    .with_properties
+                    .get(UPSTREAM_SOURCE_KEY)
+                    .cloned()
+                    .unwrap_or("".to_string())
+                    .to_uppercase(),
+                columns: source.columns.iter().map(|c| c.name().into()).collect(),
+                format: source
+                    .info
+                    .get_format()
+                    .ok()
+                    .map(|format| format.as_str_name().into()),
+                row_encode: source
+                    .info
+                    .get_row_encode()
+                    .ok()
+                    .map(|row_encode| row_encode.as_str_name().into()),
+                append_only: source.append_only,
+                associated_table_id: source.associated_table_id.map(|id| id.table_id as i32),
+                connection_id: source.connection_id.map(|id| id as i32),
+                definition: source.create_sql(),
+                acl: get_acl_items(&Object::SourceId(source.id), false, &users, username_map),
+                initialized_at: source.initialized_at_epoch.map(|e| e.as_timestamptz()),
+                created_at: source.created_at_epoch.map(|e| e.as_timestamptz()),
+                initialized_at_cluster_version: source.initialized_at_cluster_version.clone(),
+                created_at_cluster_version: source.created_at_cluster_version.clone(),
+            })
         })
         .collect())
 }
diff --git a/src/frontend/src/catalog/table_catalog.rs b/src/frontend/src/catalog/table_catalog.rs
index f5f95861d29cd..bfe4537fa7085 100644
--- a/src/frontend/src/catalog/table_catalog.rs
+++ b/src/frontend/src/catalog/table_catalog.rs
@@ -60,7 +60,7 @@ use crate::user::UserId;
 /// - **Order Key**: the primary key for storage, used to sort and access data.
 ///
 ///   For an MV, the columns in `ORDER BY` clause will be put at the beginning of the order key. And
-/// the remaining columns in pk will follow behind.
+///   the remaining columns in pk will follow behind.
 ///
 ///   If there's no `ORDER BY` clause, the order key will be the same as pk.
 ///
@@ -450,6 +450,20 @@ impl TableCatalog {
             .map(|(i, _)| i)
     }
 
+    pub fn default_column_expr(&self, col_idx: usize) -> ExprImpl {
+        if let Some(GeneratedOrDefaultColumn::DefaultColumn(DefaultColumnDesc { expr, .. })) = self
+            .columns[col_idx]
+            .column_desc
+            .generated_or_default_column
+            .as_ref()
+        {
+            ExprImpl::from_expr_proto(expr.as_ref().unwrap())
+                .expect("expr in default columns corrupted")
+        } else {
+            ExprImpl::literal_null(self.columns[col_idx].data_type().clone())
+        }
+    }
+
     pub fn default_columns(&self) -> impl Iterator<Item = (usize, ExprImpl)> + '_ {
         self.columns.iter().enumerate().filter_map(|(i, c)| {
             if let Some(GeneratedOrDefaultColumn::DefaultColumn(DefaultColumnDesc {
@@ -581,19 +595,15 @@ impl OwnedByUserCatalog for TableCatalog {
 #[cfg(test)]
 mod tests {
 
-    use risingwave_common::catalog::{
-        row_id_column_desc, ColumnCatalog, ColumnDesc, ColumnId, TableId,
-    };
+    use risingwave_common::catalog::{row_id_column_desc, ColumnDesc, ColumnId};
     use risingwave_common::test_prelude::*;
     use risingwave_common::types::*;
     use risingwave_common::util::sort_util::OrderType;
-    use risingwave_pb::catalog::{PbStreamJobStatus, PbTable};
     use risingwave_pb::plan_common::{
         AdditionalColumn, ColumnDescVersion, PbColumnCatalog, PbColumnDesc,
     };
 
     use super::*;
-    use crate::catalog::table_catalog::{TableCatalog, TableType};
 
     #[test]
     fn test_into_table_catalog() {
diff --git a/src/frontend/src/error.rs b/src/frontend/src/error.rs
index e2ec1b35fe504..93b3e627ae856 100644
--- a/src/frontend/src/error.rs
+++ b/src/frontend/src/error.rs
@@ -237,3 +237,13 @@ impl From<JoinError> for RwError {
         ErrorCode::Uncategorized(join_error.into()).into()
     }
 }
+
+// For errors without a concrete type, put them into `Uncategorized`.
+impl From<BoxedError> for RwError {
+    fn from(e: BoxedError) -> Self {
+        // Show that the error is of `BoxedKind`, instead of `AdhocKind` which loses the sources.
+        // This is essentially expanded from `anyhow::anyhow!(e)`.
+        let e = anyhow::__private::kind::BoxedKind::anyhow_kind(&e).new(e);
+        ErrorCode::Uncategorized(e).into()
+    }
+}
diff --git a/src/frontend/src/expr/agg_call.rs b/src/frontend/src/expr/agg_call.rs
index 0f9493a694952..d72fba4dbcd2c 100644
--- a/src/frontend/src/expr/agg_call.rs
+++ b/src/frontend/src/expr/agg_call.rs
@@ -12,22 +12,27 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+
 use risingwave_common::types::DataType;
 use risingwave_expr::aggregate::AggKind;
 
 use super::{infer_type, Expr, ExprImpl, Literal, OrderBy};
+use crate::catalog::function_catalog::{FunctionCatalog, FunctionKind};
 use crate::error::Result;
 use crate::utils::Condition;
 
 #[derive(Clone, Eq, PartialEq, Hash)]
 pub struct AggCall {
-    agg_kind: AggKind,
-    return_type: DataType,
-    args: Vec<ExprImpl>,
-    distinct: bool,
-    order_by: OrderBy,
-    filter: Condition,
-    direct_args: Vec<Literal>,
+    pub agg_kind: AggKind,
+    pub return_type: DataType,
+    pub args: Vec<ExprImpl>,
+    pub distinct: bool,
+    pub order_by: OrderBy,
+    pub filter: Condition,
+    pub direct_args: Vec<Literal>,
+    /// Catalog of user defined aggregate function.
+    pub user_defined: Option<Arc<FunctionCatalog>>,
 }
 
 impl std::fmt::Debug for AggCall {
@@ -69,6 +74,7 @@ impl AggCall {
             order_by,
             filter,
             direct_args,
+            user_defined: None,
         })
     }
 
@@ -86,27 +92,32 @@ impl AggCall {
             order_by: OrderBy::any(),
             filter: Condition::true_cond(),
             direct_args: vec![],
+            user_defined: None,
         })
     }
 
-    pub fn decompose(
-        self,
-    ) -> (
-        AggKind,
-        Vec<ExprImpl>,
-        bool,
-        OrderBy,
-        Condition,
-        Vec<Literal>,
-    ) {
-        (
-            self.agg_kind,
-            self.args,
-            self.distinct,
-            self.order_by,
-            self.filter,
-            self.direct_args,
-        )
+    /// Create a user-defined `AggCall`.
+    pub fn new_user_defined(
+        args: Vec<ExprImpl>,
+        distinct: bool,
+        order_by: OrderBy,
+        filter: Condition,
+        direct_args: Vec<Literal>,
+        user_defined: Arc<FunctionCatalog>,
+    ) -> Result<Self> {
+        let FunctionKind::Aggregate = &user_defined.kind else {
+            panic!("not an aggregate function");
+        };
+        Ok(AggCall {
+            agg_kind: AggKind::UserDefined,
+            return_type: user_defined.return_type.clone(),
+            args,
+            distinct,
+            order_by,
+            filter,
+            direct_args,
+            user_defined: Some(user_defined),
+        })
     }
 
     pub fn agg_kind(&self) -> AggKind {
diff --git a/src/frontend/src/expr/expr_rewriter.rs b/src/frontend/src/expr/expr_rewriter.rs
index 48655430a0db5..4d5b960d654dc 100644
--- a/src/frontend/src/expr/expr_rewriter.rs
+++ b/src/frontend/src/expr/expr_rewriter.rs
@@ -60,16 +60,33 @@ pub trait ExprRewriter {
     }
 
     fn rewrite_agg_call(&mut self, agg_call: AggCall) -> ExprImpl {
-        let (func_type, inputs, distinct, order_by, filter, direct_args) = agg_call.decompose();
-        let inputs = inputs
+        let AggCall {
+            agg_kind,
+            return_type,
+            args,
+            distinct,
+            order_by,
+            filter,
+            direct_args,
+            user_defined,
+        } = agg_call;
+        let args = args
             .into_iter()
             .map(|expr| self.rewrite_expr(expr))
             .collect();
         let order_by = order_by.rewrite_expr(self);
         let filter = filter.rewrite_expr(self);
-        AggCall::new(func_type, inputs, distinct, order_by, filter, direct_args)
-            .unwrap()
-            .into()
+        AggCall {
+            agg_kind,
+            return_type,
+            args,
+            distinct,
+            order_by,
+            filter,
+            direct_args,
+            user_defined,
+        }
+        .into()
     }
 
     fn rewrite_parameter(&mut self, parameter: Parameter) -> ExprImpl {
@@ -97,7 +114,7 @@ pub trait ExprRewriter {
             args,
             return_type,
             function_type,
-            udtf_catalog,
+            user_defined: udtf_catalog,
         } = table_func;
         let args = args
             .into_iter()
@@ -107,7 +124,7 @@ pub trait ExprRewriter {
             args,
             return_type,
             function_type,
-            udtf_catalog,
+            user_defined: udtf_catalog,
         }
         .into()
     }
diff --git a/src/frontend/src/expr/function_impl/cast_regclass.rs b/src/frontend/src/expr/function_impl/cast_regclass.rs
index b1ec47a2d3508..c350d3984ab97 100644
--- a/src/frontend/src/expr/function_impl/cast_regclass.rs
+++ b/src/frontend/src/expr/function_impl/cast_regclass.rs
@@ -15,7 +15,6 @@
 use risingwave_common::session_config::SearchPath;
 use risingwave_expr::{capture_context, function, ExprError};
 use risingwave_sqlparser::parser::{Parser, ParserError};
-use risingwave_sqlparser::tokenizer::{Token, Tokenizer};
 use thiserror::Error;
 use thiserror_ext::AsReport;
 
@@ -63,7 +62,11 @@ fn resolve_regclass_inner(
     db_name: &str,
     class_name: &str,
 ) -> Result<u32, ResolveRegclassError> {
-    let obj = parse_object_name(class_name)?;
+    // We use the full parser here because this function needs to accept every legal way
+    // of identifying an object in PG SQL as a valid value for the varchar
+    // literal.  For example: 'foo', 'public.foo', '"my table"', and
+    // '"my schema".foo' must all work as values passed pg_table_size.
+    let obj = Parser::parse_object_name_str(class_name)?;
 
     if obj.0.len() == 1 {
         let class_name = obj.0[0].real_value();
@@ -81,21 +84,6 @@ fn resolve_regclass_inner(
     }
 }
 
-fn parse_object_name(name: &str) -> Result<risingwave_sqlparser::ast::ObjectName, ParserError> {
-    // We use the full parser here because this function needs to accept every legal way
-    // of identifying an object in PG SQL as a valid value for the varchar
-    // literal.  For example: 'foo', 'public.foo', '"my table"', and
-    // '"my schema".foo' must all work as values passed pg_table_size.
-    let mut tokenizer = Tokenizer::new(name);
-    let tokens = tokenizer
-        .tokenize_with_location()
-        .map_err(ParserError::from)?;
-    let mut parser = Parser::new(tokens);
-    let object = parser.parse_object_name()?;
-    parser.expect_token(&Token::EOF)?;
-    Ok(object)
-}
-
 #[function("cast_regclass(varchar) -> int4")]
 fn cast_regclass(class_name: &str) -> Result<i32, ExprError> {
     let oid = resolve_regclass_impl_captured(class_name)?;
diff --git a/src/frontend/src/expr/function_impl/has_privilege.rs b/src/frontend/src/expr/function_impl/has_privilege.rs
new file mode 100644
index 0000000000000..735a9dd1b6d92
--- /dev/null
+++ b/src/frontend/src/expr/function_impl/has_privilege.rs
@@ -0,0 +1,189 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashSet;
+
+use risingwave_expr::{capture_context, function, ExprError, Result};
+use risingwave_pb::user::grant_privilege::{Action, Object};
+use thiserror_ext::AsReport;
+
+use super::context::{CATALOG_READER, DB_NAME, USER_INFO_READER};
+use crate::catalog::CatalogReader;
+use crate::user::user_service::UserInfoReader;
+
+#[inline(always)]
+pub fn user_not_found_err(inner_err: &str) -> ExprError {
+    ExprError::InvalidParam {
+        name: "user",
+        reason: inner_err.into(),
+    }
+}
+
+#[function("has_table_privilege(int4, int4, varchar) -> boolean")]
+fn has_table_privilege(user_id: i32, table_oid: i32, privileges: &str) -> Result<bool> {
+    // does user have privilege for table
+    let user_name = get_user_name_by_id_captured(user_id)?;
+    has_table_privilege_1(user_name.as_str(), table_oid, privileges)
+}
+
+#[function("has_table_privilege(varchar, int4, varchar) -> boolean")]
+fn has_table_privilege_1(user_name: &str, table_oid: i32, privileges: &str) -> Result<bool> {
+    let allowed_actions = HashSet::new();
+    let actions = parse_privilege(privileges, &allowed_actions)?;
+    // currently, we haven't support grant for view.
+    has_privilege_impl_captured(
+        user_name,
+        &get_grant_object_by_oid_captured(table_oid)?,
+        &actions,
+    )
+}
+
+#[function("has_any_column_privilege(int4, int4, varchar) -> boolean")]
+fn has_any_column_privilege(user_id: i32, table_oid: i32, privileges: &str) -> Result<bool> {
+    // does user have privilege for any column of table
+    let user_name = get_user_name_by_id_captured(user_id)?;
+    has_any_column_privilege_1(user_name.as_str(), table_oid, privileges)
+}
+
+#[function("has_any_column_privilege(varchar, int4, varchar) -> boolean")]
+fn has_any_column_privilege_1(user_name: &str, table_oid: i32, privileges: &str) -> Result<bool> {
+    let allowed_actions = HashSet::from_iter([Action::Select, Action::Insert, Action::Update]);
+    let actions = parse_privilege(privileges, &allowed_actions)?;
+    has_privilege_impl_captured(
+        user_name,
+        &get_grant_object_by_oid_captured(table_oid)?,
+        &actions,
+    )
+}
+
+#[function("has_schema_privilege(varchar, int4, varchar) -> boolean")]
+fn has_schema_privilege(user_name: &str, schema_oid: i32, privileges: &str) -> Result<bool> {
+    // does user have privilege for schema
+    let allowed_actions = HashSet::from_iter([Action::Create, Action::Usage]);
+    let actions = parse_privilege(privileges, &allowed_actions)?;
+    has_privilege_impl_captured(user_name, &Object::SchemaId(schema_oid as u32), &actions)
+}
+
+#[function("has_schema_privilege(int4, varchar, varchar) -> boolean")]
+fn has_schema_privilege_1(user_id: i32, schema_name: &str, privileges: &str) -> Result<bool> {
+    let user_name = get_user_name_by_id_captured(user_id)?;
+    let schema_oid = get_schema_id_by_name_captured(schema_name)?;
+    has_schema_privilege(user_name.as_str(), schema_oid, privileges)
+}
+
+#[function("has_schema_privilege(int4, int4, varchar) -> boolean")]
+fn has_schema_privilege_2(user_id: i32, schema_oid: i32, privileges: &str) -> Result<bool> {
+    let user_name = get_user_name_by_id_captured(user_id)?;
+    has_schema_privilege(user_name.as_str(), schema_oid, privileges)
+}
+
+#[function("has_schema_privilege(varchar, varchar, varchar) -> boolean")]
+fn has_schema_privilege_3(user_name: &str, schema_name: &str, privileges: &str) -> Result<bool> {
+    let schema_oid = get_schema_id_by_name_captured(schema_name)?;
+    has_schema_privilege(user_name, schema_oid, privileges)
+}
+
+#[capture_context(USER_INFO_READER)]
+fn has_privilege_impl(
+    user_info_reader: &UserInfoReader,
+    user_name: &str,
+    object: &Object,
+    actions: &Vec<(Action, bool)>,
+) -> Result<bool> {
+    let user_info = &user_info_reader.read_guard();
+    let user_catalog = user_info
+        .get_user_by_name(user_name)
+        .ok_or(user_not_found_err(
+            format!("User {} not found", user_name).as_str(),
+        ))?;
+    Ok(user_catalog.check_privilege_with_grant_option(object, actions))
+}
+
+#[capture_context(USER_INFO_READER)]
+fn get_user_name_by_id(user_info_reader: &UserInfoReader, user_id: i32) -> Result<String> {
+    let user_info = &user_info_reader.read_guard();
+    user_info
+        .get_user_name_by_id(user_id as u32)
+        .ok_or(user_not_found_err(
+            format!("User {} not found", user_id).as_str(),
+        ))
+}
+
+#[capture_context(CATALOG_READER, DB_NAME)]
+fn get_grant_object_by_oid(
+    catalog_reader: &CatalogReader,
+    db_name: &str,
+    oid: i32,
+) -> Result<Object> {
+    catalog_reader
+        .read_guard()
+        .get_database_by_name(db_name)
+        .map_err(|e| ExprError::InvalidParam {
+            name: "oid",
+            reason: e.to_report_string().into(),
+        })?
+        .get_grant_object_by_oid(oid as u32)
+        .ok_or(ExprError::InvalidParam {
+            name: "oid",
+            reason: format!("Table {} not found", oid).as_str().into(),
+        })
+}
+
+#[capture_context(CATALOG_READER, DB_NAME)]
+fn get_schema_id_by_name(
+    catalog_reader: &CatalogReader,
+    db_name: &str,
+    schema_name: &str,
+) -> Result<i32> {
+    let reader = &catalog_reader.read_guard();
+    Ok(reader
+        .get_schema_by_name(db_name, schema_name)
+        .map_err(|e| ExprError::InvalidParam {
+            name: "schema",
+            reason: e.to_report_string().into(),
+        })?
+        .id() as i32)
+}
+
+fn parse_privilege(
+    privilege_string: &str,
+    allowed_actions: &HashSet<Action>,
+) -> Result<Vec<(Action, bool)>> {
+    let mut privileges = Vec::new();
+    for part in privilege_string.split(',').map(str::trim) {
+        let (privilege_type, grant_option) = match part.rsplit_once(" WITH GRANT OPTION") {
+            Some((p, _)) => (p, true),
+            None => (part, false),
+        };
+        match Action::from_str_name(privilege_type.to_uppercase().as_str()) {
+            Some(Action::Unspecified) | None => {
+                return Err(ExprError::InvalidParam {
+                    name: "privilege",
+                    reason: format!("unrecognized privilege type: \"{}\"", part).into(),
+                })
+            }
+            Some(action) => {
+                if allowed_actions.is_empty() || allowed_actions.contains(&action) {
+                    privileges.push((action, grant_option))
+                } else {
+                    return Err(ExprError::InvalidParam {
+                        name: "privilege",
+                        reason: format!("unrecognized privilege type: \"{}\"", part).into(),
+                    });
+                }
+            }
+        }
+    }
+    Ok(privileges)
+}
diff --git a/src/frontend/src/expr/function_impl/mod.rs b/src/frontend/src/expr/function_impl/mod.rs
index ad0b3b7a853d8..3a70ebf5db474 100644
--- a/src/frontend/src/expr/function_impl/mod.rs
+++ b/src/frontend/src/expr/function_impl/mod.rs
@@ -15,6 +15,7 @@
 mod cast_regclass;
 mod col_description;
 pub mod context;
+mod has_privilege;
 mod pg_get_indexdef;
 mod pg_get_userbyid;
 mod pg_get_viewdef;
diff --git a/src/frontend/src/expr/mod.rs b/src/frontend/src/expr/mod.rs
index 893ae425b8513..03be40f955d79 100644
--- a/src/frontend/src/expr/mod.rs
+++ b/src/frontend/src/expr/mod.rs
@@ -647,10 +647,11 @@ impl ExprImpl {
                 fn is_short_circuit(&self, func_call: &FunctionCall) -> bool {
                     /// evaluate the first parameter of `Or` or `And` function call
                     fn eval_first(e: &ExprImpl, expect: bool) -> bool {
-                        let Some(Ok(Some(scalar))) = e.try_fold_const() else {
-                            return false;
-                        };
-                        scalar == ScalarImpl::Bool(expect)
+                        if let ExprImpl::Literal(l) = e {
+                            *l.get_data() == Some(ScalarImpl::Bool(expect))
+                        } else {
+                            false
+                        }
                     }
 
                     match func_call.func_type {
diff --git a/src/frontend/src/expr/pure.rs b/src/frontend/src/expr/pure.rs
index bc18959e5be55..d03b7507fdfb1 100644
--- a/src/frontend/src/expr/pure.rs
+++ b/src/frontend/src/expr/pure.rs
@@ -209,6 +209,7 @@ impl ExprVisitor for ImpureAnalyzer {
             | Type::JsonbPathMatch
             | Type::JsonbPathQueryArray
             | Type::JsonbPathQueryFirst
+            | Type::JsonbSet
             | Type::IsJson
             | Type::ToJsonb
             | Type::Sind
@@ -245,7 +246,9 @@ impl ExprVisitor for ImpureAnalyzer {
             | Type::ConvertTo
             | Type::IcebergTransform
             | Type::InetNtoa
-            | Type::InetAton =>
+            | Type::InetAton
+            | Type::QuoteLiteral
+            | Type::QuoteNullable =>
             // expression output is deterministic(same result for the same input)
             {
                 func_call
@@ -267,6 +270,9 @@ impl ExprVisitor for ImpureAnalyzer {
             | Type::PgIndexesSize
             | Type::PgRelationSize
             | Type::PgGetSerialSequence
+            | Type::HasTablePrivilege
+            | Type::HasAnyColumnPrivilege
+            | Type::HasSchemaPrivilege
             | Type::MakeTimestamptz => self.impure = true,
         }
     }
diff --git a/src/frontend/src/expr/table_function.rs b/src/frontend/src/expr/table_function.rs
index 4703112bc16d3..0db14d4736c2d 100644
--- a/src/frontend/src/expr/table_function.rs
+++ b/src/frontend/src/expr/table_function.rs
@@ -17,9 +17,7 @@ use std::sync::Arc;
 use itertools::Itertools;
 use risingwave_common::types::DataType;
 pub use risingwave_pb::expr::table_function::PbType as TableFunctionType;
-use risingwave_pb::expr::{
-    TableFunction as TableFunctionPb, UserDefinedTableFunction as UserDefinedTableFunctionPb,
-};
+use risingwave_pb::expr::PbTableFunction;
 
 use super::{infer_type, Expr, ExprImpl, ExprRewriter, RwResult};
 use crate::catalog::function_catalog::{FunctionCatalog, FunctionKind};
@@ -35,7 +33,7 @@ pub struct TableFunction {
     pub return_type: DataType,
     pub function_type: TableFunctionType,
     /// Catalog of user defined table function.
-    pub udtf_catalog: Option<Arc<FunctionCatalog>>,
+    pub user_defined: Option<Arc<FunctionCatalog>>,
 }
 
 impl TableFunction {
@@ -47,7 +45,7 @@ impl TableFunction {
             args,
             return_type,
             function_type: func_type,
-            udtf_catalog: None,
+            user_defined: None,
         })
     }
 
@@ -59,37 +57,24 @@ impl TableFunction {
         TableFunction {
             args,
             return_type: catalog.return_type.clone(),
-            function_type: TableFunctionType::Udtf,
-            udtf_catalog: Some(catalog),
+            function_type: TableFunctionType::UserDefined,
+            user_defined: Some(catalog),
         }
     }
 
-    pub fn to_protobuf(&self) -> TableFunctionPb {
-        TableFunctionPb {
+    pub fn to_protobuf(&self) -> PbTableFunction {
+        PbTableFunction {
             function_type: self.function_type as i32,
             args: self.args.iter().map(|c| c.to_expr_proto()).collect_vec(),
             return_type: Some(self.return_type.to_protobuf()),
-            udtf: self
-                .udtf_catalog
-                .as_ref()
-                .map(|c| UserDefinedTableFunctionPb {
-                    arg_names: c.arg_names.clone(),
-                    arg_types: c.arg_types.iter().map(|t| t.to_protobuf()).collect(),
-                    language: c.language.clone(),
-                    link: c.link.clone(),
-                    identifier: c.identifier.clone(),
-                    body: c.body.clone(),
-                    compressed_binary: c.compressed_binary.clone(),
-                    function_type: c.function_type.clone(),
-                    runtime: c.runtime.clone(),
-                }),
+            udf: self.user_defined.as_ref().map(|c| c.as_ref().into()),
         }
     }
 
     /// Get the name of the table function.
     pub fn name(&self) -> String {
         match self.function_type {
-            TableFunctionType::Udtf => self.udtf_catalog.as_ref().unwrap().name.clone(),
+            TableFunctionType::UserDefined => self.user_defined.as_ref().unwrap().name.clone(),
             t => t.as_str_name().to_lowercase(),
         }
     }
diff --git a/src/frontend/src/handler/alter_parallelism.rs b/src/frontend/src/handler/alter_parallelism.rs
index b21e36481e20e..5f0155e9dd46a 100644
--- a/src/frontend/src/handler/alter_parallelism.rs
+++ b/src/frontend/src/handler/alter_parallelism.rs
@@ -85,13 +85,6 @@ pub async fn handle_alter_parallelism(
                 session.check_privilege_for_drop_alter(schema_name, &**sink)?;
                 sink.id.sink_id()
             }
-            StatementType::ALTER_SUBSCRIPTION => {
-                let (subscription, schema_name) =
-                    reader.get_subscription_by_name(db_name, schema_path, &real_table_name)?;
-
-                session.check_privilege_for_drop_alter(schema_name, &**subscription)?;
-                subscription.id.subscription_id()
-            }
             _ => bail!(
                 "invalid statement type for alter parallelism: {:?}",
                 stmt_type
diff --git a/src/frontend/src/handler/alter_source_with_sr.rs b/src/frontend/src/handler/alter_source_with_sr.rs
index b15e9ba3e9867..c72cf547365d7 100644
--- a/src/frontend/src/handler/alter_source_with_sr.rs
+++ b/src/frontend/src/handler/alter_source_with_sr.rs
@@ -64,6 +64,7 @@ fn encode_type_to_encode(from: EncodeType) -> Option<Encode> {
         EncodeType::Bytes => Encode::Bytes,
         EncodeType::Template => Encode::Template,
         EncodeType::None => Encode::None,
+        EncodeType::Text => Encode::Text,
     })
 }
 
@@ -147,11 +148,7 @@ pub async fn refresh_sr_and_get_columns_diff(
     connector_schema: &ConnectorSchema,
     session: &Arc<SessionImpl>,
 ) -> Result<(StreamSourceInfo, Vec<ColumnCatalog>, Vec<ColumnCatalog>)> {
-    let mut with_properties = original_source
-        .with_properties
-        .clone()
-        .into_iter()
-        .collect();
+    let mut with_properties = original_source.with_properties.clone();
     validate_compatibility(connector_schema, &mut with_properties)?;
 
     if with_properties.is_cdc_connector() {
diff --git a/src/frontend/src/handler/alter_streaming_rate_limit.rs b/src/frontend/src/handler/alter_streaming_rate_limit.rs
new file mode 100644
index 0000000000000..4316763628779
--- /dev/null
+++ b/src/frontend/src/handler/alter_streaming_rate_limit.rs
@@ -0,0 +1,89 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use pgwire::pg_response::{PgResponse, StatementType};
+use risingwave_common::bail;
+use risingwave_pb::meta::PbThrottleTarget;
+use risingwave_sqlparser::ast::ObjectName;
+
+use super::{HandlerArgs, RwPgResponse};
+use crate::catalog::root_catalog::SchemaPath;
+use crate::catalog::table_catalog::TableType;
+use crate::error::{ErrorCode, Result};
+use crate::Binder;
+
+pub async fn handle_alter_streaming_rate_limit(
+    handler_args: HandlerArgs,
+    kind: PbThrottleTarget,
+    table_name: ObjectName,
+    rate_limit: i32,
+) -> Result<RwPgResponse> {
+    let session = handler_args.session;
+    let db_name = session.database();
+    let (schema_name, real_table_name) =
+        Binder::resolve_schema_qualified_name(db_name, table_name.clone())?;
+    let search_path = session.config().search_path();
+    let user_name = &session.auth_context().user_name;
+
+    let schema_path = SchemaPath::new(schema_name.as_deref(), &search_path, user_name);
+
+    let (stmt_type, id) = match kind {
+        PbThrottleTarget::Mv => {
+            let reader = session.env().catalog_reader().read_guard();
+            let (table, schema_name) =
+                reader.get_table_by_name(db_name, schema_path, &real_table_name)?;
+            if table.table_type != TableType::MaterializedView {
+                return Err(ErrorCode::InvalidInputSyntax(format!(
+                    "\"{table_name}\" is not a materialized view",
+                ))
+                .into());
+            }
+            session.check_privilege_for_drop_alter(schema_name, &**table)?;
+            (StatementType::ALTER_MATERIALIZED_VIEW, table.id.table_id)
+        }
+        PbThrottleTarget::Source => {
+            let reader = session.env().catalog_reader().read_guard();
+            let (source, schema_name) =
+                reader.get_source_by_name(db_name, schema_path, &real_table_name)?;
+            session.check_privilege_for_drop_alter(schema_name, &**source)?;
+            (StatementType::ALTER_SOURCE, source.id)
+        }
+        PbThrottleTarget::TableWithSource => {
+            let reader = session.env().catalog_reader().read_guard();
+            let (table, schema_name) =
+                reader.get_table_by_name(db_name, schema_path, &real_table_name)?;
+            session.check_privilege_for_drop_alter(schema_name, &**table)?;
+            // Get the corresponding source catalog.
+            let source_id = if let Some(id) = table.associated_source_id {
+                id.table_id()
+            } else {
+                bail!("ALTER STREAMING_RATE_LIMIT is not for table without source")
+            };
+            (StatementType::ALTER_SOURCE, source_id)
+        }
+        _ => bail!("Unsupported throttle target: {:?}", kind),
+    };
+
+    let meta_client = session.env().meta_client();
+
+    let rate_limit = if rate_limit < 0 {
+        None
+    } else {
+        Some(rate_limit as u32)
+    };
+
+    meta_client.apply_throttle(kind, id, rate_limit).await?;
+
+    Ok(PgResponse::empty_result(stmt_type))
+}
diff --git a/src/frontend/src/handler/create_aggregate.rs b/src/frontend/src/handler/create_aggregate.rs
new file mode 100644
index 0000000000000..b9b8a391eaffe
--- /dev/null
+++ b/src/frontend/src/handler/create_aggregate.rs
@@ -0,0 +1,141 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::Context;
+use risingwave_common::catalog::FunctionId;
+use risingwave_expr::sig::{CreateFunctionOptions, UdfKind};
+use risingwave_pb::catalog::function::{AggregateFunction, Kind};
+use risingwave_pb::catalog::Function;
+use risingwave_sqlparser::ast::DataType as AstDataType;
+
+use super::*;
+use crate::catalog::CatalogError;
+use crate::{bind_data_type, Binder};
+
+pub async fn handle_create_aggregate(
+    handler_args: HandlerArgs,
+    or_replace: bool,
+    name: ObjectName,
+    args: Vec<OperateFunctionArg>,
+    returns: AstDataType,
+    params: CreateFunctionBody,
+) -> Result<RwPgResponse> {
+    if or_replace {
+        bail_not_implemented!("CREATE OR REPLACE AGGREGATE");
+    }
+    // e.g., `language [ python / java / ...etc]`
+    let language = match params.language {
+        Some(lang) => {
+            let lang = lang.real_value().to_lowercase();
+            match &*lang {
+                "python" | "javascript" => lang,
+                _ => {
+                    return Err(ErrorCode::InvalidParameterValue(format!(
+                        "language {} is not supported",
+                        lang
+                    ))
+                    .into())
+                }
+            }
+        }
+        None => return Err(ErrorCode::InvalidParameterValue("no language".into()).into()),
+    };
+
+    let return_type = bind_data_type(&returns)?;
+
+    let mut arg_names = vec![];
+    let mut arg_types = vec![];
+    for arg in args {
+        arg_names.push(arg.name.map_or("".to_string(), |n| n.real_value()));
+        arg_types.push(bind_data_type(&arg.data_type)?);
+    }
+
+    // resolve database and schema id
+    let session = &handler_args.session;
+    let db_name = session.database();
+    let (schema_name, function_name) = Binder::resolve_schema_qualified_name(db_name, name)?;
+    let (database_id, schema_id) = session.get_database_and_schema_id_for_create(schema_name)?;
+
+    // check if the function exists in the catalog
+    if (session.env().catalog_reader().read_guard())
+        .get_schema_by_id(&database_id, &schema_id)?
+        .get_function_by_name_args(&function_name, &arg_types)
+        .is_some()
+    {
+        let name = format!(
+            "{function_name}({})",
+            arg_types.iter().map(|t| t.to_string()).join(",")
+        );
+        return Err(CatalogError::Duplicated("function", name).into());
+    }
+
+    let link = match &params.using {
+        Some(CreateFunctionUsing::Link(l)) => Some(l.as_str()),
+        _ => None,
+    };
+    let base64_decoded = match &params.using {
+        Some(CreateFunctionUsing::Base64(encoded)) => {
+            use base64::prelude::{Engine, BASE64_STANDARD};
+            let bytes = BASE64_STANDARD
+                .decode(encoded)
+                .context("invalid base64 encoding")?;
+            Some(bytes)
+        }
+        _ => None,
+    };
+    let function_type = match params.function_type {
+        Some(CreateFunctionType::Sync) => Some("sync".to_string()),
+        Some(CreateFunctionType::Async) => Some("async".to_string()),
+        Some(CreateFunctionType::Generator) => Some("generator".to_string()),
+        Some(CreateFunctionType::AsyncGenerator) => Some("async_generator".to_string()),
+        None => None,
+    };
+
+    let create_fn = risingwave_expr::sig::find_udf_impl(&language, None, link)?.create_fn;
+    let output = create_fn(CreateFunctionOptions {
+        kind: UdfKind::Aggregate,
+        name: &function_name,
+        arg_names: &arg_names,
+        arg_types: &arg_types,
+        return_type: &return_type,
+        as_: params.as_.as_ref().map(|s| s.as_str()),
+        using_link: link,
+        using_base64_decoded: base64_decoded.as_deref(),
+    })?;
+
+    let function = Function {
+        id: FunctionId::placeholder().0,
+        schema_id,
+        database_id,
+        name: function_name,
+        kind: Some(Kind::Aggregate(AggregateFunction {})),
+        arg_names,
+        arg_types: arg_types.into_iter().map(|t| t.into()).collect(),
+        return_type: Some(return_type.into()),
+        language,
+        identifier: Some(output.identifier),
+        link: link.map(|s| s.to_string()),
+        body: output.body,
+        compressed_binary: output.compressed_binary,
+        owner: session.user_id(),
+        always_retry_on_network_error: false,
+        runtime: None,
+        function_type,
+    };
+
+    let catalog_writer = session.catalog_writer()?;
+    catalog_writer.create_function(function).await?;
+
+    Ok(PgResponse::empty_result(StatementType::CREATE_AGGREGATE))
+}
diff --git a/src/frontend/src/handler/create_connection.rs b/src/frontend/src/handler/create_connection.rs
index 22491f9cb0ee3..987f0e9fdd897 100644
--- a/src/frontend/src/handler/create_connection.rs
+++ b/src/frontend/src/handler/create_connection.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 
 use pgwire::pg_response::{PgResponse, StatementType};
 use risingwave_connector::source::kafka::PRIVATELINK_CONNECTION;
@@ -36,7 +36,7 @@ pub(crate) const CLOUD_PROVIDER_AWS: &str = "aws";
 
 #[inline(always)]
 fn get_connection_property_required(
-    with_properties: &HashMap<String, String>,
+    with_properties: &BTreeMap<String, String>,
     property: &str,
 ) -> Result<String> {
     with_properties
@@ -50,7 +50,7 @@ fn get_connection_property_required(
 }
 
 fn resolve_private_link_properties(
-    with_properties: &HashMap<String, String>,
+    with_properties: &BTreeMap<String, String>,
 ) -> Result<create_connection_request::PrivateLink> {
     let provider =
         match get_connection_property_required(with_properties, CONNECTION_PROVIDER_PROP)?.as_str()
@@ -86,7 +86,7 @@ fn resolve_private_link_properties(
 }
 
 fn resolve_create_connection_payload(
-    with_properties: &HashMap<String, String>,
+    with_properties: &BTreeMap<String, String>,
 ) -> Result<create_connection_request::Payload> {
     let connection_type = get_connection_property_required(with_properties, CONNECTION_TYPE_PROP)?;
     let create_connection_payload = match connection_type.as_str() {
diff --git a/src/frontend/src/handler/create_function.rs b/src/frontend/src/handler/create_function.rs
index 428a5612e1770..ccd83a13ed81c 100644
--- a/src/frontend/src/handler/create_function.rs
+++ b/src/frontend/src/handler/create_function.rs
@@ -12,18 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use anyhow::{anyhow, Context};
-use arrow_schema::Fields;
-use bytes::Bytes;
-use itertools::Itertools;
-use pgwire::pg_response::StatementType;
+use anyhow::Context;
 use risingwave_common::catalog::FunctionId;
 use risingwave_common::types::DataType;
-use risingwave_expr::expr::get_or_create_wasm_runtime;
+use risingwave_expr::sig::{CreateFunctionOptions, UdfKind};
 use risingwave_pb::catalog::function::{Kind, ScalarFunction, TableFunction};
 use risingwave_pb::catalog::Function;
-use risingwave_sqlparser::ast::{CreateFunctionBody, ObjectName, OperateFunctionArg};
-use risingwave_udf::ArrowFlightUdfClient;
 
 use super::*;
 use crate::catalog::CatalogError;
@@ -65,10 +59,10 @@ pub async fn handle_create_function(
         None => "".to_string(),
     };
 
-    let rt = match params.runtime {
+    let runtime = match params.runtime {
         Some(runtime) => {
-            if language.as_str() == "javascript" {
-                runtime.to_string()
+            if language == "javascript" {
+                Some(runtime.real_value())
             } else {
                 return Err(ErrorCode::InvalidParameterValue(
                     "runtime is only supported for javascript".to_string(),
@@ -76,7 +70,7 @@ pub async fn handle_create_function(
                 .into());
             }
         }
-        None => "".to_string(),
+        None => None,
     };
 
     let return_type;
@@ -133,194 +127,45 @@ pub async fn handle_create_function(
         return Err(CatalogError::Duplicated("function", name).into());
     }
 
-    let identifier;
-    let mut link = None;
-    let mut body = None;
-    let mut compressed_binary = None;
-    let mut function_type = None;
-    let mut runtime = None;
-
-    match language.as_str() {
-        "python" if params.using.is_none() => {
-            identifier = function_name.to_string();
-            body = Some(
-                params
-                    .as_
-                    .ok_or_else(|| ErrorCode::InvalidParameterValue("AS must be specified".into()))?
-                    .into_string(),
-            );
-        }
-        "python" | "java" | "" => {
-            let Some(CreateFunctionUsing::Link(l)) = params.using else {
-                return Err(ErrorCode::InvalidParameterValue(
-                    "USING LINK must be specified".to_string(),
-                )
-                .into());
-            };
-            let Some(as_) = params.as_ else {
-                return Err(
-                    ErrorCode::InvalidParameterValue("AS must be specified".to_string()).into(),
-                );
-            };
-            identifier = as_.into_string();
-
-            // check UDF server
-            {
-                let client = ArrowFlightUdfClient::connect(&l)
-                    .await
-                    .map_err(|e| anyhow!(e))?;
-                /// A helper function to create a unnamed field from data type.
-                fn to_field(data_type: arrow_schema::DataType) -> arrow_schema::Field {
-                    arrow_schema::Field::new("", data_type, true)
-                }
-                let args = arrow_schema::Schema::new(
-                    arg_types
-                        .iter()
-                        .map::<Result<_>, _>(|t| Ok(to_field(t.try_into()?)))
-                        .try_collect::<_, Fields, _>()?,
-                );
-                let returns = arrow_schema::Schema::new(match kind {
-                    Kind::Scalar(_) => vec![to_field(return_type.clone().try_into()?)],
-                    Kind::Table(_) => vec![
-                        arrow_schema::Field::new("row_index", arrow_schema::DataType::Int32, true),
-                        to_field(return_type.clone().try_into()?),
-                    ],
-                    _ => unreachable!(),
-                });
-                client
-                    .check(&identifier, &args, &returns)
-                    .await
-                    .context("failed to check UDF signature")?;
-            }
-            link = Some(l);
-        }
-        "javascript" if rt.as_str() != "deno" => {
-            identifier = function_name.to_string();
-            body = Some(
-                params
-                    .as_
-                    .ok_or_else(|| ErrorCode::InvalidParameterValue("AS must be specified".into()))?
-                    .into_string(),
-            );
-            runtime = Some("quickjs".to_string());
-        }
-        "javascript" if rt.as_str() == "deno" => {
-            identifier = function_name.to_string();
-            match (params.using, params.as_) {
-                (None, None) => {
-                    return Err(ErrorCode::InvalidParameterValue(
-                        "Either USING or AS must be specified".into(),
-                    )
-                    .into())
-                }
-                (None, Some(_as)) => body = Some(_as.into_string()),
-                (Some(CreateFunctionUsing::Link(link)), None) => {
-                    let bytes = download_code_from_link(&link).await?;
-                    compressed_binary = Some(zstd::stream::encode_all(bytes.as_slice(), 0)?);
-                }
-                (Some(CreateFunctionUsing::Base64(encoded)), None) => {
-                    use base64::prelude::{Engine, BASE64_STANDARD};
-                    let bytes = BASE64_STANDARD
-                        .decode(encoded)
-                        .context("invalid base64 encoding")?;
-                    compressed_binary = Some(zstd::stream::encode_all(bytes.as_slice(), 0)?);
-                }
-                (Some(_), Some(_)) => {
-                    return Err(ErrorCode::InvalidParameterValue(
-                        "Both USING and AS cannot be specified".into(),
-                    )
-                    .into())
-                }
-            };
-
-            function_type = match params.function_type {
-                Some(CreateFunctionType::Sync) => Some("sync".to_string()),
-                Some(CreateFunctionType::Async) => Some("async".to_string()),
-                Some(CreateFunctionType::Generator) => Some("generator".to_string()),
-                Some(CreateFunctionType::AsyncGenerator) => Some("async_generator".to_string()),
-                None => None,
-            };
-
-            runtime = Some("deno".to_string());
-        }
-        "rust" => {
-            if params.using.is_some() {
-                return Err(ErrorCode::InvalidParameterValue(
-                    "USING is not supported for rust function".to_string(),
-                )
-                .into());
-            }
-            let identifier_v1 = wasm_identifier_v1(
-                &function_name,
-                &arg_types,
-                &return_type,
-                matches!(kind, Kind::Table(_)),
-            );
-            // if the function returns a struct, users need to add `#[function]` macro by themselves.
-            // otherwise, we add it automatically. the code should start with `fn ...`.
-            let function_macro = if return_type.is_struct() {
-                String::new()
-            } else {
-                format!("#[function(\"{}\")]", identifier_v1)
-            };
-            let script = params
-                .as_
-                .ok_or_else(|| ErrorCode::InvalidParameterValue("AS must be specified".into()))?
-                .into_string();
-            let script = format!(
-                "use arrow_udf::{{function, types::*}};\n{}\n{}",
-                function_macro, script
-            );
-            body = Some(script.clone());
-
-            let wasm_binary = tokio::task::spawn_blocking(move || {
-                let mut opts = arrow_udf_wasm::build::BuildOpts::default();
-                opts.script = script;
-                // use a fixed tempdir to reuse the build cache
-                opts.tempdir = Some(std::env::temp_dir().join("risingwave-rust-udf"));
-
-                arrow_udf_wasm::build::build_with(&opts)
-            })
-            .await?
-            .context("failed to build rust function")?;
-
-            let runtime = get_or_create_wasm_runtime(&wasm_binary)?;
-            identifier = find_wasm_identifier_v2(&runtime, &identifier_v1)?;
-
-            compressed_binary = Some(zstd::stream::encode_all(wasm_binary.as_slice(), 0)?);
-        }
-        "wasm" => {
-            let Some(using) = params.using else {
-                return Err(ErrorCode::InvalidParameterValue(
-                    "USING must be specified".to_string(),
-                )
-                .into());
-            };
-            let wasm_binary = match using {
-                CreateFunctionUsing::Link(link) => download_binary_from_link(&link).await?,
-                CreateFunctionUsing::Base64(encoded) => {
-                    // decode wasm binary from base64
-                    use base64::prelude::{Engine, BASE64_STANDARD};
-                    BASE64_STANDARD
-                        .decode(encoded)
-                        .context("invalid base64 encoding")?
-                        .into()
-                }
-            };
-            let runtime = get_or_create_wasm_runtime(&wasm_binary)?;
-            let identifier_v1 = wasm_identifier_v1(
-                &function_name,
-                &arg_types,
-                &return_type,
-                matches!(kind, Kind::Table(_)),
-            );
-            identifier = find_wasm_identifier_v2(&runtime, &identifier_v1)?;
-
-            compressed_binary = Some(zstd::stream::encode_all(wasm_binary.as_ref(), 0)?);
+    let link = match &params.using {
+        Some(CreateFunctionUsing::Link(l)) => Some(l.as_str()),
+        _ => None,
+    };
+    let base64_decoded = match &params.using {
+        Some(CreateFunctionUsing::Base64(encoded)) => {
+            use base64::prelude::{Engine, BASE64_STANDARD};
+            let bytes = BASE64_STANDARD
+                .decode(encoded)
+                .context("invalid base64 encoding")?;
+            Some(bytes)
         }
-        _ => unreachable!("invalid language: {language}"),
+        _ => None,
+    };
+    let function_type = match params.function_type {
+        Some(CreateFunctionType::Sync) => Some("sync".to_string()),
+        Some(CreateFunctionType::Async) => Some("async".to_string()),
+        Some(CreateFunctionType::Generator) => Some("generator".to_string()),
+        Some(CreateFunctionType::AsyncGenerator) => Some("async_generator".to_string()),
+        None => None,
     };
 
+    let create_fn =
+        risingwave_expr::sig::find_udf_impl(&language, runtime.as_deref(), link)?.create_fn;
+    let output = create_fn(CreateFunctionOptions {
+        kind: match kind {
+            Kind::Scalar(_) => UdfKind::Scalar,
+            Kind::Table(_) => UdfKind::Table,
+            Kind::Aggregate(_) => unreachable!(),
+        },
+        name: &function_name,
+        arg_names: &arg_names,
+        arg_types: &arg_types,
+        return_type: &return_type,
+        as_: params.as_.as_ref().map(|s| s.as_str()),
+        using_link: link,
+        using_base64_decoded: base64_decoded.as_deref(),
+    })?;
+
     let function = Function {
         id: FunctionId::placeholder().0,
         schema_id,
@@ -331,10 +176,10 @@ pub async fn handle_create_function(
         arg_types: arg_types.into_iter().map(|t| t.into()).collect(),
         return_type: Some(return_type.into()),
         language,
-        identifier: Some(identifier),
-        link,
-        body,
-        compressed_binary,
+        identifier: Some(output.identifier),
+        link: link.map(|s| s.to_string()),
+        body: output.body,
+        compressed_binary: output.compressed_binary,
         owner: session.user_id(),
         always_retry_on_network_error: with_options
             .always_retry_on_network_error
@@ -348,136 +193,3 @@ pub async fn handle_create_function(
 
     Ok(PgResponse::empty_result(StatementType::CREATE_FUNCTION))
 }
-
-/// Download wasm binary from a link.
-#[allow(clippy::unused_async)]
-async fn download_binary_from_link(link: &str) -> Result<Bytes> {
-    // currently only local file system is supported
-    if let Some(path) = link.strip_prefix("fs://") {
-        let content =
-            std::fs::read(path).context("failed to read wasm binary from local file system")?;
-        Ok(content.into())
-    } else {
-        Err(ErrorCode::InvalidParameterValue("only 'fs://' is supported".to_string()).into())
-    }
-}
-
-/// Convert a v0.1 function identifier to v0.2 format.
-///
-/// In arrow-udf v0.1 format, struct type is inline in the identifier. e.g.
-///
-/// ```text
-/// keyvalue(varchar,varchar)->struct<key:varchar,value:varchar>
-/// ```
-///
-/// However, since arrow-udf v0.2, struct type is no longer inline.
-/// The above identifier is divided into a function and a type.
-///
-/// ```text
-/// keyvalue(varchar,varchar)->struct KeyValue
-/// KeyValue=key:varchar,value:varchar
-/// ```
-///
-/// For compatibility, we should call `find_wasm_identifier_v2` to
-/// convert v0.1 identifiers to v0.2 format before looking up the function.
-fn find_wasm_identifier_v2(
-    runtime: &arrow_udf_wasm::Runtime,
-    inlined_signature: &str,
-) -> Result<String> {
-    // Inline types in function signature.
-    //
-    // # Example
-    //
-    // ```text
-    // types = { "KeyValue": "key:varchar,value:varchar" }
-    // input = "keyvalue(varchar, varchar) -> struct KeyValue"
-    // output = "keyvalue(varchar, varchar) -> struct<key:varchar,value:varchar>"
-    // ```
-    let inline_types = |s: &str| -> String {
-        let mut inlined = s.to_string();
-        // iteratively replace `struct Xxx` with `struct<...>` until no replacement is made.
-        loop {
-            let replaced = inlined.clone();
-            for (k, v) in runtime.types() {
-                inlined = inlined.replace(&format!("struct {k}"), &format!("struct<{v}>"));
-            }
-            if replaced == inlined {
-                return inlined;
-            }
-        }
-    };
-    // Function signature in arrow-udf is case sensitive.
-    // However, SQL identifiers are usually case insensitive and stored in lowercase.
-    // So we should convert the signature to lowercase before comparison.
-    let identifier = runtime
-        .functions()
-        .find(|f| inline_types(f).to_lowercase() == inlined_signature)
-        .ok_or_else(|| {
-            ErrorCode::InvalidParameterValue(format!(
-                "function not found in wasm binary: \"{}\"\nHINT: available functions:\n  {}\navailable types:\n  {}",
-                inlined_signature,
-                runtime.functions().join("\n  "),
-                runtime.types().map(|(k, v)| format!("{k}: {v}")).join("\n  "),
-            ))
-        })?;
-    Ok(identifier.into())
-}
-
-/// Download wasm binary from a link.
-#[allow(clippy::unused_async)]
-async fn download_code_from_link(link: &str) -> Result<Vec<u8>> {
-    // currently only local file system is supported
-    if let Some(path) = link.strip_prefix("fs://") {
-        let content =
-            std::fs::read(path).context("failed to read the code from local file system")?;
-        Ok(content)
-    } else {
-        Err(ErrorCode::InvalidParameterValue("only 'fs://' is supported".to_string()).into())
-    }
-}
-
-/// Generate a function identifier in v0.1 format from the function signature.
-fn wasm_identifier_v1(
-    name: &str,
-    args: &[DataType],
-    ret: &DataType,
-    table_function: bool,
-) -> String {
-    format!(
-        "{}({}){}{}",
-        name,
-        args.iter().map(datatype_name).join(","),
-        if table_function { "->>" } else { "->" },
-        datatype_name(ret)
-    )
-}
-
-/// Convert a data type to string used in identifier.
-fn datatype_name(ty: &DataType) -> String {
-    match ty {
-        DataType::Boolean => "boolean".to_string(),
-        DataType::Int16 => "int2".to_string(),
-        DataType::Int32 => "int4".to_string(),
-        DataType::Int64 => "int8".to_string(),
-        DataType::Float32 => "float4".to_string(),
-        DataType::Float64 => "float8".to_string(),
-        DataType::Date => "date".to_string(),
-        DataType::Time => "time".to_string(),
-        DataType::Timestamp => "timestamp".to_string(),
-        DataType::Timestamptz => "timestamptz".to_string(),
-        DataType::Interval => "interval".to_string(),
-        DataType::Decimal => "decimal".to_string(),
-        DataType::Jsonb => "json".to_string(),
-        DataType::Serial => "serial".to_string(),
-        DataType::Int256 => "int256".to_string(),
-        DataType::Bytea => "bytea".to_string(),
-        DataType::Varchar => "varchar".to_string(),
-        DataType::List(inner) => format!("{}[]", datatype_name(inner)),
-        DataType::Struct(s) => format!(
-            "struct<{}>",
-            s.iter()
-                .map(|(name, ty)| format!("{}:{}", name, datatype_name(ty)))
-                .join(",")
-        ),
-    }
-}
diff --git a/src/frontend/src/handler/create_index.rs b/src/frontend/src/handler/create_index.rs
index aeca32253f2c7..6e9793646c7e7 100644
--- a/src/frontend/src/handler/create_index.rs
+++ b/src/frontend/src/handler/create_index.rs
@@ -382,7 +382,7 @@ fn assemble_materialize(
         }))
         .collect_vec();
 
-    PlanRoot::new(
+    PlanRoot::new_with_logical_plan(
         logical_project,
         // schema of logical_project is such that index columns come first.
         // so we can use distributed_by_columns_len to represent distributed by columns indices.
diff --git a/src/frontend/src/handler/create_mv.rs b/src/frontend/src/handler/create_mv.rs
index 94ff4eebbc66e..8e6882e6bb4a5 100644
--- a/src/frontend/src/handler/create_mv.rs
+++ b/src/frontend/src/handler/create_mv.rs
@@ -16,9 +16,8 @@ use either::Either;
 use itertools::Itertools;
 use pgwire::pg_response::{PgResponse, StatementType};
 use risingwave_common::acl::AclMode;
-use risingwave_pb::catalog::{CreateType, PbTable};
+use risingwave_pb::catalog::PbTable;
 use risingwave_pb::stream_plan::stream_fragment_graph::Parallelism;
-use risingwave_pb::stream_plan::StreamScanType;
 use risingwave_sqlparser::ast::{EmitMode, Ident, ObjectName, Query};
 
 use super::privilege::resolve_relation_privileges;
@@ -37,21 +36,24 @@ use crate::scheduler::streaming_manager::CreatingStreamingJobInfo;
 use crate::session::SessionImpl;
 use crate::stream_fragmenter::build_graph;
 
+pub(super) fn parse_column_names(columns: &[Ident]) -> Option<Vec<String>> {
+    if columns.is_empty() {
+        None
+    } else {
+        Some(columns.iter().map(|v| v.real_value()).collect())
+    }
+}
+
+/// If columns is empty, it means that the user did not specify the column names.
+/// In this case, we extract the column names from the query.
+/// If columns is not empty, it means that user specify the column names and the user
+/// should guarantee that the column names number are consistent with the query.
 pub(super) fn get_column_names(
     bound: &BoundQuery,
     session: &SessionImpl,
     columns: Vec<Ident>,
 ) -> Result<Option<Vec<String>>> {
-    // If columns is empty, it means that the user did not specify the column names.
-    // In this case, we extract the column names from the query.
-    // If columns is not empty, it means that user specify the column names and the user
-    // should guarantee that the column names number are consistent with the query.
-    let col_names: Option<Vec<String>> = if columns.is_empty() {
-        None
-    } else {
-        Some(columns.iter().map(|v| v.real_value()).collect())
-    };
-
+    let col_names = parse_column_names(&columns);
     if let BoundSetExpr::Select(select) = &bound.body {
         // `InputRef`'s alias will be implicitly assigned in `bind_project`.
         // If user provide columns name (col_names.is_some()), we don't need alias.
@@ -164,7 +166,7 @@ pub async fn handle_create_mv(
         return Ok(resp);
     }
 
-    let (mut table, graph, can_run_in_background) = {
+    let (table, graph) = {
         let context = OptimizerContext::from_handler_args(handler_args);
         if !context.with_options().is_empty() {
             // get other useful fields by `remove`, the logic here is to reject unknown options.
@@ -183,21 +185,7 @@ It only indicates the physical clustering of the data, which may improve the per
 
         let (plan, table) =
             gen_create_mv_plan(&session, context.into(), query, name, columns, emit_mode)?;
-        // All leaf nodes must be stream table scan, no other scan operators support recovery.
-        fn plan_has_backfill_leaf_nodes(plan: &PlanRef) -> bool {
-            if plan.inputs().is_empty() {
-                if let Some(scan) = plan.as_stream_table_scan() {
-                    scan.stream_scan_type() == StreamScanType::Backfill
-                        || scan.stream_scan_type() == StreamScanType::ArrangementBackfill
-                } else {
-                    false
-                }
-            } else {
-                assert!(!plan.inputs().is_empty());
-                plan.inputs().iter().all(plan_has_backfill_leaf_nodes)
-            }
-        }
-        let can_run_in_background = plan_has_backfill_leaf_nodes(&plan);
+
         let context = plan.plan_base().ctx().clone();
         let mut graph = build_graph(plan)?;
         graph.parallelism =
@@ -211,7 +199,7 @@ It only indicates the physical clustering of the data, which may improve the per
         let ctx = graph.ctx.as_mut().unwrap();
         ctx.timezone = context.get_session_timezone();
 
-        (table, graph, can_run_in_background)
+        (table, graph)
     };
 
     // Ensure writes to `StreamJobTracker` are atomic.
@@ -226,14 +214,6 @@ It only indicates the physical clustering of the data, which may improve the per
                 table.name.clone(),
             ));
 
-    let run_in_background = session.config().background_ddl();
-    let create_type = if run_in_background && can_run_in_background {
-        CreateType::Background
-    } else {
-        CreateType::Foreground
-    };
-    table.create_type = create_type.into();
-
     let session = session.clone();
     let catalog_writer = session.catalog_writer()?;
     catalog_writer
diff --git a/src/frontend/src/handler/create_schema.rs b/src/frontend/src/handler/create_schema.rs
index 806fbce8af61d..46972a2757e3e 100644
--- a/src/frontend/src/handler/create_schema.rs
+++ b/src/frontend/src/handler/create_schema.rs
@@ -29,6 +29,7 @@ pub async fn handle_create_schema(
     handler_args: HandlerArgs,
     schema_name: ObjectName,
     if_not_exist: bool,
+    user_specified: Option<ObjectName>,
 ) -> Result<RwPgResponse> {
     let session = handler_args.session;
     let database_name = session.database();
@@ -62,6 +63,19 @@ pub async fn handle_create_schema(
         (db.id(), db.owner())
     };
 
+    let schema_owner = if let Some(user_specified) = user_specified {
+        let user_specified = Binder::resolve_user_name(user_specified)?;
+        session
+            .env()
+            .user_info_reader()
+            .read_guard()
+            .get_user_by_name(&user_specified)
+            .map(|u| u.id)
+            .ok_or_else(|| CatalogError::NotFound("user", user_specified.to_string()))?
+    } else {
+        session.user_id()
+    };
+
     session.check_privileges(&[ObjectCheckItem::new(
         db_owner,
         AclMode::Create,
@@ -70,7 +84,7 @@ pub async fn handle_create_schema(
 
     let catalog_writer = session.catalog_writer()?;
     catalog_writer
-        .create_schema(db_id, &schema_name, session.user_id())
+        .create_schema(db_id, &schema_name, schema_owner)
         .await?;
     Ok(PgResponse::empty_result(StatementType::CREATE_SCHEMA))
 }
diff --git a/src/frontend/src/handler/create_secret.rs b/src/frontend/src/handler/create_secret.rs
new file mode 100644
index 0000000000000..2e99f26e97cb8
--- /dev/null
+++ b/src/frontend/src/handler/create_secret.rs
@@ -0,0 +1,102 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use pgwire::pg_response::{PgResponse, StatementType};
+use prost::Message;
+use risingwave_common::bail_not_implemented;
+use risingwave_sqlparser::ast::{CreateSecretStatement, SqlOption, Value};
+
+use crate::error::{ErrorCode, Result};
+use crate::handler::{HandlerArgs, RwPgResponse};
+use crate::{Binder, WithOptions};
+
+const SECRET_BACKEND_KEY: &str = "backend";
+
+const SECRET_BACKEND_META: &str = "meta";
+const SECRET_BACKEND_HASHICORP_VAULT: &str = "hashicorp_vault";
+
+pub async fn handle_create_secret(
+    handler_args: HandlerArgs,
+    stmt: CreateSecretStatement,
+) -> Result<RwPgResponse> {
+    let session = handler_args.session.clone();
+    let db_name = session.database();
+    let (schema_name, connection_name) =
+        Binder::resolve_schema_qualified_name(db_name, stmt.secret_name.clone())?;
+
+    if let Err(e) = session.check_secret_name_duplicated(stmt.secret_name.clone()) {
+        return if stmt.if_not_exists {
+            Ok(PgResponse::builder(StatementType::CREATE_SECRET)
+                .notice(format!("secret \"{}\" exists, skipping", connection_name))
+                .into())
+        } else {
+            Err(e)
+        };
+    }
+
+    // check if the secret backend is supported
+    let with_props = WithOptions::try_from(stmt.with_properties.0.as_ref() as &[SqlOption])?;
+    let secret_payload: Vec<u8> = {
+        if let Some(backend) = with_props.inner().get(SECRET_BACKEND_KEY) {
+            match backend.to_lowercase().as_ref() {
+                SECRET_BACKEND_META => {
+                    let backend = risingwave_pb::secret::Secret {
+                        secret_backend: Some(risingwave_pb::secret::secret::SecretBackend::Meta(
+                            risingwave_pb::secret::SecretMetaBackend { value: vec![] },
+                        )),
+                    };
+                    backend.encode_to_vec()
+                }
+                SECRET_BACKEND_HASHICORP_VAULT => {
+                    if stmt.credential != Value::Null {
+                        return Err(ErrorCode::InvalidParameterValue(
+                            "credential must be null for hashicorp_vault backend".to_string(),
+                        )
+                        .into());
+                    }
+                    bail_not_implemented!("hashicorp_vault backend is not implemented yet")
+                }
+                _ => {
+                    return Err(ErrorCode::InvalidParameterValue(format!(
+                        "secret backend \"{}\" is not supported. Supported backends are: {}",
+                        backend,
+                        [SECRET_BACKEND_META, SECRET_BACKEND_HASHICORP_VAULT].join(",")
+                    ))
+                    .into());
+                }
+            }
+        } else {
+            return Err(ErrorCode::InvalidParameterValue(format!(
+                "secret backend is not specified in with clause. Supported backends are: {}",
+                [SECRET_BACKEND_META, SECRET_BACKEND_HASHICORP_VAULT].join(",")
+            ))
+            .into());
+        }
+    };
+
+    let (database_id, schema_id) = session.get_database_and_schema_id_for_create(schema_name)?;
+
+    let catalog_writer = session.catalog_writer()?;
+    catalog_writer
+        .create_secret(
+            stmt.secret_name.real_value(),
+            database_id,
+            schema_id,
+            session.user_id(),
+            secret_payload,
+        )
+        .await?;
+
+    Ok(PgResponse::empty_result(StatementType::CREATE_SECRET))
+}
diff --git a/src/frontend/src/handler/create_sink.rs b/src/frontend/src/handler/create_sink.rs
index bed409de178f1..f6b6e5f563d2d 100644
--- a/src/frontend/src/handler/create_sink.rs
+++ b/src/frontend/src/handler/create_sink.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::rc::Rc;
 use std::sync::{Arc, LazyLock};
 
@@ -22,9 +22,9 @@ use either::Either;
 use itertools::Itertools;
 use maplit::{convert_args, hashmap};
 use pgwire::pg_response::{PgResponse, StatementType};
-use risingwave_common::catalog::{ConnectionId, DatabaseId, SchemaId, TableId, UserId};
-use risingwave_common::types::{DataType, Datum};
-use risingwave_common::util::value_encoding::DatumFromProtoExt;
+use risingwave_common::array::arrow::{FromArrow, IcebergArrowConvert};
+use risingwave_common::catalog::{ConnectionId, DatabaseId, Schema, SchemaId, TableId, UserId};
+use risingwave_common::types::DataType;
 use risingwave_common::{bail, catalog};
 use risingwave_connector::sink::catalog::{SinkCatalog, SinkFormatDesc, SinkType};
 use risingwave_connector::sink::iceberg::{IcebergConfig, ICEBERG_SINK};
@@ -33,7 +33,6 @@ use risingwave_connector::sink::{
 };
 use risingwave_pb::catalog::{PbSource, Table};
 use risingwave_pb::ddl_service::ReplaceTablePlan;
-use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn;
 use risingwave_pb::stream_plan::stream_fragment_graph::Parallelism;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
 use risingwave_pb::stream_plan::{DispatcherType, MergeNode, StreamFragmentGraph, StreamNode};
@@ -49,9 +48,11 @@ use super::RwPgResponse;
 use crate::binder::Binder;
 use crate::catalog::catalog_service::CatalogReadGuard;
 use crate::catalog::source_catalog::SourceCatalog;
+use crate::catalog::view_catalog::ViewCatalog;
 use crate::error::{ErrorCode, Result, RwError};
-use crate::expr::{ExprImpl, InputRef, Literal};
+use crate::expr::{ExprImpl, InputRef};
 use crate::handler::alter_table_column::fetch_table_catalog_for_alter;
+use crate::handler::create_mv::parse_column_names;
 use crate::handler::create_table::{generate_stream_graph_for_table, ColumnIdGenerator};
 use crate::handler::privilege::resolve_query_privileges;
 use crate::handler::util::SourceSchemaCompatExt;
@@ -63,7 +64,7 @@ use crate::optimizer::{OptimizerContext, OptimizerContextRef, PlanRef, RelationC
 use crate::scheduler::streaming_manager::CreatingStreamingJobInfo;
 use crate::session::SessionImpl;
 use crate::stream_fragmenter::build_graph;
-use crate::utils::resolve_privatelink_in_with_option;
+use crate::utils::{resolve_privatelink_in_with_option, resolve_secret_in_with_options};
 use crate::{Explain, Planner, TableCatalog, WithOptions};
 
 // used to store result of `gen_sink_plan`
@@ -80,6 +81,7 @@ pub fn gen_sink_plan(
     stmt: CreateSinkStatement,
     partition_info: Option<PartitionComputeInfo>,
 ) -> Result<SinkPlanContext> {
+    let user_specified_columns = !stmt.columns.is_empty();
     let db_name = session.database();
     let (sink_schema_name, sink_table_name) =
         Binder::resolve_schema_qualified_name(db_name, stmt.sink_name.clone())?;
@@ -118,8 +120,12 @@ pub fn gen_sink_plan(
     let check_items = resolve_query_privileges(&bound);
     session.check_privileges(&check_items)?;
 
-    // If column names not specified, use the name in materialized view.
-    let col_names = get_column_names(&bound, session, stmt.columns)?;
+    let col_names = if sink_into_table_name.is_some() {
+        parse_column_names(&stmt.columns)
+    } else {
+        // If column names not specified, use the name in the bound query, which is equal with the plan root's original field name.
+        get_column_names(&bound, session, stmt.columns)?
+    };
 
     let mut with_options = context.with_options().clone();
 
@@ -140,6 +146,7 @@ pub fn gen_sink_plan(
             resolve_privatelink_in_with_option(&mut with_options, &sink_schema_name, session)?;
         conn_id.map(ConnectionId)
     };
+    let secret_ref = resolve_secret_in_with_options(&mut with_options, session)?;
 
     let emit_on_window_close = stmt.emit_mode == Some(EmitMode::OnWindowClose);
     if emit_on_window_close {
@@ -172,8 +179,8 @@ pub fn gen_sink_plan(
     };
 
     let mut plan_root = Planner::new(context).plan_query(bound)?;
-    if let Some(col_names) = col_names {
-        plan_root.set_out_names(col_names)?;
+    if let Some(col_names) = &col_names {
+        plan_root.set_out_names(col_names.clone())?;
     };
 
     let without_backfill = match with_options.remove(SINK_WITHOUT_BACKFILL) {
@@ -196,6 +203,25 @@ pub fn gen_sink_plan(
         .map(|table_name| fetch_table_catalog_for_alter(session, table_name))
         .transpose()?;
 
+    if let Some(target_table_catalog) = &target_table_catalog {
+        if let Some(col_names) = col_names {
+            let target_table_columns = target_table_catalog
+                .columns()
+                .iter()
+                .map(|c| c.name())
+                .collect::<BTreeSet<_>>();
+            for c in col_names {
+                if !target_table_columns.contains(c.as_str()) {
+                    return Err(RwError::from(ErrorCode::BindError(format!(
+                        "Column {} not found in table {}",
+                        c,
+                        target_table_catalog.name()
+                    ))));
+                }
+            }
+        }
+    }
+
     let target_table = target_table_catalog.as_ref().map(|catalog| catalog.id());
 
     let sink_plan = plan_root.gen_sink_plan(
@@ -230,6 +256,7 @@ pub fn gen_sink_plan(
         UserId::new(session.user_id()),
         connection_id,
         dependent_relations.into_iter().collect_vec(),
+        secret_ref,
     );
 
     if let Some(table_catalog) = &target_table_catalog {
@@ -251,7 +278,12 @@ pub fn gen_sink_plan(
             )));
         }
 
-        let exprs = derive_default_column_project_for_sink(&sink_catalog, table_catalog)?;
+        let exprs = derive_default_column_project_for_sink(
+            &sink_catalog,
+            sink_plan.schema(),
+            table_catalog,
+            user_specified_columns,
+        )?;
 
         let logical_project = generic::Project::new(exprs, sink_plan);
 
@@ -280,13 +312,13 @@ pub fn gen_sink_plan(
 pub async fn get_partition_compute_info(
     with_options: &WithOptions,
 ) -> Result<Option<PartitionComputeInfo>> {
-    let properties = HashMap::from_iter(with_options.clone().into_inner().into_iter());
+    let properties = with_options.clone().into_inner();
     let Some(connector) = properties.get(UPSTREAM_SOURCE_KEY) else {
         return Ok(None);
     };
     match connector.as_str() {
         ICEBERG_SINK => {
-            let iceberg_config = IcebergConfig::from_hashmap(properties)?;
+            let iceberg_config = IcebergConfig::from_btreemap(properties)?;
             get_partition_compute_info_for_iceberg(&iceberg_config).await
         }
         _ => Ok(None),
@@ -351,14 +383,14 @@ async fn get_partition_compute_info_for_iceberg(
         })
         .collect::<Result<Vec<_>>>()?;
 
-    let DataType::Struct(partition_type) = arrow_type.into() else {
+    let ArrowDataType::Struct(partition_type) = arrow_type else {
         return Err(RwError::from(ErrorCode::SinkError(
             "Partition type of iceberg should be a struct type".into(),
         )));
     };
 
     Ok(Some(PartitionComputeInfo::Iceberg(IcebergPartitionInfo {
-        partition_type,
+        partition_type: IcebergArrowConvert.from_fields(&partition_type)?,
         partition_fields,
     })))
 }
@@ -465,6 +497,7 @@ fn check_cycle_for_sink(
 
     let mut sinks = HashMap::new();
     let mut sources = HashMap::new();
+    let mut views = HashMap::new();
     let db_name = session.database();
     for schema in reader.iter_schemas(db_name)? {
         for sink in schema.iter_sink() {
@@ -474,12 +507,17 @@ fn check_cycle_for_sink(
         for source in schema.iter_source() {
             sources.insert(source.id, source.as_ref());
         }
+
+        for view in schema.iter_view() {
+            views.insert(view.id, view.as_ref());
+        }
     }
 
     struct Context<'a> {
         reader: &'a CatalogReadGuard,
         sink_index: &'a HashMap<u32, &'a SinkCatalog>,
         source_index: &'a HashMap<u32, &'a SourceCatalog>,
+        view_index: &'a HashMap<u32, &'a ViewCatalog>,
     }
 
     impl Context<'_> {
@@ -527,7 +565,9 @@ fn check_cycle_for_sink(
                     path.push(table.name.clone());
                     self.visit_table(table.as_ref(), target_table_id, path)?;
                     path.pop();
-                } else if self.source_index.get(&table_id.table_id).is_some() {
+                } else if self.source_index.contains_key(&table_id.table_id)
+                    || self.view_index.contains_key(&table_id.table_id)
+                {
                     continue;
                 } else {
                     bail!("streaming job not found: {:?}", table_id);
@@ -546,6 +586,7 @@ fn check_cycle_for_sink(
         reader: &reader,
         sink_index: &sinks,
         source_index: &sources,
+        view_index: &views,
     };
 
     ctx.visit_dependent_jobs(&sink_catalog.dependent_relations, table_id, &mut path)?;
@@ -632,66 +673,78 @@ pub(crate) fn insert_merger_to_union(node: &mut StreamNode) {
         insert_merger_to_union(input);
     }
 }
+
+fn derive_sink_to_table_expr(
+    sink_schema: &Schema,
+    idx: usize,
+    target_type: &DataType,
+) -> Result<ExprImpl> {
+    let input_type = &sink_schema.fields()[idx].data_type;
+
+    if target_type != input_type {
+        bail!(
+            "column type mismatch: {:?} vs {:?}",
+            target_type,
+            input_type
+        );
+    } else {
+        Ok(ExprImpl::InputRef(Box::new(InputRef::new(
+            idx,
+            input_type.clone(),
+        ))))
+    }
+}
+
 fn derive_default_column_project_for_sink(
     sink: &SinkCatalog,
+    sink_schema: &Schema,
     target_table_catalog: &Arc<TableCatalog>,
+    user_specified_columns: bool,
 ) -> Result<Vec<ExprImpl>> {
+    assert_eq!(sink.full_schema().len(), sink_schema.len());
+
     let mut exprs = vec![];
 
-    let sink_visible_columns = sink
+    let sink_visible_col_idxes = sink
         .full_columns()
         .iter()
-        .enumerate()
-        .filter(|(_i, c)| !c.is_hidden())
+        .positions(|c| !c.is_hidden())
         .collect_vec();
+    let sink_visible_col_idxes_by_name = sink
+        .full_columns()
+        .iter()
+        .enumerate()
+        .filter(|(_, c)| !c.is_hidden())
+        .map(|(i, c)| (c.name(), i))
+        .collect::<BTreeMap<_, _>>();
 
     for (idx, table_column) in target_table_catalog.columns().iter().enumerate() {
         if table_column.is_generated() {
             continue;
         }
 
-        let data_type = table_column.data_type();
-
-        if idx < sink_visible_columns.len() {
-            let (sink_col_idx, sink_column) = sink_visible_columns[idx];
-
-            let sink_col_type = sink_column.data_type();
+        let default_col_expr = || -> ExprImpl { target_table_catalog.default_column_expr(idx) };
+        let sink_col_expr = |sink_col_idx: usize| -> Result<ExprImpl> {
+            derive_sink_to_table_expr(sink_schema, sink_col_idx, table_column.data_type())
+        };
 
-            if data_type != sink_col_type {
-                bail!(
-                    "column type mismatch: {:?} vs {:?}",
-                    data_type,
-                    sink_col_type
-                );
+        // If users specified the columns to be inserted e.g. `CREATE SINK s INTO t(a, b)`, the expressions of `Project` will be generated accordingly.
+        // The missing columns will be filled with default value (`null` if not explicitly defined).
+        // Otherwise, e.g. `CREATE SINK s INTO t`, the columns will be matched by their order in `select` query and the target table.
+        #[allow(clippy::collapsible_else_if)]
+        if user_specified_columns {
+            if let Some(idx) = sink_visible_col_idxes_by_name.get(table_column.name()) {
+                exprs.push(sink_col_expr(*idx)?);
             } else {
-                exprs.push(ExprImpl::InputRef(Box::new(InputRef::new(
-                    sink_col_idx,
-                    data_type.clone(),
-                ))));
+                exprs.push(default_col_expr());
             }
         } else {
-            let data = match table_column
-                .column_desc
-                .generated_or_default_column
-                .as_ref()
-            {
-                // default column with default value
-                Some(GeneratedOrDefaultColumn::DefaultColumn(default_column)) => {
-                    Datum::from_protobuf(default_column.get_snapshot_value().unwrap(), data_type)
-                        .unwrap()
-                }
-                // default column with no default value
-                None => None,
-
-                // generated column is unreachable
-                _ => unreachable!(),
+            if idx < sink_visible_col_idxes.len() {
+                exprs.push(sink_col_expr(sink_visible_col_idxes[idx])?);
+            } else {
+                exprs.push(default_col_expr());
             };
-
-            exprs.push(ExprImpl::Literal(Box::new(Literal::new(
-                data,
-                data_type.clone(),
-            ))));
-        };
+        }
     }
     Ok(exprs)
 }
@@ -717,10 +770,23 @@ fn bind_sink_format_desc(value: ConnectorSchema) -> Result<SinkFormatDesc> {
         E::Protobuf => SinkEncode::Protobuf,
         E::Avro => SinkEncode::Avro,
         E::Template => SinkEncode::Template,
-        e @ (E::Native | E::Csv | E::Bytes | E::None) => {
+        e @ (E::Native | E::Csv | E::Bytes | E::None | E::Text) => {
             return Err(ErrorCode::BindError(format!("sink encode unsupported: {e}")).into());
         }
     };
+
+    let mut key_encode = None;
+    if let Some(encode) = value.key_encode {
+        if encode == E::Text {
+            key_encode = Some(SinkEncode::Text);
+        } else {
+            return Err(ErrorCode::BindError(format!(
+                "sink key encode unsupported: {encode}, only TEXT supported"
+            ))
+            .into());
+        }
+    }
+
     let mut options = WithOptions::try_from(value.row_options.as_slice())?.into_inner();
 
     options
@@ -731,11 +797,13 @@ fn bind_sink_format_desc(value: ConnectorSchema) -> Result<SinkFormatDesc> {
         format,
         encode,
         options,
+        key_encode,
     })
 }
 
 static CONNECTORS_COMPATIBLE_FORMATS: LazyLock<HashMap<String, HashMap<Format, Vec<Encode>>>> =
     LazyLock::new(|| {
+        use risingwave_connector::sink::google_pubsub::GooglePubSubSink;
         use risingwave_connector::sink::kafka::KafkaSink;
         use risingwave_connector::sink::kinesis::KinesisSink;
         use risingwave_connector::sink::mqtt::MqttSink;
@@ -744,8 +812,11 @@ static CONNECTORS_COMPATIBLE_FORMATS: LazyLock<HashMap<String, HashMap<Format, V
         use risingwave_connector::sink::Sink as _;
 
         convert_args!(hashmap!(
+                GooglePubSubSink::SINK_NAME => hashmap!(
+                    Format::Plain => vec![Encode::Json],
+                ),
                 KafkaSink::SINK_NAME => hashmap!(
-                    Format::Plain => vec![Encode::Json, Encode::Protobuf],
+                    Format::Plain => vec![Encode::Json, Encode::Avro, Encode::Protobuf],
                     Format::Upsert => vec![Encode::Json, Encode::Avro],
                     Format::Debezium => vec![Encode::Json],
                 ),
@@ -763,8 +834,8 @@ static CONNECTORS_COMPATIBLE_FORMATS: LazyLock<HashMap<String, HashMap<Format, V
                     Format::Debezium => vec![Encode::Json],
                 ),
                 RedisSink::SINK_NAME => hashmap!(
-                    Format::Plain => vec![Encode::Json,Encode::Template],
-                    Format::Upsert => vec![Encode::Json,Encode::Template],
+                    Format::Plain => vec![Encode::Json, Encode::Template],
+                    Format::Upsert => vec![Encode::Json, Encode::Template],
                 ),
         ))
     });
diff --git a/src/frontend/src/handler/create_source.rs b/src/frontend/src/handler/create_source.rs
index 0830cdb5392de..a29aa86907e0f 100644
--- a/src/frontend/src/handler/create_source.rs
+++ b/src/frontend/src/handler/create_source.rs
@@ -21,6 +21,7 @@ use either::Either;
 use itertools::Itertools;
 use maplit::{convert_args, hashmap};
 use pgwire::pg_response::{PgResponse, StatementType};
+use risingwave_common::array::arrow::{FromArrow, IcebergArrowConvert};
 use risingwave_common::bail_not_implemented;
 use risingwave_common::catalog::{
     is_column_ids_dedup, ColumnCatalog, ColumnDesc, ColumnId, Schema, TableId,
@@ -28,11 +29,11 @@ use risingwave_common::catalog::{
 };
 use risingwave_common::types::DataType;
 use risingwave_connector::parser::additional_columns::{
-    build_additional_column_catalog, COMPATIBLE_ADDITIONAL_COLUMNS,
+    build_additional_column_catalog, get_supported_additional_columns,
 };
 use risingwave_connector::parser::{
-    schema_to_columns, AvroParserConfig, DebeziumAvroParserConfig, ProtobufParserConfig,
-    SpecificParserConfig, TimestamptzHandling, DEBEZIUM_IGNORE_KEY,
+    fetch_json_schema_and_map_to_columns, AvroParserConfig, DebeziumAvroParserConfig,
+    ProtobufParserConfig, SpecificParserConfig, TimestamptzHandling, DEBEZIUM_IGNORE_KEY,
 };
 use risingwave_connector::schema::schema_registry::{
     name_strategy_from_str, SchemaRegistryAuth, SCHEMA_REGISTRY_PASSWORD, SCHEMA_REGISTRY_USERNAME,
@@ -40,7 +41,8 @@ use risingwave_connector::schema::schema_registry::{
 use risingwave_connector::sink::iceberg::IcebergConfig;
 use risingwave_connector::source::cdc::{
     CDC_SHARING_MODE_KEY, CDC_SNAPSHOT_BACKFILL, CDC_SNAPSHOT_MODE_KEY, CDC_TRANSACTIONAL_KEY,
-    CITUS_CDC_CONNECTOR, MONGODB_CDC_CONNECTOR, MYSQL_CDC_CONNECTOR, POSTGRES_CDC_CONNECTOR,
+    CDC_WAIT_FOR_STREAMING_START_TIMEOUT, CITUS_CDC_CONNECTOR, MONGODB_CDC_CONNECTOR,
+    MYSQL_CDC_CONNECTOR, POSTGRES_CDC_CONNECTOR,
 };
 use risingwave_connector::source::datagen::DATAGEN_CONNECTOR;
 use risingwave_connector::source::iceberg::ICEBERG_CONNECTOR;
@@ -52,15 +54,13 @@ use risingwave_connector::source::{
     POSIX_FS_CONNECTOR, PULSAR_CONNECTOR, S3_CONNECTOR,
 };
 use risingwave_connector::WithPropertiesExt;
-use risingwave_pb::catalog::{
-    PbSchemaRegistryNameStrategy, PbSource, StreamSourceInfo, WatermarkDesc,
-};
+use risingwave_pb::catalog::{PbSchemaRegistryNameStrategy, StreamSourceInfo, WatermarkDesc};
 use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType;
 use risingwave_pb::plan_common::{EncodeType, FormatType};
 use risingwave_pb::stream_plan::stream_fragment_graph::Parallelism;
 use risingwave_sqlparser::ast::{
     get_delimiter, AstString, ColumnDef, ConnectorSchema, CreateSourceStatement, Encode, Format,
-    ProtobufSchema, SourceWatermark,
+    ObjectName, ProtobufSchema, SourceWatermark, TableConstraint,
 };
 use risingwave_sqlparser::parser::IncludeOption;
 use thiserror_ext::AsReport;
@@ -68,6 +68,7 @@ use thiserror_ext::AsReport;
 use super::RwPgResponse;
 use crate::binder::Binder;
 use crate::catalog::source_catalog::SourceCatalog;
+use crate::catalog::{DatabaseId, SchemaId};
 use crate::error::ErrorCode::{self, Deprecated, InvalidInputSyntax, NotSupported, ProtocolError};
 use crate::error::{Result, RwError};
 use crate::expr::Expr;
@@ -80,7 +81,7 @@ use crate::handler::HandlerArgs;
 use crate::optimizer::plan_node::generic::SourceNodeKind;
 use crate::optimizer::plan_node::{LogicalSource, ToStream, ToStreamContext};
 use crate::session::SessionImpl;
-use crate::utils::resolve_privatelink_in_with_option;
+use crate::utils::{resolve_privatelink_in_with_option, resolve_secret_in_with_options};
 use crate::{bind_data_type, build_graph, OptimizerContext, WithOptions};
 
 pub(crate) const UPSTREAM_SOURCE_KEY: &str = "connector";
@@ -88,7 +89,7 @@ pub(crate) const UPSTREAM_SOURCE_KEY: &str = "connector";
 /// Map a JSON schema to a relational schema
 async fn extract_json_table_schema(
     schema_config: &Option<(AstString, bool)>,
-    with_properties: &HashMap<String, String>,
+    with_properties: &BTreeMap<String, String>,
     format_encode_options: &mut BTreeMap<String, String>,
 ) -> Result<Option<Vec<ColumnCatalog>>> {
     match schema_config {
@@ -101,14 +102,18 @@ async fn extract_json_table_schema(
                 auth
             });
             Ok(Some(
-                schema_to_columns(&schema_location.0, schema_registry_auth, with_properties)
-                    .await?
-                    .into_iter()
-                    .map(|col| ColumnCatalog {
-                        column_desc: col.into(),
-                        is_hidden: false,
-                    })
-                    .collect_vec(),
+                fetch_json_schema_and_map_to_columns(
+                    &schema_location.0,
+                    schema_registry_auth,
+                    with_properties,
+                )
+                .await?
+                .into_iter()
+                .map(|col| ColumnCatalog {
+                    column_desc: col.into(),
+                    is_hidden: false,
+                })
+                .collect_vec(),
             ))
         }
     }
@@ -138,7 +143,7 @@ fn json_schema_infer_use_schema_registry(schema_config: &Option<(AstString, bool
 /// Map an Avro schema to a relational schema.
 async fn extract_avro_table_schema(
     info: &StreamSourceInfo,
-    with_properties: &HashMap<String, String>,
+    with_properties: &BTreeMap<String, String>,
     format_encode_options: &mut BTreeMap<String, String>,
     is_debezium: bool,
 ) -> Result<Vec<ColumnCatalog>> {
@@ -174,7 +179,7 @@ async fn extract_avro_table_schema(
 
 async fn extract_debezium_avro_table_pk_columns(
     info: &StreamSourceInfo,
-    with_properties: &HashMap<String, String>,
+    with_properties: &WithOptions,
 ) -> Result<Vec<String>> {
     let parser_config = SpecificParserConfig::new(info, with_properties)?;
     let conf = DebeziumAvroParserConfig::new(parser_config.encoding_config).await?;
@@ -184,7 +189,7 @@ async fn extract_debezium_avro_table_pk_columns(
 /// Map a protobuf schema to a relational schema.
 async fn extract_protobuf_table_schema(
     schema: &ProtobufSchema,
-    with_properties: &HashMap<String, String>,
+    with_properties: &BTreeMap<String, String>,
     format_encode_options: &mut BTreeMap<String, String>,
 ) -> Result<Vec<ColumnCatalog>> {
     let info = StreamSourceInfo {
@@ -289,7 +294,7 @@ fn get_name_strategy_or_default(name_strategy: Option<AstString>) -> Result<Opti
 pub(crate) async fn bind_columns_from_source(
     session: &SessionImpl,
     source_schema: &ConnectorSchema,
-    with_properties: &HashMap<String, String>,
+    with_properties: &BTreeMap<String, String>,
 ) -> Result<(Option<Vec<ColumnCatalog>>, StreamSourceInfo)> {
     const MESSAGE_NAME_KEY: &str = "message";
     const KEY_MESSAGE_NAME_KEY: &str = "key.message";
@@ -445,6 +450,7 @@ pub(crate) async fn bind_columns_from_source(
                 // Parse the value but throw it away.
                 // It would be too late to report error in `SpecificParserConfig::new`,
                 // which leads to recovery loop.
+                // TODO: rely on SpecificParserConfig::new to validate, like Avro
                 TimestamptzHandling::from_options(&format_encode_options_to_consume)
                     .map_err(|err| InvalidInputSyntax(err.message))?;
                 try_consume_string_from_options(
@@ -502,7 +508,6 @@ pub(crate) async fn bind_columns_from_source(
 fn bind_columns_from_source_for_cdc(
     session: &SessionImpl,
     source_schema: &ConnectorSchema,
-    _with_properties: &HashMap<String, String>,
 ) -> Result<(Option<Vec<ColumnCatalog>>, StreamSourceInfo)> {
     let format_encode_options = WithOptions::try_from(source_schema.row_options())?.into_inner();
     let mut format_encode_options_to_consume = format_encode_options.clone();
@@ -548,15 +553,14 @@ fn bind_columns_from_source_for_cdc(
 
 /// add connector-spec columns to the end of column catalog
 pub fn handle_addition_columns(
-    with_properties: &HashMap<String, String>,
+    with_properties: &BTreeMap<String, String>,
     mut additional_columns: IncludeOption,
     columns: &mut Vec<ColumnCatalog>,
+    is_cdc_backfill_table: bool,
 ) -> Result<()> {
     let connector_name = with_properties.get_connector().unwrap(); // there must be a connector in source
 
-    if COMPATIBLE_ADDITIONAL_COLUMNS
-        .get(connector_name.as_str())
-        .is_none()
+    if get_supported_additional_columns(connector_name.as_str(), is_cdc_backfill_table).is_none()
         && !additional_columns.is_empty()
     {
         return Err(RwError::from(ProtocolError(format!(
@@ -595,6 +599,7 @@ pub fn handle_addition_columns(
             item.inner_field.as_deref(),
             data_type_name.as_deref(),
             true,
+            is_cdc_backfill_table,
         )?);
     }
 
@@ -727,7 +732,7 @@ pub(crate) async fn bind_source_pk(
     source_info: &StreamSourceInfo,
     columns: &mut [ColumnCatalog],
     sql_defined_pk_names: Vec<String>,
-    with_properties: &HashMap<String, String>,
+    with_properties: &WithOptions,
 ) -> Result<Vec<String>> {
     let sql_defined_pk = !sql_defined_pk_names.is_empty();
     let key_column_name: Option<String> = {
@@ -838,12 +843,6 @@ pub(crate) async fn bind_source_pk(
             }
         }
         (Format::DebeziumMongo, Encode::Json) => {
-            if !additional_column_names.is_empty() {
-                return Err(RwError::from(ProtocolError(format!(
-                    "FORMAT DEBEZIUMMONGO forbids additional columns, but got {:?}",
-                    additional_column_names
-                ))));
-            }
             if sql_defined_pk {
                 sql_defined_pk_names
             } else {
@@ -893,10 +892,7 @@ pub(crate) async fn bind_source_pk(
 }
 
 // Add a hidden column `_rw_kafka_timestamp` to each message from Kafka source.
-fn check_and_add_timestamp_column(
-    with_properties: &HashMap<String, String>,
-    columns: &mut Vec<ColumnCatalog>,
-) {
+fn check_and_add_timestamp_column(with_properties: &WithOptions, columns: &mut Vec<ColumnCatalog>) {
     if with_properties.is_kafka_connector() {
         if columns.iter().any(|col| {
             matches!(
@@ -917,6 +913,7 @@ fn check_and_add_timestamp_column(
             None,
             None,
             true,
+            false,
         )
         .unwrap();
         catalog.is_hidden = true;
@@ -1044,7 +1041,7 @@ static CONNECTORS_COMPATIBLE_FORMATS: LazyLock<HashMap<String, HashMap<Format, V
 
 pub fn validate_compatibility(
     source_schema: &ConnectorSchema,
-    props: &mut HashMap<String, String>,
+    props: &mut BTreeMap<String, String>,
 ) -> Result<()> {
     let connector = props
         .get_connector()
@@ -1128,7 +1125,7 @@ pub fn validate_compatibility(
 /// One should only call this function after all properties of all columns are resolved, like
 /// generated column descriptors.
 pub(super) async fn check_source_schema(
-    props: &HashMap<String, String>,
+    props: &WithOptions,
     row_id_index: Option<usize>,
     columns: &[ColumnCatalog],
 ) -> Result<()> {
@@ -1137,9 +1134,9 @@ pub(super) async fn check_source_schema(
     };
 
     if connector == NEXMARK_CONNECTOR {
-        check_nexmark_schema(props, row_id_index, columns)
+        check_nexmark_schema(props.inner(), row_id_index, columns)
     } else if connector == ICEBERG_CONNECTOR {
-        Ok(check_iceberg_source(props, columns)
+        Ok(check_iceberg_source(props.inner(), columns)
             .await
             .map_err(|err| ProtocolError(err.to_report_string()))?)
     } else {
@@ -1148,7 +1145,7 @@ pub(super) async fn check_source_schema(
 }
 
 pub(super) fn check_nexmark_schema(
-    props: &HashMap<String, String>,
+    props: &BTreeMap<String, String>,
     row_id_index: Option<usize>,
     columns: &[ColumnCatalog],
 ) -> Result<()> {
@@ -1202,7 +1199,7 @@ pub(super) fn check_nexmark_schema(
 }
 
 pub async fn extract_iceberg_columns(
-    with_properties: &HashMap<String, String>,
+    with_properties: &BTreeMap<String, String>,
 ) -> anyhow::Result<Vec<ColumnCatalog>> {
     let props = ConnectorProperties::extract(with_properties.clone(), true)?;
     if let ConnectorProperties::Iceberg(properties) = props {
@@ -1219,11 +1216,10 @@ pub async fn extract_iceberg_columns(
             .iter()
             .enumerate()
             .map(|(i, field)| {
-                let data_type = field.data_type().clone();
                 let column_desc = ColumnDesc::named(
                     field.name(),
                     ColumnId::new((i as u32).try_into().unwrap()),
-                    data_type.into(),
+                    IcebergArrowConvert.from_field(field).unwrap(),
                 );
                 ColumnCatalog {
                     column_desc,
@@ -1242,7 +1238,7 @@ pub async fn extract_iceberg_columns(
 }
 
 pub async fn check_iceberg_source(
-    props: &HashMap<String, String>,
+    props: &BTreeMap<String, String>,
     columns: &[ColumnCatalog],
 ) -> anyhow::Result<()> {
     let props = ConnectorProperties::extract(props.clone(), true)?;
@@ -1288,73 +1284,92 @@ pub async fn check_iceberg_source(
         .collect::<Vec<_>>();
     let new_iceberg_schema = arrow_schema::Schema::new(new_iceberg_field);
 
-    risingwave_connector::sink::iceberg::try_matches_arrow_schema(
-        &schema,
-        &new_iceberg_schema,
-        true,
-    )?;
+    risingwave_connector::sink::iceberg::try_matches_arrow_schema(&schema, &new_iceberg_schema)?;
 
     Ok(())
 }
 
-pub async fn handle_create_source(
-    handler_args: HandlerArgs,
-    stmt: CreateSourceStatement,
-) -> Result<RwPgResponse> {
-    let session = handler_args.session.clone();
-
-    if let Either::Right(resp) = session.check_relation_name_duplicated(
-        stmt.source_name.clone(),
-        StatementType::CREATE_SOURCE,
-        stmt.if_not_exists,
-    )? {
-        return Ok(resp);
+pub fn bind_connector_props(
+    handler_args: &HandlerArgs,
+    source_schema: &ConnectorSchema,
+    is_create_source: bool,
+) -> Result<WithOptions> {
+    let mut with_properties = handler_args.with_options.clone().into_connector_props();
+    validate_compatibility(source_schema, &mut with_properties)?;
+    let create_cdc_source_job = with_properties.is_shareable_cdc_connector();
+    if is_create_source && create_cdc_source_job {
+        // set connector to backfill mode
+        with_properties.insert(CDC_SNAPSHOT_MODE_KEY.into(), CDC_SNAPSHOT_BACKFILL.into());
+        // enable cdc sharing mode, which will capture all tables in the given `database.name`
+        with_properties.insert(CDC_SHARING_MODE_KEY.into(), "true".into());
+        // enable transactional cdc
+        with_properties.insert(CDC_TRANSACTIONAL_KEY.into(), "true".into());
+        with_properties.insert(
+            CDC_WAIT_FOR_STREAMING_START_TIMEOUT.into(),
+            handler_args
+                .session
+                .config()
+                .cdc_source_wait_streaming_start_timeout()
+                .to_string(),
+        );
     }
-
-    let db_name = session.database();
-    let (schema_name, name) = Binder::resolve_schema_qualified_name(db_name, stmt.source_name)?;
+    Ok(WithOptions::new(with_properties))
+}
+#[allow(clippy::too_many_arguments)]
+pub async fn bind_create_source(
+    handler_args: HandlerArgs,
+    full_name: ObjectName,
+    source_schema: ConnectorSchema,
+    with_properties: WithOptions,
+    sql_columns_defs: &[ColumnDef],
+    constraints: Vec<TableConstraint>,
+    wildcard_idx: Option<usize>,
+    source_watermarks: Vec<SourceWatermark>,
+    columns_from_resolve_source: Option<Vec<ColumnCatalog>>,
+    source_info: StreamSourceInfo,
+    include_column_options: IncludeOption,
+    col_id_gen: &mut ColumnIdGenerator,
+    // `true` for "create source", `false` for "create table with connector"
+    is_create_source: bool,
+) -> Result<(SourceCatalog, DatabaseId, SchemaId)> {
+    let session = &handler_args.session;
+    let db_name: &str = session.database();
+    let (schema_name, source_name) = Binder::resolve_schema_qualified_name(db_name, full_name)?;
     let (database_id, schema_id) =
         session.get_database_and_schema_id_for_create(schema_name.clone())?;
 
-    if handler_args.with_options.is_empty() {
-        return Err(RwError::from(InvalidInputSyntax(
-            "missing WITH clause".to_string(),
-        )));
+    if !is_create_source && with_properties.is_iceberg_connector() {
+        return Err(
+            ErrorCode::BindError("can't create table with iceberg connector".to_string()).into(),
+        );
     }
 
-    let source_schema = stmt.source_schema.into_v2_with_warning();
+    ensure_table_constraints_supported(&constraints)?;
+    let sql_pk_names = bind_sql_pk_names(sql_columns_defs, &constraints)?;
 
-    let mut with_properties = handler_args.with_options.clone().into_connector_props();
-    validate_compatibility(&source_schema, &mut with_properties)?;
-
-    ensure_table_constraints_supported(&stmt.constraints)?;
-    let sql_pk_names = bind_sql_pk_names(&stmt.columns, &stmt.constraints)?;
-
-    let create_cdc_source_job = with_properties.is_shareable_cdc_connector();
-    let is_shared = create_cdc_source_job
-        || (with_properties.is_kafka_connector() && session.config().rw_enable_shared_source());
-
-    let (columns_from_resolve_source, mut source_info) = if create_cdc_source_job {
-        bind_columns_from_source_for_cdc(&session, &source_schema, &with_properties)?
-    } else {
-        bind_columns_from_source(&session, &source_schema, &with_properties).await?
-    };
-    if is_shared {
-        // Note: this field should be called is_shared. Check field doc for more details.
-        source_info.cdc_source_job = true;
-        source_info.is_distributed = !create_cdc_source_job;
-    }
-    let columns_from_sql = bind_sql_columns(&stmt.columns)?;
+    let columns_from_sql = bind_sql_columns(sql_columns_defs)?;
 
     let mut columns = bind_all_columns(
         &source_schema,
         columns_from_resolve_source,
         columns_from_sql,
-        &stmt.columns,
-        stmt.wildcard_idx,
+        sql_columns_defs,
+        wildcard_idx,
     )?;
+
     // add additional columns before bind pk, because `format upsert` requires the key column
-    handle_addition_columns(&with_properties, stmt.include_column_options, &mut columns)?;
+    handle_addition_columns(
+        &with_properties,
+        include_column_options,
+        &mut columns,
+        false,
+    )?;
+    // compatible with the behavior that add a hidden column `_rw_kafka_timestamp` to each message from Kafka source
+    if is_create_source {
+        // must behind `handle_addition_columns`
+        check_and_add_timestamp_column(&with_properties, &mut columns);
+    }
+
     let pk_names = bind_source_pk(
         &source_schema,
         &source_info,
@@ -1364,79 +1379,140 @@ pub async fn handle_create_source(
     )
     .await?;
 
-    if create_cdc_source_job {
-        // set connector to backfill mode
-        with_properties.insert(CDC_SNAPSHOT_MODE_KEY.into(), CDC_SNAPSHOT_BACKFILL.into());
-        // enable cdc sharing mode, which will capture all tables in the given `database.name`
-        with_properties.insert(CDC_SHARING_MODE_KEY.into(), "true".into());
-        // enable transactional cdc
-        with_properties.insert(CDC_TRANSACTIONAL_KEY.into(), "true".into());
-    }
-
-    // must behind `handle_addition_columns`
-    check_and_add_timestamp_column(&with_properties, &mut columns);
-
-    let mut col_id_gen = ColumnIdGenerator::new_initial();
-    for c in &mut columns {
-        c.column_desc.column_id = col_id_gen.generate(c.name())
-    }
-
-    if !pk_names.is_empty() {
+    if is_create_source && !pk_names.is_empty() {
         return Err(ErrorCode::InvalidInputSyntax(
             "Source does not support PRIMARY KEY constraint, please use \"CREATE TABLE\" instead"
                 .to_owned(),
         )
         .into());
     }
-    let (mut columns, pk_column_ids, row_id_index) =
-        bind_pk_on_relation(columns, pk_names, with_properties.connector_need_pk())?;
 
+    for c in &mut columns {
+        c.column_desc.column_id = col_id_gen.generate(c.name())
+    }
     debug_assert!(is_column_ids_dedup(&columns));
 
+    let must_need_pk = if is_create_source {
+        with_properties.connector_need_pk()
+    } else {
+        // For those connectors that do not need generate a `row_id`` column in the source schema such as iceberg.
+        // But in such case, we can not create mv or table on the source because there is not a pk.
+        assert!(with_properties.connector_need_pk());
+
+        true
+    };
+
+    let (mut columns, pk_col_ids, row_id_index) =
+        bind_pk_on_relation(columns, pk_names, must_need_pk)?;
+
     let watermark_descs =
-        bind_source_watermark(&session, name.clone(), stmt.source_watermarks, &columns)?;
+        bind_source_watermark(session, source_name.clone(), source_watermarks, &columns)?;
     // TODO(yuhao): allow multiple watermark on source.
     assert!(watermark_descs.len() <= 1);
 
     bind_sql_column_constraints(
-        &session,
-        name.clone(),
+        session,
+        source_name.clone(),
         &mut columns,
-        stmt.columns,
-        &pk_column_ids,
+        // TODO(st1page): pass the ref
+        sql_columns_defs.to_vec(),
+        &pk_col_ids,
     )?;
-
     check_source_schema(&with_properties, row_id_index, &columns).await?;
 
-    let pk_column_ids = pk_column_ids.into_iter().map(Into::into).collect();
-
-    let mut with_options = WithOptions::new(with_properties);
-    // resolve privatelink connection for Kafka source
+    // resolve privatelink connection for Kafka
+    let mut with_properties = with_properties;
     let connection_id =
-        resolve_privatelink_in_with_option(&mut with_options, &schema_name, &session)?;
-    let definition = handler_args.normalized_sql.clone();
+        resolve_privatelink_in_with_option(&mut with_properties, &schema_name, session)?;
+    let _secret_ref = resolve_secret_in_with_options(&mut with_properties, session)?;
+
+    let definition: String = handler_args.normalized_sql.clone();
 
-    let source = PbSource {
+    let associated_table_id = if is_create_source {
+        None
+    } else {
+        Some(TableId::placeholder())
+    };
+    let source = SourceCatalog {
         id: TableId::placeholder().table_id,
-        schema_id,
-        database_id,
-        name,
-        row_id_index: row_id_index.map(|idx| idx as u32),
-        columns: columns.iter().map(|c| c.to_protobuf()).collect_vec(),
-        pk_column_ids,
-        with_properties: with_options.into_inner().into_iter().collect(),
-        info: Some(source_info),
+        name: source_name,
+        columns,
+        pk_col_ids,
+        append_only: row_id_index.is_some(),
         owner: session.user_id(),
+        info: source_info,
+        row_id_index,
+        with_properties: with_properties.into_inner().into_iter().collect(),
         watermark_descs,
+        associated_table_id,
         definition,
         connection_id,
-        initialized_at_epoch: None,
         created_at_epoch: None,
-        optional_associated_table_id: None,
+        initialized_at_epoch: None,
         version: INITIAL_SOURCE_VERSION_ID,
-        initialized_at_cluster_version: None,
         created_at_cluster_version: None,
+        initialized_at_cluster_version: None,
+    };
+    Ok((source, database_id, schema_id))
+}
+
+pub async fn handle_create_source(
+    handler_args: HandlerArgs,
+    stmt: CreateSourceStatement,
+) -> Result<RwPgResponse> {
+    let session = handler_args.session.clone();
+
+    if let Either::Right(resp) = session.check_relation_name_duplicated(
+        stmt.source_name.clone(),
+        StatementType::CREATE_SOURCE,
+        stmt.if_not_exists,
+    )? {
+        return Ok(resp);
+    }
+
+    if handler_args.with_options.is_empty() {
+        return Err(RwError::from(InvalidInputSyntax(
+            "missing WITH clause".to_string(),
+        )));
+    }
+
+    let source_schema = stmt.source_schema.into_v2_with_warning();
+    let with_properties = bind_connector_props(&handler_args, &source_schema, true)?;
+
+    let create_cdc_source_job = with_properties.is_shareable_cdc_connector();
+    let is_shared = create_cdc_source_job
+        || (with_properties.is_kafka_connector() && session.config().rw_enable_shared_source());
+
+    let (columns_from_resolve_source, mut source_info) = if create_cdc_source_job {
+        bind_columns_from_source_for_cdc(&session, &source_schema)?
+    } else {
+        bind_columns_from_source(&session, &source_schema, &with_properties).await?
     };
+    if is_shared {
+        // Note: this field should be called is_shared. Check field doc for more details.
+        source_info.cdc_source_job = true;
+        source_info.is_distributed = !create_cdc_source_job;
+    }
+    let mut col_id_gen = ColumnIdGenerator::new_initial();
+
+    let (source_catalog, database_id, schema_id) = bind_create_source(
+        handler_args.clone(),
+        stmt.source_name,
+        source_schema,
+        with_properties,
+        &stmt.columns,
+        stmt.constraints,
+        stmt.wildcard_idx,
+        stmt.source_watermarks,
+        columns_from_resolve_source,
+        source_info,
+        stmt.include_column_options,
+        &mut col_id_gen,
+        true,
+    )
+    .await?;
+
+    let source = source_catalog.to_prost(schema_id, database_id);
 
     let catalog_writer = session.catalog_writer()?;
 
@@ -1444,7 +1520,7 @@ pub async fn handle_create_source(
         let graph = {
             let context = OptimizerContext::from_handler_args(handler_args);
             let source_node = LogicalSource::with_catalog(
-                Rc::new(SourceCatalog::from(&source)),
+                Rc::new(source_catalog),
                 SourceNodeKind::CreateSharedSource,
                 context.into(),
                 None,
@@ -1494,6 +1570,7 @@ fn row_encode_to_prost(row_encode: &Encode) -> EncodeType {
         Encode::Bytes => EncodeType::Bytes,
         Encode::Template => EncodeType::Template,
         Encode::None => EncodeType::None,
+        Encode::Text => EncodeType::Text,
     }
 }
 
@@ -1631,7 +1708,7 @@ pub mod tests {
         );
 
         // Options are not merged into props.
-        assert!(source.with_properties.get("schema.location").is_none());
+        assert!(!source.with_properties.contains_key("schema.location"));
     }
 
     #[tokio::test]
diff --git a/src/frontend/src/handler/create_sql_function.rs b/src/frontend/src/handler/create_sql_function.rs
index ae7b2730e25d5..a9ac2f394c68f 100644
--- a/src/frontend/src/handler/create_sql_function.rs
+++ b/src/frontend/src/handler/create_sql_function.rs
@@ -15,15 +15,10 @@
 use std::collections::HashMap;
 
 use fancy_regex::Regex;
-use itertools::Itertools;
-use pgwire::pg_response::StatementType;
 use risingwave_common::catalog::FunctionId;
 use risingwave_common::types::DataType;
 use risingwave_pb::catalog::function::{Kind, ScalarFunction, TableFunction};
 use risingwave_pb::catalog::Function;
-use risingwave_sqlparser::ast::{
-    CreateFunctionBody, FunctionDefinition, ObjectName, OperateFunctionArg,
-};
 use risingwave_sqlparser::parser::{Parser, ParserError};
 
 use super::*;
diff --git a/src/frontend/src/handler/create_subscription.rs b/src/frontend/src/handler/create_subscription.rs
index 371806cc93a48..8d4ed82cc82ee 100644
--- a/src/frontend/src/handler/create_subscription.rs
+++ b/src/frontend/src/handler/create_subscription.rs
@@ -17,94 +17,56 @@ use std::rc::Rc;
 use either::Either;
 use pgwire::pg_response::{PgResponse, StatementType};
 use risingwave_common::catalog::UserId;
-use risingwave_pb::stream_plan::stream_fragment_graph::Parallelism;
-use risingwave_sqlparser::ast::{CreateSubscriptionStatement, Query};
+use risingwave_sqlparser::ast::CreateSubscriptionStatement;
 
-use super::privilege::resolve_query_privileges;
-use super::util::gen_query_from_table_name;
 use super::{HandlerArgs, RwPgResponse};
-use crate::catalog::subscription_catalog::SubscriptionCatalog;
+use crate::catalog::subscription_catalog::{SubscriptionCatalog, SubscriptionId};
 use crate::error::Result;
-use crate::optimizer::RelationCollectorVisitor;
 use crate::scheduler::streaming_manager::CreatingStreamingJobInfo;
 use crate::session::SessionImpl;
-use crate::{
-    build_graph, Binder, Explain, OptimizerContext, OptimizerContextRef, PlanRef, Planner,
-};
+use crate::{Binder, OptimizerContext, OptimizerContextRef};
 
-// used to store result of `gen_subscription_plan`
-pub struct SubscriptionPlanContext {
-    pub query: Box<Query>,
-    pub subscription_plan: PlanRef,
-    pub subscription_catalog: SubscriptionCatalog,
-}
-
-pub fn gen_subscription_plan(
+pub fn create_subscription_catalog(
     session: &SessionImpl,
     context: OptimizerContextRef,
     stmt: CreateSubscriptionStatement,
-) -> Result<SubscriptionPlanContext> {
+) -> Result<SubscriptionCatalog> {
     let db_name = session.database();
-    let (schema_name, subscription_name) =
+    let (subscription_schema_name, subscription_name) =
         Binder::resolve_schema_qualified_name(db_name, stmt.subscription_name.clone())?;
-    let subscription_from_table_name = stmt
-        .subscription_from
-        .0
-        .last()
-        .unwrap()
-        .real_value()
-        .clone();
-    let query = Box::new(gen_query_from_table_name(stmt.subscription_from));
-
-    let (database_id, schema_id) =
-        session.get_database_and_schema_id_for_create(schema_name.clone())?;
-
+    let (table_schema_name, subscription_from_table_name) =
+        Binder::resolve_schema_qualified_name(db_name, stmt.subscription_from.clone())?;
+    let (table_database_id, table_schema_id) =
+        session.get_database_and_schema_id_for_create(table_schema_name.clone())?;
+    let (subscription_database_id, subscription_schema_id) =
+        session.get_database_and_schema_id_for_create(subscription_schema_name.clone())?;
     let definition = context.normalized_sql().to_owned();
-
-    let (dependent_relations, bound) = {
-        let mut binder = Binder::new_for_stream(session);
-        let bound = binder.bind_query(*query.clone())?;
-        (binder.included_relations(), bound)
-    };
-
-    let check_items = resolve_query_privileges(&bound);
-    session.check_privileges(&check_items)?;
-
-    let with_options = context.with_options().clone();
-    let mut plan_root = Planner::new(context).plan_query(bound)?;
-
-    let subscription_plan = plan_root.gen_subscription_plan(
-        database_id,
-        schema_id,
-        dependent_relations.clone(),
-        subscription_name,
+    let dependent_table_id = session
+        .get_table_by_name(
+            &subscription_from_table_name,
+            table_database_id,
+            table_schema_id,
+        )?
+        .id;
+
+    let mut subscription_catalog = SubscriptionCatalog {
+        id: SubscriptionId::placeholder(),
+        name: subscription_name,
         definition,
-        with_options,
-        false,
-        subscription_from_table_name,
-        UserId::new(session.user_id()),
-    )?;
-
-    let subscription_catalog = subscription_plan.subscription_catalog();
-
-    let subscription_plan: PlanRef = subscription_plan.into();
-
-    let ctx = subscription_plan.ctx();
-    let explain_trace = ctx.is_explain_trace();
-    if explain_trace {
-        ctx.trace("Create Subscription:");
-        ctx.trace(subscription_plan.explain_to_string());
-    }
+        retention_seconds: 0,
+        database_id: subscription_database_id,
+        schema_id: subscription_schema_id,
+        dependent_table_id,
+        owner: UserId::new(session.user_id()),
+        initialized_at_epoch: None,
+        created_at_epoch: None,
+        created_at_cluster_version: None,
+        initialized_at_cluster_version: None,
+    };
 
-    let dependent_relations =
-        RelationCollectorVisitor::collect_with(dependent_relations, subscription_plan.clone());
+    subscription_catalog.set_retention_seconds(context.with_options())?;
 
-    let subscription_catalog = subscription_catalog.add_dependent_relations(dependent_relations);
-    Ok(SubscriptionPlanContext {
-        query,
-        subscription_plan,
-        subscription_catalog,
-    })
+    Ok(subscription_catalog)
 }
 
 pub async fn handle_create_subscription(
@@ -120,27 +82,9 @@ pub async fn handle_create_subscription(
     )? {
         return Ok(resp);
     };
-
-    let (subscription_catalog, graph) = {
+    let subscription_catalog = {
         let context = Rc::new(OptimizerContext::from_handler_args(handle_args));
-
-        let SubscriptionPlanContext {
-            query: _,
-            subscription_plan,
-            subscription_catalog,
-        } = gen_subscription_plan(&session, context.clone(), stmt)?;
-
-        let mut graph = build_graph(subscription_plan)?;
-
-        graph.parallelism =
-            session
-                .config()
-                .streaming_parallelism()
-                .map(|parallelism| Parallelism {
-                    parallelism: parallelism.get(),
-                });
-
-        (subscription_catalog, graph)
+        create_subscription_catalog(&session, context.clone(), stmt)?
     };
 
     let _job_guard =
@@ -156,7 +100,7 @@ pub async fn handle_create_subscription(
 
     let catalog_writer = session.catalog_writer()?;
     catalog_writer
-        .create_subscription(subscription_catalog.to_proto(), graph)
+        .create_subscription(subscription_catalog.to_proto())
         .await?;
 
     Ok(PgResponse::empty_result(StatementType::CREATE_SUBSCRIPTION))
diff --git a/src/frontend/src/handler/create_table.rs b/src/frontend/src/handler/create_table.rs
index 9c1290b59ba50..0f3693653ced5 100644
--- a/src/frontend/src/handler/create_table.rs
+++ b/src/frontend/src/handler/create_table.rs
@@ -14,10 +14,9 @@
 
 use std::collections::{BTreeMap, HashMap};
 use std::rc::Rc;
-use std::str::FromStr;
 use std::sync::Arc;
 
-use anyhow::anyhow;
+use anyhow::{anyhow, Context};
 use either::Either;
 use fixedbitset::FixedBitSet;
 use itertools::Itertools;
@@ -25,17 +24,15 @@ use pgwire::pg_response::{PgResponse, StatementType};
 use risingwave_common::bail_not_implemented;
 use risingwave_common::catalog::{
     CdcTableDesc, ColumnCatalog, ColumnDesc, TableId, TableVersionId, DEFAULT_SCHEMA_NAME,
-    INITIAL_SOURCE_VERSION_ID, INITIAL_TABLE_VERSION_ID, USER_COLUMN_ID_OFFSET,
+    INITIAL_TABLE_VERSION_ID, USER_COLUMN_ID_OFFSET,
 };
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use risingwave_common::util::value_encoding::DatumToProtoExt;
+use risingwave_connector::source;
 use risingwave_connector::source::cdc::external::{
-    DATABASE_NAME_KEY, SCHEMA_NAME_KEY, TABLE_NAME_KEY,
+    ExternalTableConfig, ExternalTableImpl, DATABASE_NAME_KEY, SCHEMA_NAME_KEY, TABLE_NAME_KEY,
 };
-use risingwave_connector::source::cdc::CDC_BACKFILL_ENABLE_KEY;
-use risingwave_connector::{source, WithPropertiesExt};
-use risingwave_pb::catalog::source::OptionalAssociatedTableId;
-use risingwave_pb::catalog::{PbSource, PbTable, StreamSourceInfo, Table, WatermarkDesc};
+use risingwave_pb::catalog::{PbSource, PbTable, Table, WatermarkDesc};
 use risingwave_pb::ddl_service::TableJobType;
 use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn;
 use risingwave_pb::plan_common::{
@@ -54,21 +51,20 @@ use crate::binder::{bind_data_type, bind_struct_field, Clause};
 use crate::catalog::root_catalog::SchemaPath;
 use crate::catalog::source_catalog::SourceCatalog;
 use crate::catalog::table_catalog::TableVersion;
-use crate::catalog::{check_valid_column_name, ColumnId};
+use crate::catalog::{check_valid_column_name, ColumnId, DatabaseId, SchemaId};
 use crate::error::{ErrorCode, Result, RwError};
 use crate::expr::{Expr, ExprImpl, ExprRewriter, InlineNowProcTime};
 use crate::handler::create_source::{
-    bind_all_columns, bind_columns_from_source, bind_source_pk, bind_source_watermark,
-    check_source_schema, handle_addition_columns, validate_compatibility, UPSTREAM_SOURCE_KEY,
+    bind_columns_from_source, bind_connector_props, bind_create_source, bind_source_watermark,
+    handle_addition_columns, UPSTREAM_SOURCE_KEY,
 };
 use crate::handler::HandlerArgs;
-use crate::optimizer::plan_node::generic::SourceNodeKind;
+use crate::optimizer::plan_node::generic::{CdcScanOptions, SourceNodeKind};
 use crate::optimizer::plan_node::{LogicalCdcScan, LogicalSource};
 use crate::optimizer::property::{Order, RequiredDist};
 use crate::optimizer::{OptimizerContext, OptimizerContextRef, PlanRef, PlanRoot};
 use crate::session::SessionImpl;
 use crate::stream_fragmenter::build_graph;
-use crate::utils::resolve_privatelink_in_with_option;
 use crate::{Binder, TableCatalog, WithOptions};
 
 /// Column ID generator for a new table or a new version of an existing table to alter.
@@ -482,85 +478,44 @@ pub(crate) async fn gen_create_table_plan_with_source(
     }
 
     let session = &handler_args.session;
-    let mut with_properties = handler_args.with_options.clone().into_connector_props();
-    validate_compatibility(&source_schema, &mut with_properties)?;
-
-    ensure_table_constraints_supported(&constraints)?;
-
-    let sql_pk_names = bind_sql_pk_names(&column_defs, &constraints)?;
+    let with_properties = bind_connector_props(&handler_args, &source_schema, false)?;
 
     let (columns_from_resolve_source, source_info) =
         bind_columns_from_source(session, &source_schema, &with_properties).await?;
-    let columns_from_sql = bind_sql_columns(&column_defs)?;
 
-    let mut columns = bind_all_columns(
-        &source_schema,
-        columns_from_resolve_source,
-        columns_from_sql,
+    let (source_catalog, database_id, schema_id) = bind_create_source(
+        handler_args.clone(),
+        table_name,
+        source_schema,
+        with_properties,
         &column_defs,
+        constraints,
         wildcard_idx,
-    )?;
-
-    // add additional columns before bind pk, because `format upsert` requires the key column
-    handle_addition_columns(&with_properties, include_column_options, &mut columns)?;
-    let pk_names = bind_source_pk(
-        &source_schema,
-        &source_info,
-        &mut columns,
-        sql_pk_names,
-        &with_properties,
+        source_watermarks,
+        columns_from_resolve_source,
+        source_info,
+        include_column_options,
+        &mut col_id_gen,
+        false,
     )
     .await?;
 
-    for c in &mut columns {
-        c.column_desc.column_id = col_id_gen.generate(c.name())
-    }
-
-    if with_properties.is_iceberg_connector() {
-        return Err(
-            ErrorCode::BindError("can't create table with iceberg connector".to_string()).into(),
-        );
-    }
-    let (mut columns, pk_column_ids, row_id_index) = bind_pk_on_relation(columns, pk_names, true)?;
-
-    let watermark_descs = bind_source_watermark(
-        session,
-        table_name.real_value(),
-        source_watermarks,
-        &columns,
-    )?;
-    // TODO(yuhao): allow multiple watermark on source.
-    assert!(watermark_descs.len() <= 1);
-
-    let definition = handler_args.normalized_sql.clone();
-
-    bind_sql_column_constraints(
-        session,
-        table_name.real_value(),
-        &mut columns,
-        column_defs,
-        &pk_column_ids,
-    )?;
-
-    check_source_schema(&with_properties, row_id_index, &columns).await?;
+    let pb_source = source_catalog.to_prost(schema_id, database_id);
 
     let context = OptimizerContext::new(handler_args, explain_options);
 
-    gen_table_plan_inner(
+    let (plan, table) = gen_table_plan_with_source(
         context.into(),
-        table_name,
-        columns,
-        with_properties,
-        pk_column_ids,
-        row_id_index,
-        Some(source_info),
-        definition,
-        watermark_descs,
+        source_catalog,
         append_only,
         on_conflict,
         with_version_column,
         Some(col_id_gen.into_version()),
-    )
+        database_id,
+        schema_id,
+    )?;
+
+    Ok((plan, Some(pb_source), table))
 }
 
 /// `gen_create_table_plan` generates the plan for creating a table without an external stream
@@ -575,14 +530,14 @@ pub(crate) fn gen_create_table_plan(
     append_only: bool,
     on_conflict: Option<OnConflict>,
     with_version_column: Option<String>,
-) -> Result<(PlanRef, Option<PbSource>, PbTable)> {
+) -> Result<(PlanRef, PbTable)> {
     let definition = context.normalized_sql().to_owned();
     let mut columns = bind_sql_columns(&column_defs)?;
     for c in &mut columns {
         c.column_desc.column_id = col_id_gen.generate(c.name())
     }
-    let with_properties = context.with_options().inner().clone().into_iter().collect();
-    gen_create_table_plan_without_bind(
+    let with_properties = context.with_options().inner().clone();
+    gen_create_table_plan_without_source(
         context,
         table_name,
         columns,
@@ -599,25 +554,25 @@ pub(crate) fn gen_create_table_plan(
 }
 
 #[allow(clippy::too_many_arguments)]
-pub(crate) fn gen_create_table_plan_without_bind(
+pub(crate) fn gen_create_table_plan_without_source(
     context: OptimizerContext,
     table_name: ObjectName,
     columns: Vec<ColumnCatalog>,
     column_defs: Vec<ColumnDef>,
     constraints: Vec<TableConstraint>,
-    with_properties: HashMap<String, String>,
+    with_properties: BTreeMap<String, String>,
     definition: String,
     source_watermarks: Vec<SourceWatermark>,
     append_only: bool,
     on_conflict: Option<OnConflict>,
     with_version_column: Option<String>,
     version: Option<TableVersion>,
-) -> Result<(PlanRef, Option<PbSource>, PbTable)> {
+) -> Result<(PlanRef, PbTable)> {
     ensure_table_constraints_supported(&constraints)?;
     let pk_names = bind_sql_pk_names(&column_defs, &constraints)?;
     let (mut columns, pk_column_ids, row_id_index) = bind_pk_on_relation(columns, pk_names, true)?;
 
-    let watermark_descs = bind_source_watermark(
+    let watermark_descs: Vec<WatermarkDesc> = bind_source_watermark(
         context.session_ctx(),
         table_name.real_value(),
         source_watermarks,
@@ -631,33 +586,71 @@ pub(crate) fn gen_create_table_plan_without_bind(
         column_defs,
         &pk_column_ids,
     )?;
+    let session = context.session_ctx().clone();
+
+    let db_name = session.database();
+    let (schema_name, name) = Binder::resolve_schema_qualified_name(db_name, table_name)?;
+    let (database_id, schema_id) =
+        session.get_database_and_schema_id_for_create(schema_name.clone())?;
 
     gen_table_plan_inner(
         context.into(),
-        table_name,
+        name,
         columns,
         with_properties,
         pk_column_ids,
         row_id_index,
-        None,
         definition,
         watermark_descs,
         append_only,
         on_conflict,
         with_version_column,
         version,
+        None,
+        database_id,
+        schema_id,
+    )
+}
+
+fn gen_table_plan_with_source(
+    context: OptimizerContextRef,
+    source_catalog: SourceCatalog,
+    append_only: bool,
+    on_conflict: Option<OnConflict>,
+    with_version_column: Option<String>,
+    version: Option<TableVersion>, /* TODO: this should always be `Some` if we support `ALTER
+                                    * TABLE` for `CREATE TABLE AS`. */
+    database_id: DatabaseId,
+    schema_id: SchemaId,
+) -> Result<(PlanRef, PbTable)> {
+    let cloned_source_catalog = source_catalog.clone();
+    gen_table_plan_inner(
+        context,
+        source_catalog.name,
+        source_catalog.columns,
+        source_catalog.with_properties.clone(),
+        source_catalog.pk_col_ids,
+        source_catalog.row_id_index,
+        source_catalog.definition,
+        source_catalog.watermark_descs,
+        append_only,
+        on_conflict,
+        with_version_column,
+        version,
+        Some(cloned_source_catalog),
+        database_id,
+        schema_id,
     )
 }
 
 #[allow(clippy::too_many_arguments)]
 fn gen_table_plan_inner(
     context: OptimizerContextRef,
-    table_name: ObjectName,
+    table_name: String,
     columns: Vec<ColumnCatalog>,
-    with_properties: HashMap<String, String>,
+    with_properties: BTreeMap<String, String>,
     pk_column_ids: Vec<ColumnId>,
     row_id_index: Option<usize>,
-    source_info: Option<StreamSourceInfo>,
     definition: String,
     watermark_descs: Vec<WatermarkDesc>,
     append_only: bool,
@@ -665,51 +658,16 @@ fn gen_table_plan_inner(
     with_version_column: Option<String>,
     version: Option<TableVersion>, /* TODO: this should always be `Some` if we support `ALTER
                                     * TABLE` for `CREATE TABLE AS`. */
-) -> Result<(PlanRef, Option<PbSource>, PbTable)> {
+    source_catalog: Option<SourceCatalog>,
+    database_id: DatabaseId,
+    schema_id: SchemaId,
+) -> Result<(PlanRef, PbTable)> {
     let session = context.session_ctx().clone();
-    let db_name = session.database();
-    let (schema_name, name) = Binder::resolve_schema_qualified_name(db_name, table_name)?;
-    let (database_id, schema_id) =
-        session.get_database_and_schema_id_for_create(schema_name.clone())?;
-
-    // resolve privatelink connection for Table backed by Kafka source
-    let mut with_properties = WithOptions::new(with_properties);
-    let connection_id =
-        resolve_privatelink_in_with_option(&mut with_properties, &schema_name, &session)?;
+    let with_properties = WithOptions::new(with_properties);
     let retention_seconds = with_properties.retention_seconds();
-
-    let is_external_source = source_info.is_some();
-
-    let source = source_info.map(|source_info| PbSource {
-        id: TableId::placeholder().table_id,
-        schema_id,
-        database_id,
-        name: name.clone(),
-        row_id_index: row_id_index.map(|i| i as _),
-        columns: columns
-            .iter()
-            .map(|column| column.to_protobuf())
-            .collect_vec(),
-        pk_column_ids: pk_column_ids.iter().map(Into::into).collect_vec(),
-        with_properties: with_properties.into_inner().into_iter().collect(),
-        info: Some(source_info),
-        owner: session.user_id(),
-        watermark_descs: watermark_descs.clone(),
-        definition: "".to_string(),
-        connection_id,
-        initialized_at_epoch: None,
-        created_at_epoch: None,
-        optional_associated_table_id: Some(OptionalAssociatedTableId::AssociatedTableId(
-            TableId::placeholder().table_id,
-        )),
-        version: INITIAL_SOURCE_VERSION_ID,
-        initialized_at_cluster_version: None,
-        created_at_cluster_version: None,
-    });
-
-    let source_catalog = source.as_ref().map(|source| Rc::new((source).into()));
+    let is_external_source = source_catalog.is_some();
     let source_node: PlanRef = LogicalSource::new(
-        source_catalog.clone(),
+        source_catalog.map(|source| Rc::new(source.clone())),
         columns.clone(),
         row_id_index,
         SourceNodeKind::CreateTable,
@@ -719,7 +677,7 @@ fn gen_table_plan_inner(
     .into();
 
     let required_cols = FixedBitSet::with_capacity(columns.len());
-    let mut plan_root = PlanRoot::new(
+    let plan_root = PlanRoot::new_with_logical_plan(
         source_node,
         RequiredDist::Any,
         Order::any(),
@@ -752,7 +710,7 @@ fn gen_table_plan_inner(
 
     let materialize = plan_root.gen_table_plan(
         context,
-        name,
+        table_name,
         columns,
         definition,
         pk_column_ids,
@@ -769,52 +727,42 @@ fn gen_table_plan_inner(
     let mut table = materialize.table().to_prost(schema_id, database_id);
 
     table.owner = session.user_id();
-    Ok((materialize.into(), source, table))
+    Ok((materialize.into(), table))
 }
 
 #[allow(clippy::too_many_arguments)]
-pub(crate) fn gen_create_table_plan_for_cdc_source(
-    context: OptimizerContextRef,
-    source_name: ObjectName,
-    table_name: ObjectName,
+pub(crate) fn gen_create_table_plan_for_cdc_table(
+    handler_args: HandlerArgs,
+    explain_options: ExplainOptions,
+    source: Arc<SourceCatalog>,
     external_table_name: String,
-    column_defs: Vec<ColumnDef>,
-    constraints: Vec<TableConstraint>,
+    mut columns: Vec<ColumnCatalog>,
+    pk_names: Vec<String>,
+    connect_properties: BTreeMap<String, String>,
     mut col_id_gen: ColumnIdGenerator,
     on_conflict: Option<OnConflict>,
     with_version_column: Option<String>,
+    include_column_options: IncludeOption,
+    resolved_table_name: String,
+    database_id: DatabaseId,
+    schema_id: SchemaId,
 ) -> Result<(PlanRef, PbTable)> {
+    let context: OptimizerContextRef = OptimizerContext::new(handler_args, explain_options).into();
     let session = context.session_ctx().clone();
-    let db_name = session.database();
-    let (schema_name, name) = Binder::resolve_schema_qualified_name(db_name, table_name)?;
-    let (database_id, schema_id) =
-        session.get_database_and_schema_id_for_create(schema_name.clone())?;
-
-    // cdc table cannot be append-only
-    let append_only = false;
-    let (source_schema, source_name) = Binder::resolve_schema_qualified_name(db_name, source_name)?;
 
-    let source = {
-        let catalog_reader = session.env().catalog_reader().read_guard();
-        let schema_name = source_schema
-            .clone()
-            .unwrap_or(DEFAULT_SCHEMA_NAME.to_string());
-        let (source, _) = catalog_reader.get_source_by_name(
-            db_name,
-            SchemaPath::Name(schema_name.as_str()),
-            source_name.as_str(),
-        )?;
-        source.clone()
-    };
-
-    let mut columns = bind_sql_columns(&column_defs)?;
+    // append additional columns to the end
+    handle_addition_columns(
+        &connect_properties,
+        include_column_options,
+        &mut columns,
+        true,
+    )?;
 
     for c in &mut columns {
         c.column_desc.column_id = col_id_gen.generate(c.name())
     }
 
-    let pk_names = bind_sql_pk_names(&column_defs, &constraints)?;
-    let (columns, pk_column_ids, _) = bind_pk_on_relation(columns, pk_names, true)?;
+    let (columns, pk_column_ids, _row_id_index) = bind_pk_on_relation(columns, pk_names, true)?;
 
     let definition = context.normalized_sql().to_owned();
 
@@ -834,9 +782,6 @@ pub(crate) fn gen_create_table_plan_for_cdc_source(
         .map(|idx| ColumnOrder::new(*idx, OrderType::ascending()))
         .collect();
 
-    let connect_properties =
-        derive_connect_properties(source.as_ref(), external_table_name.clone())?;
-
     let cdc_table_desc = CdcTableDesc {
         table_id: TableId::placeholder(), // will be filled in meta node
         source_id: source.id.into(),      // id of cdc source streaming job
@@ -844,31 +789,23 @@ pub(crate) fn gen_create_table_plan_for_cdc_source(
         pk: table_pk,
         columns: columns.iter().map(|c| c.column_desc.clone()).collect(),
         stream_key: pk_column_indices,
-        value_indices: (0..columns.len()).collect_vec(),
-        connect_properties,
+        connect_properties: connect_properties.into_iter().collect(),
     };
 
     tracing::debug!(?cdc_table_desc, "create cdc table");
 
-    // disable backfill if 'snapshot=false'
-    let disable_backfill = match context.with_options().get(CDC_BACKFILL_ENABLE_KEY) {
-        None => false,
-        Some(v) => {
-            !(bool::from_str(v)
-                .map_err(|_| anyhow!("Invalid value for {}", CDC_BACKFILL_ENABLE_KEY))?)
-        }
-    };
+    let options = CdcScanOptions::from_with_options(context.with_options())?;
 
     let logical_scan = LogicalCdcScan::create(
         external_table_name,
         Rc::new(cdc_table_desc),
         context.clone(),
-        disable_backfill,
+        options,
     );
 
     let scan_node: PlanRef = logical_scan.into();
     let required_cols = FixedBitSet::with_capacity(columns.len());
-    let mut plan_root = PlanRoot::new(
+    let plan_root = PlanRoot::new_with_logical_plan(
         scan_node,
         RequiredDist::Any,
         Order::any(),
@@ -878,12 +815,12 @@ pub(crate) fn gen_create_table_plan_for_cdc_source(
 
     let materialize = plan_root.gen_table_plan(
         context,
-        name,
+        resolved_table_name,
         columns,
         definition,
         pk_column_ids,
         None,
-        append_only,
+        false,
         on_conflict,
         with_version_column,
         vec![],
@@ -900,13 +837,13 @@ pub(crate) fn gen_create_table_plan_for_cdc_source(
 }
 
 fn derive_connect_properties(
-    source: &SourceCatalog,
+    source_with_properties: &BTreeMap<String, String>,
     external_table_name: String,
 ) -> Result<BTreeMap<String, String>> {
     use source::cdc::{MYSQL_CDC_CONNECTOR, POSTGRES_CDC_CONNECTOR};
     // we should remove the prefix from `full_table_name`
-    let mut connect_properties = source.with_properties.clone();
-    if let Some(connector) = source.with_properties.get(UPSTREAM_SOURCE_KEY) {
+    let mut connect_properties = source_with_properties.clone();
+    if let Some(connector) = source_with_properties.get(UPSTREAM_SOURCE_KEY) {
         let table_name = match connector.as_str() {
             MYSQL_CDC_CONNECTOR => {
                 let db_name = connect_properties.get(DATABASE_NAME_KEY).ok_or_else(|| {
@@ -937,7 +874,7 @@ fn derive_connect_properties(
         };
         connect_properties.insert(TABLE_NAME_KEY.into(), table_name.into());
     }
-    Ok(connect_properties.into_iter().collect())
+    Ok(connect_properties)
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -961,6 +898,7 @@ pub(super) async fn handle_create_table_plan(
         &handler_args.with_options,
         source_schema,
         &include_column_options,
+        &cdc_table_info,
     )?;
 
     let ((plan, source, table), job_type) =
@@ -986,34 +924,81 @@ pub(super) async fn handle_create_table_plan(
             ),
             (None, None) => {
                 let context = OptimizerContext::new(handler_args, explain_options);
-                (
-                    gen_create_table_plan(
-                        context,
-                        table_name.clone(),
-                        column_defs,
-                        constraints,
-                        col_id_gen,
-                        source_watermarks,
-                        append_only,
-                        on_conflict,
-                        with_version_column,
-                    )?,
-                    TableJobType::General,
-                )
+                let (plan, table) = gen_create_table_plan(
+                    context,
+                    table_name.clone(),
+                    column_defs,
+                    constraints,
+                    col_id_gen,
+                    source_watermarks,
+                    append_only,
+                    on_conflict,
+                    with_version_column,
+                )?;
+
+                ((plan, None, table), TableJobType::General)
             }
 
             (None, Some(cdc_table)) => {
-                let context = OptimizerContext::new(handler_args, explain_options);
-                let (plan, table) = gen_create_table_plan_for_cdc_source(
-                    context.into(),
-                    cdc_table.source_name.clone(),
-                    table_name.clone(),
+                sanity_check_for_cdc_table(
+                    append_only,
+                    &column_defs,
+                    &wildcard_idx,
+                    &constraints,
+                    &source_watermarks,
+                )?;
+
+                let session = &handler_args.session;
+                let db_name = session.database();
+                let (schema_name, resolved_table_name) =
+                    Binder::resolve_schema_qualified_name(db_name, table_name)?;
+                let (database_id, schema_id) =
+                    session.get_database_and_schema_id_for_create(schema_name.clone())?;
+
+                // cdc table cannot be append-only
+                let (source_schema, source_name) =
+                    Binder::resolve_schema_qualified_name(db_name, cdc_table.source_name.clone())?;
+
+                let source = {
+                    let catalog_reader = session.env().catalog_reader().read_guard();
+                    let schema_name = source_schema
+                        .clone()
+                        .unwrap_or(DEFAULT_SCHEMA_NAME.to_string());
+                    let (source, _) = catalog_reader.get_source_by_name(
+                        db_name,
+                        SchemaPath::Name(schema_name.as_str()),
+                        source_name.as_str(),
+                    )?;
+                    source.clone()
+                };
+                let connect_properties = derive_connect_properties(
+                    &source.with_properties,
                     cdc_table.external_table_name.clone(),
-                    column_defs,
-                    constraints,
+                )?;
+
+                let (columns, pk_names) = derive_schema_for_cdc_table(
+                    &column_defs,
+                    &constraints,
+                    connect_properties.clone(),
+                    wildcard_idx.is_some(),
+                )
+                .await?;
+
+                let (plan, table) = gen_create_table_plan_for_cdc_table(
+                    handler_args,
+                    explain_options,
+                    source,
+                    cdc_table.external_table_name.clone(),
+                    columns,
+                    pk_names,
+                    connect_properties,
                     col_id_gen,
                     on_conflict,
                     with_version_column,
+                    include_column_options,
+                    resolved_table_name,
+                    database_id,
+                    schema_id,
                 )?;
 
                 ((plan, None, table), TableJobType::SharedCdcSource)
@@ -1028,6 +1013,110 @@ pub(super) async fn handle_create_table_plan(
     Ok((plan, source, table, job_type))
 }
 
+fn sanity_check_for_cdc_table(
+    append_only: bool,
+    column_defs: &Vec<ColumnDef>,
+    wildcard_idx: &Option<usize>,
+    constraints: &Vec<TableConstraint>,
+    source_watermarks: &Vec<SourceWatermark>,
+) -> Result<()> {
+    for c in column_defs {
+        for op in &c.options {
+            if let ColumnOption::GeneratedColumns(_) = op.option {
+                return Err(ErrorCode::NotSupported(
+                    "generated column defined on the table created from a CDC source".into(),
+                    "Remove the generated column in the column list".into(),
+                )
+                .into());
+            }
+        }
+    }
+
+    // wildcard cannot be used with column definitions
+    if wildcard_idx.is_some() && !column_defs.is_empty() {
+        return Err(ErrorCode::NotSupported(
+            "wildcard(*) and column definitions cannot be used together".to_owned(),
+            "Remove the wildcard or column definitions".to_owned(),
+        )
+        .into());
+    }
+
+    // cdc table must have primary key constraint or primary key column
+    if !wildcard_idx.is_some()
+        && !constraints.iter().any(|c| {
+            matches!(
+                c,
+                TableConstraint::Unique {
+                    is_primary: true,
+                    ..
+                }
+            )
+        })
+        && !column_defs.iter().any(|col| {
+            col.options
+                .iter()
+                .any(|opt| matches!(opt.option, ColumnOption::Unique { is_primary: true }))
+        })
+    {
+        return Err(ErrorCode::NotSupported(
+            "CDC table without primary key constraint is not supported".to_owned(),
+            "Please define a primary key".to_owned(),
+        )
+        .into());
+    }
+    if append_only {
+        return Err(ErrorCode::NotSupported(
+            "append only modifier on the table created from a CDC source".into(),
+            "Remove the APPEND ONLY clause".into(),
+        )
+        .into());
+    }
+
+    if !source_watermarks.is_empty() {
+        return Err(ErrorCode::NotSupported(
+            "watermark defined on the table created from a CDC source".into(),
+            "Remove the Watermark definitions".into(),
+        )
+        .into());
+    }
+
+    Ok(())
+}
+
+async fn derive_schema_for_cdc_table(
+    column_defs: &Vec<ColumnDef>,
+    constraints: &Vec<TableConstraint>,
+    connect_properties: BTreeMap<String, String>,
+    need_auto_schema_map: bool,
+) -> Result<(Vec<ColumnCatalog>, Vec<String>)> {
+    // read cdc table schema from external db or parsing the schema from SQL definitions
+    if need_auto_schema_map {
+        let config = ExternalTableConfig::try_from_btreemap(connect_properties)
+            .context("failed to extract external table config")?;
+
+        let table = ExternalTableImpl::connect(config)
+            .await
+            .context("failed to auto derive table schema")?;
+        Ok((
+            table
+                .column_descs()
+                .iter()
+                .cloned()
+                .map(|column_desc| ColumnCatalog {
+                    column_desc,
+                    is_hidden: false,
+                })
+                .collect(),
+            table.pk_names().clone(),
+        ))
+    } else {
+        Ok((
+            bind_sql_columns(column_defs)?,
+            bind_sql_pk_names(column_defs, constraints)?,
+        ))
+    }
+}
+
 #[allow(clippy::too_many_arguments)]
 pub async fn handle_create_table(
     handler_args: HandlerArgs,
@@ -1107,7 +1196,12 @@ pub fn check_create_table_with_source(
     with_options: &WithOptions,
     source_schema: Option<ConnectorSchema>,
     include_column_options: &IncludeOption,
+    cdc_table_info: &Option<CdcTableInfo>,
 ) -> Result<Option<ConnectorSchema>> {
+    // skip check for cdc table
+    if cdc_table_info.is_some() {
+        return Ok(source_schema);
+    }
     let defined_source = with_options.inner().contains_key(UPSTREAM_SOURCE_KEY);
     if !include_column_options.is_empty() && !defined_source {
         return Err(ErrorCode::InvalidInputSyntax(
@@ -1162,7 +1256,7 @@ pub async fn generate_stream_graph_for_table(
         }
         None => {
             let context = OptimizerContext::from_handler_args(handler_args);
-            gen_create_table_plan(
+            let (plan, table) = gen_create_table_plan(
                 context,
                 table_name,
                 columns,
@@ -1172,7 +1266,8 @@ pub async fn generate_stream_graph_for_table(
                 append_only,
                 on_conflict,
                 with_version_column,
-            )?
+            )?;
+            (plan, None, table)
         }
     };
 
@@ -1207,15 +1302,10 @@ pub async fn generate_stream_graph_for_table(
 
 #[cfg(test)]
 mod tests {
-    use std::collections::HashMap;
-
-    use risingwave_common::catalog::{
-        Field, DEFAULT_DATABASE_NAME, DEFAULT_SCHEMA_NAME, ROWID_PREFIX,
-    };
+    use risingwave_common::catalog::{Field, DEFAULT_DATABASE_NAME, ROWID_PREFIX};
     use risingwave_common::types::DataType;
 
     use super::*;
-    use crate::catalog::root_catalog::SchemaPath;
     use crate::test_utils::{create_proto_file, LocalFrontend, PROTO_FILE_DATA};
 
     #[test]
@@ -1432,6 +1522,6 @@ mod tests {
         );
 
         // Options are not merged into props.
-        assert!(source.with_properties.get("schema.location").is_none());
+        assert!(!source.with_properties.contains_key("schema.location"));
     }
 }
diff --git a/src/frontend/src/handler/create_table_as.rs b/src/frontend/src/handler/create_table_as.rs
index a828cb9708b48..bdd6e9d078e90 100644
--- a/src/frontend/src/handler/create_table_as.rs
+++ b/src/frontend/src/handler/create_table_as.rs
@@ -22,7 +22,7 @@ use risingwave_sqlparser::ast::{ColumnDef, ObjectName, OnConflict, Query, Statem
 use super::{HandlerArgs, RwPgResponse};
 use crate::binder::BoundStatement;
 use crate::error::{ErrorCode, Result};
-use crate::handler::create_table::{gen_create_table_plan_without_bind, ColumnIdGenerator};
+use crate::handler::create_table::{gen_create_table_plan_without_source, ColumnIdGenerator};
 use crate::handler::query::handle_query;
 use crate::{build_graph, Binder, OptimizerContext};
 pub async fn handle_create_as(
@@ -96,7 +96,7 @@ pub async fn handle_create_as(
             .clone()
             .into_iter()
             .collect();
-        let (plan, source, table) = gen_create_table_plan_without_bind(
+        let (plan, table) = gen_create_table_plan_without_source(
             context,
             table_name.clone(),
             columns,
@@ -118,7 +118,7 @@ pub async fn handle_create_as(
                 .map(|parallelism| Parallelism {
                     parallelism: parallelism.get(),
                 });
-        (graph, source, table)
+        (graph, None, table)
     };
 
     tracing::trace!(
diff --git a/src/frontend/src/handler/declare_cursor.rs b/src/frontend/src/handler/declare_cursor.rs
index 27f029bf3e4a5..25e146fa714ce 100644
--- a/src/frontend/src/handler/declare_cursor.rs
+++ b/src/frontend/src/handler/declare_cursor.rs
@@ -18,7 +18,7 @@ use risingwave_common::util::epoch::Epoch;
 use risingwave_sqlparser::ast::{DeclareCursorStatement, ObjectName, Query, Since, Statement};
 
 use super::query::{gen_batch_plan_by_statement, gen_batch_plan_fragmenter};
-use super::util::{convert_epoch_to_logstore_i64, convert_unix_millis_to_logstore_i64};
+use super::util::convert_unix_millis_to_logstore_u64;
 use super::RwPgResponse;
 use crate::error::{ErrorCode, Result};
 use crate::handler::query::create_stream;
@@ -61,16 +61,16 @@ async fn handle_declare_subscription_cursor(
     // Start the first query of cursor, which includes querying the table and querying the subscription's logstore
     let start_rw_timestamp = match rw_timestamp {
         Some(risingwave_sqlparser::ast::Since::TimestampMsNum(start_rw_timestamp)) => {
-            check_cursor_unix_millis(start_rw_timestamp, subscription.get_retention_seconds()?)?;
-            Some(convert_unix_millis_to_logstore_i64(start_rw_timestamp))
-        }
-        Some(risingwave_sqlparser::ast::Since::ProcessTime) => {
-            Some(convert_epoch_to_logstore_i64(Epoch::now().0))
+            check_cursor_unix_millis(start_rw_timestamp, subscription.retention_seconds)?;
+            Some(convert_unix_millis_to_logstore_u64(start_rw_timestamp))
         }
+        Some(risingwave_sqlparser::ast::Since::ProcessTime) => Some(Epoch::now().0),
         Some(risingwave_sqlparser::ast::Since::Begin) => {
             let min_unix_millis =
-                Epoch::now().as_unix_millis() - subscription.get_retention_seconds()? * 1000;
-            Some(convert_unix_millis_to_logstore_i64(min_unix_millis))
+                Epoch::now().as_unix_millis() - subscription.retention_seconds * 1000;
+            let subscription_build_millis = subscription.created_at_epoch.unwrap().as_unix_millis();
+            let min_unix_millis = std::cmp::max(min_unix_millis, subscription_build_millis);
+            Some(convert_unix_millis_to_logstore_u64(min_unix_millis))
         }
         None => None,
     };
@@ -80,6 +80,7 @@ async fn handle_declare_subscription_cursor(
         .add_subscription_cursor(
             cursor_name.clone(),
             start_rw_timestamp,
+            subscription.dependent_table_id,
             subscription,
             &handle_args,
         )
@@ -111,7 +112,7 @@ async fn handle_declare_query_cursor(
     query: Box<Query>,
 ) -> Result<RwPgResponse> {
     let (row_stream, pg_descs) =
-        create_stream_for_cursor(handle_args.clone(), Statement::Query(query)).await?;
+        create_stream_for_cursor_stmt(handle_args.clone(), Statement::Query(query)).await?;
     handle_args
         .session
         .get_cursor_manager()
@@ -120,7 +121,7 @@ async fn handle_declare_query_cursor(
     Ok(PgResponse::empty_result(StatementType::DECLARE_CURSOR))
 }
 
-pub async fn create_stream_for_cursor(
+pub async fn create_stream_for_cursor_stmt(
     handle_args: HandlerArgs,
     stmt: Statement,
 ) -> Result<(PgResponseStream, Vec<PgFieldDescriptor>)> {
diff --git a/src/frontend/src/handler/drop_function.rs b/src/frontend/src/handler/drop_function.rs
index 61eaa8d0c0ad0..945cf4816bf9e 100644
--- a/src/frontend/src/handler/drop_function.rs
+++ b/src/frontend/src/handler/drop_function.rs
@@ -12,23 +12,27 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use pgwire::pg_response::StatementType;
-use risingwave_sqlparser::ast::{FunctionDesc, ReferentialAction};
-
 use super::*;
 use crate::catalog::root_catalog::SchemaPath;
 use crate::catalog::CatalogError;
 use crate::{bind_data_type, Binder};
 
+/// Drop a function or an aggregate.
 pub async fn handle_drop_function(
     handler_args: HandlerArgs,
     if_exists: bool,
     mut func_desc: Vec<FunctionDesc>,
     _option: Option<ReferentialAction>,
+    aggregate: bool,
 ) -> Result<RwPgResponse> {
     if func_desc.len() != 1 {
         bail_not_implemented!("only support dropping 1 function");
     }
+    let stmt_type = if aggregate {
+        StatementType::DROP_AGGREGATE
+    } else {
+        StatementType::DROP_FUNCTION
+    };
     let func_desc = func_desc.remove(0);
 
     let session = handler_args.session;
@@ -73,10 +77,21 @@ pub async fn handle_drop_function(
         match res {
             Ok((function, schema_name)) => {
                 session.check_privilege_for_drop_alter(schema_name, &**function)?;
+                if !aggregate && function.kind.is_aggregate() {
+                    return Err(ErrorCode::CatalogError(
+                        format!("\"{function_name}\" is an aggregate function\nHINT:  Use DROP AGGREGATE to drop aggregate functions.").into(),
+                    )
+                    .into());
+                } else if aggregate && !function.kind.is_aggregate() {
+                    return Err(ErrorCode::CatalogError(
+                        format!("\"{function_name}\" is not an aggregate").into(),
+                    )
+                    .into());
+                }
                 function.id
             }
             Err(CatalogError::NotFound(kind, _)) if kind == "function" && if_exists => {
-                return Ok(RwPgResponse::builder(StatementType::DROP_FUNCTION)
+                return Ok(RwPgResponse::builder(stmt_type)
                     .notice(format!(
                         "function \"{}\" does not exist, skipping",
                         function_name
@@ -90,5 +105,5 @@ pub async fn handle_drop_function(
     let catalog_writer = session.catalog_writer()?;
     catalog_writer.drop_function(function_id).await?;
 
-    Ok(PgResponse::empty_result(StatementType::DROP_FUNCTION))
+    Ok(PgResponse::empty_result(stmt_type))
 }
diff --git a/src/frontend/src/handler/drop_secret.rs b/src/frontend/src/handler/drop_secret.rs
new file mode 100644
index 0000000000000..37fbd2cedd408
--- /dev/null
+++ b/src/frontend/src/handler/drop_secret.rs
@@ -0,0 +1,64 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use pgwire::pg_response::StatementType;
+use risingwave_sqlparser::ast::ObjectName;
+
+use crate::catalog::root_catalog::SchemaPath;
+use crate::error::Result;
+use crate::handler::{HandlerArgs, RwPgResponse};
+use crate::Binder;
+
+pub async fn handle_drop_secret(
+    handler_args: HandlerArgs,
+    secret_name: ObjectName,
+    if_exists: bool,
+) -> Result<RwPgResponse> {
+    let session = handler_args.session;
+    let db_name = session.database();
+    let (schema_name, secret_name) = Binder::resolve_schema_qualified_name(db_name, secret_name)?;
+    let search_path = session.config().search_path();
+    let user_name = &session.auth_context().user_name;
+    let schema_path = SchemaPath::new(schema_name.as_deref(), &search_path, user_name);
+
+    let secret_id = {
+        let reader = session.env().catalog_reader().read_guard();
+        let (secret, schema_name) =
+            match reader.get_secret_by_name(db_name, schema_path, secret_name.as_str()) {
+                Ok((c, s)) => (c, s),
+                Err(e) => {
+                    return if if_exists {
+                        Ok(RwPgResponse::builder(StatementType::DROP_SECRET)
+                            .notice(format!(
+                                "secret \"{}\" does not exist, skipping",
+                                secret_name
+                            ))
+                            .into())
+                    } else {
+                        Err(e.into())
+                    };
+                }
+            };
+        session.check_privilege_for_drop_alter(schema_name, &**secret)?;
+
+        secret.secret_id
+    };
+
+    let catalog_writer = session.catalog_writer()?;
+    catalog_writer.drop_secret(secret_id).await?;
+
+    Ok(RwPgResponse::builder(StatementType::DROP_SECRET)
+        .notice(format!("dropped secret \"{}\"", secret_name))
+        .into())
+}
diff --git a/src/frontend/src/handler/explain.rs b/src/frontend/src/handler/explain.rs
index cfc416f0d8b0d..9f46087c206e8 100644
--- a/src/frontend/src/handler/explain.rs
+++ b/src/frontend/src/handler/explain.rs
@@ -22,7 +22,6 @@ use thiserror_ext::AsReport;
 use super::create_index::{gen_create_index_plan, resolve_index_schema};
 use super::create_mv::gen_create_mv_plan;
 use super::create_sink::{gen_sink_plan, get_partition_compute_info};
-use super::create_subscription::gen_subscription_plan;
 use super::create_table::ColumnIdGenerator;
 use super::query::gen_batch_plan_by_statement;
 use super::util::SourceSchemaCompatExt;
@@ -135,9 +134,11 @@ async fn do_handle_explain(
                         ).into());
                     }
 
-                    Statement::CreateSubscription { stmt } => {
-                        gen_subscription_plan(&session, context.clone(), stmt)
-                            .map(|plan| plan.subscription_plan)
+                    Statement::CreateSubscription { .. } => {
+                        return Err(ErrorCode::NotSupported(
+                            "EXPLAIN CREATE SUBSCRIPTION".into(),
+                            "A created SUBSCRIPTION only incremental data queries on the table, not supported EXPLAIN".into()
+                        ).into());
                     }
                     Statement::CreateIndex {
                         name,
diff --git a/src/frontend/src/handler/handle_privilege.rs b/src/frontend/src/handler/handle_privilege.rs
index 44e5b8fdac33c..e8a37b0ab407b 100644
--- a/src/frontend/src/handler/handle_privilege.rs
+++ b/src/frontend/src/handler/handle_privilege.rs
@@ -21,6 +21,7 @@ use super::RwPgResponse;
 use crate::binder::Binder;
 use crate::catalog::root_catalog::SchemaPath;
 use crate::catalog::table_catalog::TableType;
+use crate::catalog::CatalogError;
 use crate::error::{ErrorCode, Result};
 use crate::handler::HandlerArgs;
 use crate::session::SessionImpl;
@@ -93,17 +94,31 @@ fn make_prost_privilege(
                     Binder::resolve_schema_qualified_name(db_name, name)?;
                 let schema_path = SchemaPath::new(schema_name.as_deref(), &search_path, user_name);
 
-                let (table, _) = reader.get_table_by_name(db_name, schema_path, &table_name)?;
-                match table.table_type() {
-                    TableType::Table => {}
-                    _ => {
-                        return Err(ErrorCode::InvalidInputSyntax(format!(
-                            "{table_name} is not a table",
-                        ))
-                        .into());
+                match reader.get_table_by_name(db_name, schema_path, &table_name) {
+                    Ok((table, _)) => {
+                        match table.table_type() {
+                            TableType::Table => {
+                                grant_objs.push(PbObject::TableId(table.id().table_id));
+                                continue;
+                            }
+                            _ => {
+                                return Err(ErrorCode::InvalidInputSyntax(format!(
+                                    "{table_name} is not a table",
+                                ))
+                                .into());
+                            }
+                        };
+                    }
+                    Err(CatalogError::NotFound("table", _)) => {
+                        let (view, _) = reader
+                            .get_view_by_name(db_name, schema_path, &table_name)
+                            .map_err(|_| CatalogError::NotFound("table", table_name))?;
+                        grant_objs.push(PbObject::ViewId(view.id));
+                    }
+                    Err(e) => {
+                        return Err(e.into());
                     }
                 }
-                grant_objs.push(PbObject::TableId(table.id().table_id));
             }
         }
         GrantObjects::Sources(sources) => {
@@ -138,7 +153,7 @@ fn make_prost_privilege(
             for schema in schemas {
                 let schema_name = Binder::resolve_schema_name(schema)?;
                 let schema = reader.get_schema_by_name(session.database(), &schema_name)?;
-                grant_objs.push(PbObject::AllDmlTablesSchemaId(schema.id()));
+                grant_objs.push(PbObject::AllDmlRelationsSchemaId(schema.id()));
             }
         }
         o => {
diff --git a/src/frontend/src/handler/mod.rs b/src/frontend/src/handler/mod.rs
index 80c91485c7a74..f8beeedb19438 100644
--- a/src/frontend/src/handler/mod.rs
+++ b/src/frontend/src/handler/mod.rs
@@ -27,6 +27,7 @@ use pgwire::types::{Format, Row};
 use risingwave_common::bail_not_implemented;
 use risingwave_common::types::Fields;
 use risingwave_common::util::iter_util::ZipEqFast;
+use risingwave_pb::meta::PbThrottleTarget;
 use risingwave_sqlparser::ast::*;
 
 use self::util::{DataChunkToRowSetAdapter, SourceSchemaCompatExt};
@@ -44,6 +45,7 @@ mod alter_rename;
 mod alter_set_schema;
 mod alter_source_column;
 mod alter_source_with_sr;
+mod alter_streaming_rate_limit;
 mod alter_system;
 mod alter_table_column;
 mod alter_table_with_sr;
@@ -51,12 +53,14 @@ pub mod alter_user;
 pub mod cancel_job;
 pub mod close_cursor;
 mod comment;
+pub mod create_aggregate;
 pub mod create_connection;
 mod create_database;
 pub mod create_function;
 pub mod create_index;
 pub mod create_mv;
 pub mod create_schema;
+pub mod create_secret;
 pub mod create_sink;
 pub mod create_source;
 pub mod create_sql_function;
@@ -74,6 +78,7 @@ pub mod drop_function;
 mod drop_index;
 pub mod drop_mv;
 mod drop_schema;
+pub mod drop_secret;
 pub mod drop_sink;
 pub mod drop_source;
 pub mod drop_subscription;
@@ -254,6 +259,9 @@ pub async fn handle(
         Statement::CreateConnection { stmt } => {
             create_connection::handle_create_connection(handler_args, stmt).await
         }
+        Statement::CreateSecret { stmt } => {
+            create_secret::handle_create_secret(handler_args, stmt).await
+        }
         Statement::CreateFunction {
             or_replace,
             temporary,
@@ -273,7 +281,6 @@ pub async fn handle(
                     .real_value()
                     .eq_ignore_ascii_case("sql")
             {
-                // User defined function with external source (e.g., language [ python / java ])
                 create_function::handle_create_function(
                     handler_args,
                     or_replace,
@@ -298,6 +305,24 @@ pub async fn handle(
                 .await
             }
         }
+        Statement::CreateAggregate {
+            or_replace,
+            name,
+            args,
+            returns,
+            params,
+            ..
+        } => {
+            create_aggregate::handle_create_aggregate(
+                handler_args,
+                or_replace,
+                name,
+                args,
+                returns,
+                params,
+            )
+            .await
+        }
         Statement::CreateTable {
             name,
             columns,
@@ -361,7 +386,16 @@ pub async fn handle(
         Statement::CreateSchema {
             schema_name,
             if_not_exists,
-        } => create_schema::handle_create_schema(handler_args, schema_name, if_not_exists).await,
+            user_specified,
+        } => {
+            create_schema::handle_create_schema(
+                handler_args,
+                schema_name,
+                if_not_exists,
+                user_specified,
+            )
+            .await
+        }
         Statement::CreateUser(stmt) => create_user::handle_create_user(handler_args, stmt).await,
         Statement::DeclareCursor { stmt } => {
             declare_cursor::handle_declare_cursor(handler_args, stmt).await
@@ -412,7 +446,8 @@ pub async fn handle(
                     ObjectType::Schema
                     | ObjectType::Database
                     | ObjectType::User
-                    | ObjectType::Connection => {
+                    | ObjectType::Connection
+                    | ObjectType::Secret => {
                         bail_not_implemented!("DROP CASCADE");
                     }
                 };
@@ -479,6 +514,9 @@ pub async fn handle(
                     drop_connection::handle_drop_connection(handler_args, object_name, if_exists)
                         .await
                 }
+                ObjectType::Secret => {
+                    drop_secret::handle_drop_secret(handler_args, object_name, if_exists).await
+                }
             }
         }
         // XXX: should we reuse Statement::Drop for DROP FUNCTION?
@@ -486,7 +524,18 @@ pub async fn handle(
             if_exists,
             func_desc,
             option,
-        } => drop_function::handle_drop_function(handler_args, if_exists, func_desc, option).await,
+        } => {
+            drop_function::handle_drop_function(handler_args, if_exists, func_desc, option, false)
+                .await
+        }
+        Statement::DropAggregate {
+            if_exists,
+            func_desc,
+            option,
+        } => {
+            drop_function::handle_drop_function(handler_args, if_exists, func_desc, option, true)
+                .await
+        }
         Statement::Query(_)
         | Statement::Insert { .. }
         | Statement::Delete { .. }
@@ -646,6 +695,18 @@ pub async fn handle(
             name,
             operation: AlterTableOperation::RefreshSchema,
         } => alter_table_with_sr::handle_refresh_schema(handler_args, name).await,
+        Statement::AlterTable {
+            name,
+            operation: AlterTableOperation::SetStreamingRateLimit { rate_limit },
+        } => {
+            alter_streaming_rate_limit::handle_alter_streaming_rate_limit(
+                handler_args,
+                PbThrottleTarget::TableWithSource,
+                name,
+                rate_limit,
+            )
+            .await
+        }
         Statement::AlterIndex {
             name,
             operation: AlterIndexOperation::RenameIndex { index_name },
@@ -750,6 +811,19 @@ pub async fn handle(
                 .await
             }
         }
+        Statement::AlterView {
+            materialized,
+            name,
+            operation: AlterViewOperation::SetStreamingRateLimit { rate_limit },
+        } if materialized => {
+            alter_streaming_rate_limit::handle_alter_streaming_rate_limit(
+                handler_args,
+                PbThrottleTarget::Mv,
+                name,
+                rate_limit,
+            )
+            .await
+        }
         Statement::AlterSink {
             name,
             operation: AlterSinkOperation::RenameSink { sink_name },
@@ -826,23 +900,6 @@ pub async fn handle(
             )
             .await
         }
-        Statement::AlterSubscription {
-            name,
-            operation:
-                AlterSubscriptionOperation::SetParallelism {
-                    parallelism,
-                    deferred,
-                },
-        } => {
-            alter_parallelism::handle_alter_parallelism(
-                handler_args,
-                name,
-                parallelism,
-                StatementType::ALTER_SUBSCRIPTION,
-                deferred,
-            )
-            .await
-        }
         Statement::AlterSource {
             name,
             operation: AlterSourceOperation::RenameSource { source_name },
@@ -887,6 +944,18 @@ pub async fn handle(
             name,
             operation: AlterSourceOperation::RefreshSchema,
         } => alter_source_with_sr::handler_refresh_schema(handler_args, name).await,
+        Statement::AlterSource {
+            name,
+            operation: AlterSourceOperation::SetStreamingRateLimit { rate_limit },
+        } => {
+            alter_streaming_rate_limit::handle_alter_streaming_rate_limit(
+                handler_args,
+                PbThrottleTarget::Source,
+                name,
+                rate_limit,
+            )
+            .await
+        }
         Statement::AlterFunction {
             name,
             args,
diff --git a/src/frontend/src/handler/query.rs b/src/frontend/src/handler/query.rs
index 68f0653a3588c..745749c9aef20 100644
--- a/src/frontend/src/handler/query.rs
+++ b/src/frontend/src/handler/query.rs
@@ -197,8 +197,8 @@ fn gen_batch_query_plan(
 
     let physical = match query_mode {
         QueryMode::Auto => unreachable!(),
-        QueryMode::Local => logical.gen_batch_local_plan(batch_plan)?,
-        QueryMode::Distributed => logical.gen_batch_distributed_plan(batch_plan)?,
+        QueryMode::Local => logical.gen_batch_local_plan()?,
+        QueryMode::Distributed => logical.gen_batch_distributed_plan()?,
     };
 
     Ok(BatchQueryPlanResult {
@@ -386,9 +386,7 @@ async fn execute(
                         "no affected rows in output".to_string(),
                     )))
                 }
-                Some(row) => {
-                    row.map_err(|err| RwError::from(ErrorCode::InternalError(format!("{}", err))))?
-                }
+                Some(row) => row?,
             };
             let affected_rows_str = first_row_set[0].values()[0]
                 .as_ref()
diff --git a/src/frontend/src/handler/show.rs b/src/frontend/src/handler/show.rs
index dfeeb0965ad64..b93a8032cbcd4 100644
--- a/src/frontend/src/handler/show.rs
+++ b/src/frontend/src/handler/show.rs
@@ -21,7 +21,8 @@ use pgwire::pg_response::{PgResponse, StatementType};
 use pgwire::pg_server::Session;
 use risingwave_common::bail_not_implemented;
 use risingwave_common::catalog::{ColumnCatalog, ColumnDesc, DEFAULT_SCHEMA_NAME};
-use risingwave_common::types::{DataType, Fields};
+use risingwave_common::session_config::{SearchPath, USER_NAME_WILD_CARD};
+use risingwave_common::types::{DataType, Fields, Timestamptz};
 use risingwave_common::util::addr::HostAddr;
 use risingwave_connector::source::kafka::PRIVATELINK_CONNECTION;
 use risingwave_expr::scalar::like::{i_like_default, like_default};
@@ -29,7 +30,6 @@ use risingwave_pb::catalog::connection;
 use risingwave_sqlparser::ast::{
     display_comma_separated, Ident, ObjectName, ShowCreateType, ShowObject, ShowStatementFilter,
 };
-use serde_json;
 
 use super::{fields_to_descriptors, RwPgResponse, RwPgResponseBuilderExt};
 use crate::binder::{Binder, Relation};
@@ -106,6 +106,28 @@ fn schema_or_default(schema: &Option<Ident>) -> String {
         .map_or_else(|| DEFAULT_SCHEMA_NAME.to_string(), |s| s.real_value())
 }
 
+fn schema_or_search_path(
+    session: &Arc<SessionImpl>,
+    schema: &Option<Ident>,
+    search_path: &SearchPath,
+) -> Vec<String> {
+    if let Some(s) = schema {
+        vec![s.real_value()]
+    } else {
+        search_path
+            .real_path()
+            .iter()
+            .map(|s| {
+                if s.eq(USER_NAME_WILD_CARD) {
+                    session.auth_context().user_name.clone()
+                } else {
+                    s.to_string()
+                }
+            })
+            .collect()
+    }
+}
+
 #[derive(Fields)]
 #[fields(style = "Title Case")]
 struct ShowObjectRow {
@@ -188,12 +210,15 @@ impl From<Arc<IndexCatalog>> for ShowIndexRow {
 #[derive(Fields)]
 #[fields(style = "Title Case")]
 struct ShowClusterRow {
+    id: i32,
     addr: String,
+    r#type: String,
     state: String,
     parallel_units: String,
-    is_streaming: String,
-    is_serving: String,
-    is_unschedulable: String,
+    is_streaming: Option<bool>,
+    is_serving: Option<bool>,
+    is_unschedulable: Option<bool>,
+    started_at: Option<Timestamptz>,
 }
 
 #[derive(Fields)]
@@ -251,12 +276,22 @@ pub async fn handle_show_object(
 
     let names = match command {
         // If not include schema name, use default schema name
-        ShowObject::Table { schema } => catalog_reader
-            .read_guard()
-            .get_schema_by_name(session.database(), &schema_or_default(&schema))?
-            .iter_table()
-            .map(|t| t.name.clone())
-            .collect(),
+        ShowObject::Table { schema } => {
+            let search_path = session.config().search_path();
+            let mut table_names_in_schema = vec![];
+            for schema in schema_or_search_path(&session, &schema, &search_path) {
+                // If the schema is not found, skip it
+                if let Ok(schema_catalog) = catalog_reader
+                    .read_guard()
+                    .get_schema_by_name(session.database(), schema.as_ref())
+                {
+                    table_names_in_schema
+                        .extend(schema_catalog.iter_table().map(|t| t.name.clone()));
+                }
+            }
+
+            table_names_in_schema
+        }
         ShowObject::InternalTable { schema } => catalog_reader
             .read_guard()
             .get_schema_by_name(session.database(), &schema_or_default(&schema))?
@@ -283,7 +318,6 @@ pub async fn handle_show_object(
             .read_guard()
             .get_schema_by_name(session.database(), &schema_or_default(&schema))?
             .iter_source()
-            .filter(|t| t.associated_table_id.is_none())
             .map(|t| t.name.clone())
             .collect(),
         ShowObject::Sink { schema } => catalog_reader
@@ -298,6 +332,12 @@ pub async fn handle_show_object(
             .iter_subscription()
             .map(|t| t.name.clone())
             .collect(),
+        ShowObject::Secret { schema } => catalog_reader
+            .read_guard()
+            .get_schema_by_name(session.database(), &schema_or_default(&schema))?
+            .iter_secret()
+            .map(|t| t.name.clone())
+            .collect(),
         ShowObject::Columns { table } => {
             let Ok(columns) = get_columns_from_table(&session, table.clone())
                 .or(get_columns_from_sink(&session, table.clone()))
@@ -386,17 +426,22 @@ pub async fn handle_show_object(
                 .into());
         }
         ShowObject::Cluster => {
-            let workers = session.env().worker_node_manager().list_worker_nodes();
-            let rows = workers.into_iter().map(|worker| {
+            let workers = session.env().meta_client().list_all_nodes().await?;
+            let rows = workers.into_iter().sorted_by_key(|w| w.id).map(|worker| {
                 let addr: HostAddr = worker.host.as_ref().unwrap().into();
-                let property = worker.property.as_ref().unwrap();
+                let property = worker.property.as_ref();
                 ShowClusterRow {
+                    id: worker.id as _,
                     addr: addr.to_string(),
+                    r#type: worker.get_type().unwrap().as_str_name().into(),
                     state: worker.get_state().unwrap().as_str_name().to_string(),
                     parallel_units: worker.parallel_units.into_iter().map(|pu| pu.id).join(", "),
-                    is_streaming: property.is_streaming.to_string(),
-                    is_serving: property.is_serving.to_string(),
-                    is_unschedulable: property.is_unschedulable.to_string(),
+                    is_streaming: property.map(|p| p.is_streaming),
+                    is_serving: property.map(|p| p.is_serving),
+                    is_unschedulable: property.map(|p| p.is_unschedulable),
+                    started_at: worker
+                        .started_at
+                        .map(|ts| Timestamptz::from_secs(ts as i64).unwrap()),
                 }
             });
             return Ok(PgResponse::builder(StatementType::SHOW_COMMAND)
diff --git a/src/frontend/src/handler/util.rs b/src/frontend/src/handler/util.rs
index 011b078958946..7fd4f0b92822b 100644
--- a/src/frontend/src/handler/util.rs
+++ b/src/frontend/src/handler/util.rs
@@ -32,12 +32,11 @@ use risingwave_common::types::{write_date_time_tz, DataType, ScalarRefImpl, Time
 use risingwave_common::util::epoch::Epoch;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_sqlparser::ast::{
-    BinaryOperator, CompatibleSourceSchema, ConnectorSchema, Expr, ObjectName, OrderByExpr, Query,
-    Select, SelectItem, SetExpr, TableFactor, TableWithJoins, Value,
+    CompatibleSourceSchema, ConnectorSchema, ObjectName, Query, Select, SelectItem, SetExpr,
+    TableFactor, TableWithJoins,
 };
 
 use crate::error::{ErrorCode, Result as RwResult};
-use crate::session::cursor_manager::{KV_LOG_STORE_EPOCH, KV_LOG_STORE_SEQ_ID, KV_LOG_STORE_VNODE};
 use crate::session::{current, SessionImpl};
 
 pin_project! {
@@ -222,67 +221,18 @@ pub fn gen_query_from_table_name(from_name: ObjectName) -> Query {
     }
 }
 
-pub fn gen_query_from_logstore_ge_rw_timestamp(logstore_name: &str, rw_timestamp: i64) -> Query {
-    let table_factor = TableFactor::Table {
-        name: ObjectName(vec![logstore_name.into()]),
-        alias: None,
-        as_of: None,
-    };
-    let from = vec![TableWithJoins {
-        relation: table_factor,
-        joins: vec![],
-    }];
-    let selection = Some(Expr::BinaryOp {
-        left: Box::new(Expr::Identifier(KV_LOG_STORE_EPOCH.into())),
-        op: BinaryOperator::GtEq,
-        right: Box::new(Expr::Value(Value::Number(rw_timestamp.to_string()))),
-    });
-    let except_columns = vec![
-        Expr::Identifier(KV_LOG_STORE_SEQ_ID.into()),
-        Expr::Identifier(KV_LOG_STORE_VNODE.into()),
-    ];
-    let select = Select {
-        from,
-        projection: vec![SelectItem::Wildcard(Some(except_columns))],
-        selection,
-        ..Default::default()
-    };
-    let order_by = vec![OrderByExpr {
-        expr: Expr::Identifier(KV_LOG_STORE_EPOCH.into()),
-        asc: None,
-        nulls_first: None,
-    }];
-    let body = SetExpr::Select(Box::new(select));
-    Query {
-        with: None,
-        body,
-        order_by,
-        limit: None,
-        offset: None,
-        fetch: None,
-    }
-}
-
-pub fn convert_unix_millis_to_logstore_i64(unix_millis: u64) -> i64 {
-    let epoch = Epoch::from_unix_millis(unix_millis);
-    convert_epoch_to_logstore_i64(epoch.0)
-}
-
-pub fn convert_epoch_to_logstore_i64(epoch: u64) -> i64 {
-    epoch as i64 ^ (1i64 << 63)
+pub fn convert_unix_millis_to_logstore_u64(unix_millis: u64) -> u64 {
+    Epoch::from_unix_millis(unix_millis).0
 }
 
-pub fn convert_logstore_i64_to_unix_millis(logstore_i64: i64) -> u64 {
-    let epoch = Epoch::from(logstore_i64 as u64 ^ (1u64 << 63));
-    epoch.as_unix_millis()
+pub fn convert_logstore_u64_to_unix_millis(logstore_u64: u64) -> u64 {
+    Epoch::from(logstore_u64).as_unix_millis()
 }
 
 #[cfg(test)]
 mod tests {
-    use bytes::BytesMut;
     use postgres_types::{ToSql, Type};
     use risingwave_common::array::*;
-    use risingwave_common::types::Timestamptz;
 
     use super::*;
 
diff --git a/src/frontend/src/lib.rs b/src/frontend/src/lib.rs
index 7f41de5cdc190..5f0c03061c0ab 100644
--- a/src/frontend/src/lib.rs
+++ b/src/frontend/src/lib.rs
@@ -32,7 +32,6 @@
 #![feature(impl_trait_in_assoc_type)]
 #![feature(result_flattening)]
 #![feature(error_generic_member_access)]
-#![feature(round_ties_even)]
 #![feature(iterator_try_collect)]
 #![feature(used_with_arg)]
 #![feature(entry_insert)]
@@ -43,6 +42,9 @@ risingwave_expr_impl::enable!();
 
 #[macro_use]
 mod catalog;
+
+use std::collections::HashSet;
+
 pub use catalog::TableCatalog;
 mod binder;
 pub use binder::{bind_data_type, Binder};
@@ -90,7 +92,7 @@ pub struct FrontendOpts {
     // TODO: rename to listen_addr and separate out the port.
     /// The address that this service listens to.
     /// Usually the localhost + desired port.
-    #[clap(long, env = "RW_LISTEN_ADDR", default_value = "127.0.0.1:4566")]
+    #[clap(long, env = "RW_LISTEN_ADDR", default_value = "0.0.0.0:4566")]
     pub listen_addr: String,
 
     /// The address for contacting this instance of the service.
@@ -169,8 +171,22 @@ pub fn start(opts: FrontendOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
     Box::pin(async move {
         let listen_addr = opts.listen_addr.clone();
         let session_mgr = Arc::new(SessionManagerImpl::new(opts).await.unwrap());
-        pg_serve(&listen_addr, session_mgr, TlsConfig::new_default())
-            .await
-            .unwrap();
+        let redact_sql_option_keywords = Arc::new(
+            session_mgr
+                .env()
+                .batch_config()
+                .redact_sql_option_keywords
+                .iter()
+                .map(|s| s.to_lowercase())
+                .collect::<HashSet<_>>(),
+        );
+        pg_serve(
+            &listen_addr,
+            session_mgr,
+            TlsConfig::new_default(),
+            Some(redact_sql_option_keywords),
+        )
+        .await
+        .unwrap()
     })
 }
diff --git a/src/frontend/src/meta_client.rs b/src/frontend/src/meta_client.rs
index 3cc7e22cf8b24..70b53cb82f9be 100644
--- a/src/frontend/src/meta_client.rs
+++ b/src/frontend/src/meta_client.rs
@@ -33,7 +33,7 @@ use risingwave_pb::meta::list_fragment_distribution_response::FragmentDistributi
 use risingwave_pb::meta::list_object_dependencies_response::PbObjectDependencies;
 use risingwave_pb::meta::list_table_fragment_states_response::TableFragmentState;
 use risingwave_pb::meta::list_table_fragments_response::TableFragmentInfo;
-use risingwave_pb::meta::EventLog;
+use risingwave_pb::meta::{EventLog, PbThrottleTarget};
 use risingwave_rpc_client::error::Result;
 use risingwave_rpc_client::{HummockMetaClient, MetaClient};
 
@@ -117,6 +117,13 @@ pub trait FrontendMetaClient: Send + Sync {
     async fn list_all_nodes(&self) -> Result<Vec<WorkerNode>>;
 
     async fn list_compact_task_progress(&self) -> Result<Vec<CompactTaskProgress>>;
+
+    async fn apply_throttle(
+        &self,
+        kind: PbThrottleTarget,
+        id: u32,
+        rate_limit: Option<u32>,
+    ) -> Result<()>;
 }
 
 pub struct FrontendMetaClientImpl(pub MetaClient);
@@ -293,4 +300,16 @@ impl FrontendMetaClient for FrontendMetaClientImpl {
     async fn list_compact_task_progress(&self) -> Result<Vec<CompactTaskProgress>> {
         self.0.list_compact_task_progress().await
     }
+
+    async fn apply_throttle(
+        &self,
+        kind: PbThrottleTarget,
+        id: u32,
+        rate_limit: Option<u32>,
+    ) -> Result<()> {
+        self.0
+            .apply_throttle(kind, id, rate_limit)
+            .await
+            .map(|_| ())
+    }
 }
diff --git a/src/frontend/src/observer/observer_manager.rs b/src/frontend/src/observer/observer_manager.rs
index ddf6ca489bf0c..e5313b9601eb4 100644
--- a/src/frontend/src/observer/observer_manager.rs
+++ b/src/frontend/src/observer/observer_manager.rs
@@ -19,7 +19,7 @@ use itertools::Itertools;
 use parking_lot::RwLock;
 use risingwave_batch::worker_manager::worker_node_manager::WorkerNodeManagerRef;
 use risingwave_common::catalog::CatalogVersion;
-use risingwave_common::hash::ParallelUnitMapping;
+use risingwave_common::hash::WorkerSlotMapping;
 use risingwave_common::session_config::SessionConfig;
 use risingwave_common::system_param::local_manager::LocalSystemParamsManagerRef;
 use risingwave_common_service::observer_manager::{ObserverState, SubscribeFrontend};
@@ -27,12 +27,12 @@ use risingwave_pb::common::WorkerNode;
 use risingwave_pb::hummock::HummockVersionStats;
 use risingwave_pb::meta::relation::RelationInfo;
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
-use risingwave_pb::meta::{FragmentParallelUnitMapping, MetaSnapshot, SubscribeResponse};
+use risingwave_pb::meta::{FragmentWorkerSlotMapping, MetaSnapshot, SubscribeResponse};
 use risingwave_rpc_client::ComputeClientPoolRef;
 use tokio::sync::watch::Sender;
 
 use crate::catalog::root_catalog::Catalog;
-use crate::catalog::FragmentId;
+use crate::catalog::{FragmentId, SecretId};
 use crate::scheduler::HummockSnapshotManagerRef;
 use crate::user::user_manager::UserInfoManager;
 use crate::user::UserInfoVersion;
@@ -63,6 +63,7 @@ impl ObserverState for FrontendObserverNode {
             | Info::Schema(_)
             | Info::RelationGroup(_)
             | Info::Function(_)
+            | Info::Secret(_)
             | Info::Connection(_) => {
                 self.handle_catalog_notification(resp);
             }
@@ -72,7 +73,6 @@ impl ObserverState for FrontendObserverNode {
             Info::User(_) => {
                 self.handle_user_notification(resp);
             }
-            Info::ParallelUnitMapping(_) => self.handle_fragment_mapping_notification(resp),
             Info::Snapshot(_) => {
                 panic!(
                     "receiving a snapshot in the middle is unsupported now {:?}",
@@ -103,8 +103,9 @@ impl ObserverState for FrontendObserverNode {
             Info::HummockStats(stats) => {
                 self.handle_table_stats_notification(stats);
             }
-            Info::ServingParallelUnitMappings(m) => {
-                self.handle_fragment_serving_mapping_notification(m.mappings, resp.operation());
+            Info::StreamingWorkerSlotMapping(_) => self.handle_fragment_mapping_notification(resp),
+            Info::ServingWorkerSlotMappings(m) => {
+                self.handle_fragment_serving_mapping_notification(m.mappings, resp.operation())
             }
             Info::Recovery(_) => {
                 self.compute_client_pool.invalidate_all();
@@ -133,15 +134,16 @@ impl ObserverState for FrontendObserverNode {
             functions,
             connections,
             users,
-            parallel_unit_mappings,
-            serving_parallel_unit_mappings,
             nodes,
             hummock_snapshot,
             hummock_version: _,
             meta_backup_manifest_id: _,
             hummock_write_limits: _,
+            streaming_worker_slot_mappings,
+            serving_worker_slot_mappings,
             session_params,
             version,
+            secrets,
         } = snapshot;
 
         for db in databases {
@@ -174,13 +176,17 @@ impl ObserverState for FrontendObserverNode {
         for connection in connections {
             catalog_guard.create_connection(&connection)
         }
+        for secret in secrets {
+            catalog_guard.create_secret(&secret)
+        }
         for user in users {
             user_guard.create_user(user)
         }
+
         self.worker_node_manager.refresh(
             nodes,
-            convert_pu_mapping(&parallel_unit_mappings),
-            convert_pu_mapping(&serving_parallel_unit_mappings),
+            convert_worker_slot_mapping(&streaming_worker_slot_mappings),
+            convert_worker_slot_mapping(&serving_worker_slot_mappings),
         );
         self.hummock_snapshot_manager
             .update(hummock_snapshot.unwrap());
@@ -345,6 +351,16 @@ impl FrontendObserverNode {
                 Operation::Update => catalog_guard.update_connection(connection),
                 _ => panic!("receive an unsupported notify {:?}", resp),
             },
+            Info::Secret(secret) => match resp.operation() {
+                Operation::Add => catalog_guard.create_secret(secret),
+                Operation::Delete => catalog_guard.drop_secret(
+                    secret.database_id,
+                    secret.schema_id,
+                    SecretId::new(secret.id),
+                ),
+                Operation::Update => catalog_guard.update_secret(secret),
+                _ => panic!("receive an unsupported notify {:?}", resp),
+            },
             _ => unreachable!(),
         }
         assert!(
@@ -387,11 +403,11 @@ impl FrontendObserverNode {
             return;
         };
         match info {
-            Info::ParallelUnitMapping(parallel_unit_mapping) => {
-                let fragment_id = parallel_unit_mapping.fragment_id;
+            Info::StreamingWorkerSlotMapping(streaming_worker_slot_mapping) => {
+                let fragment_id = streaming_worker_slot_mapping.fragment_id;
                 let mapping = || {
-                    ParallelUnitMapping::from_protobuf(
-                        parallel_unit_mapping.mapping.as_ref().unwrap(),
+                    WorkerSlotMapping::from_protobuf(
+                        streaming_worker_slot_mapping.mapping.as_ref().unwrap(),
                     )
                 };
 
@@ -417,20 +433,20 @@ impl FrontendObserverNode {
 
     fn handle_fragment_serving_mapping_notification(
         &mut self,
-        mappings: Vec<FragmentParallelUnitMapping>,
+        mappings: Vec<FragmentWorkerSlotMapping>,
         op: Operation,
     ) {
         match op {
             Operation::Add | Operation::Update => {
                 self.worker_node_manager
-                    .upsert_serving_fragment_mapping(convert_pu_mapping(&mappings));
+                    .upsert_serving_fragment_mapping(convert_worker_slot_mapping(&mappings));
             }
             Operation::Delete => self.worker_node_manager.remove_serving_fragment_mapping(
                 &mappings.into_iter().map(|m| m.fragment_id).collect_vec(),
             ),
             Operation::Snapshot => {
                 self.worker_node_manager
-                    .set_serving_fragment_mapping(convert_pu_mapping(&mappings));
+                    .set_serving_fragment_mapping(convert_worker_slot_mapping(&mappings));
             }
             _ => panic!("receive an unsupported notify {:?}", op),
         }
@@ -470,17 +486,17 @@ impl FrontendObserverNode {
     }
 }
 
-fn convert_pu_mapping(
-    parallel_unit_mappings: &[FragmentParallelUnitMapping],
-) -> HashMap<FragmentId, ParallelUnitMapping> {
-    parallel_unit_mappings
+fn convert_worker_slot_mapping(
+    worker_slot_mappings: &[FragmentWorkerSlotMapping],
+) -> HashMap<FragmentId, WorkerSlotMapping> {
+    worker_slot_mappings
         .iter()
         .map(
-            |FragmentParallelUnitMapping {
+            |FragmentWorkerSlotMapping {
                  fragment_id,
                  mapping,
              }| {
-                let mapping = ParallelUnitMapping::from_protobuf(mapping.as_ref().unwrap());
+                let mapping = WorkerSlotMapping::from_protobuf(mapping.as_ref().unwrap());
                 (*fragment_id, mapping)
             },
         )
diff --git a/src/frontend/src/optimizer/logical_optimization.rs b/src/frontend/src/optimizer/logical_optimization.rs
index 14fa66be68d5f..d452626bb9418 100644
--- a/src/frontend/src/optimizer/logical_optimization.rs
+++ b/src/frontend/src/optimizer/logical_optimization.rs
@@ -172,7 +172,7 @@ static GENERAL_UNNESTING_TRANS_APPLY_WITH_SHARE: LazyLock<OptimizationStage> =
                 // can't handle a join with `output_indices`.
                 ProjectJoinSeparateRule::create(),
             ],
-            ApplyOrder::BottomUp,
+            ApplyOrder::TopDown,
         )
     });
 
@@ -186,7 +186,7 @@ static GENERAL_UNNESTING_TRANS_APPLY_WITHOUT_SHARE: LazyLock<OptimizationStage>
                 // can't handle a join with `output_indices`.
                 ProjectJoinSeparateRule::create(),
             ],
-            ApplyOrder::BottomUp,
+            ApplyOrder::TopDown,
         )
     });
 
@@ -732,8 +732,8 @@ impl LogicalOptimizer {
         // Do a final column pruning and predicate pushing down to clean up the plan.
         plan = Self::column_pruning(plan, explain_trace, &ctx);
         if last_total_rule_applied_before_predicate_pushdown != ctx.total_rule_applied() {
-            #[allow(unused_assignments)]
-            last_total_rule_applied_before_predicate_pushdown = ctx.total_rule_applied();
+            (#[allow(unused_assignments)]
+            last_total_rule_applied_before_predicate_pushdown) = ctx.total_rule_applied();
             plan = Self::predicate_pushdown(plan, explain_trace, &ctx);
         }
 
diff --git a/src/frontend/src/optimizer/mod.rs b/src/frontend/src/optimizer/mod.rs
index e0b69d27f647c..1993e1293880b 100644
--- a/src/frontend/src/optimizer/mod.rs
+++ b/src/frontend/src/optimizer/mod.rs
@@ -40,7 +40,7 @@ mod plan_expr_visitor;
 mod rule;
 
 use std::assert_matches::assert_matches;
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 
 use fixedbitset::FixedBitSet;
 use itertools::Itertools as _;
@@ -50,7 +50,7 @@ use plan_expr_rewriter::ConstEvalRewriter;
 use property::Order;
 use risingwave_common::bail;
 use risingwave_common::catalog::{
-    ColumnCatalog, ColumnDesc, ColumnId, ConflictBehavior, Field, Schema, TableId, UserId,
+    ColumnCatalog, ColumnDesc, ColumnId, ConflictBehavior, Field, Schema, TableId,
 };
 use risingwave_common::types::DataType;
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
@@ -64,7 +64,7 @@ use self::plan_node::generic::{self, PhysicalPlanRef};
 use self::plan_node::{
     stream_enforce_eowc_requirement, BatchProject, Convention, LogicalProject, LogicalSource,
     PartitionComputeInfo, StreamDml, StreamMaterialize, StreamProject, StreamRowIdGen, StreamSink,
-    StreamSubscription, StreamWatermarkFilter, ToStreamContext,
+    StreamWatermarkFilter, ToStreamContext,
 };
 #[cfg(debug_assertions)]
 use self::plan_visitor::InputRefValidator;
@@ -96,20 +96,74 @@ use crate::WithOptions;
 /// column in the result.
 #[derive(Debug, Clone)]
 pub struct PlanRoot {
+    // The current plan node.
     plan: PlanRef,
+    // The phase of the plan.
+    phase: PlanPhase,
     required_dist: RequiredDist,
     required_order: Order,
     out_fields: FixedBitSet,
     out_names: Vec<String>,
 }
 
+/// `PlanPhase` is used to track the phase of the `PlanRoot`.
+/// Usually, it begins from `Logical` and ends with `Batch` or `Stream`, unless we want to construct a `PlanRoot` from an intermediate phase.
+/// Typical phase transformation are:
+/// - `Logical` -> `OptimizedLogicalForBatch` -> `Batch`
+/// - `Logical` -> `OptimizedLogicalForStream` -> `Stream`
+#[derive(Debug, Clone, PartialEq)]
+pub enum PlanPhase {
+    Logical,
+    OptimizedLogicalForBatch,
+    OptimizedLogicalForStream,
+    Batch,
+    Stream,
+}
+
 impl PlanRoot {
-    pub fn new(
+    pub fn new_with_logical_plan(
+        plan: PlanRef,
+        required_dist: RequiredDist,
+        required_order: Order,
+        out_fields: FixedBitSet,
+        out_names: Vec<String>,
+    ) -> Self {
+        assert_eq!(plan.convention(), Convention::Logical);
+        Self::new_inner(
+            plan,
+            PlanPhase::Logical,
+            required_dist,
+            required_order,
+            out_fields,
+            out_names,
+        )
+    }
+
+    pub fn new_with_batch_plan(
         plan: PlanRef,
         required_dist: RequiredDist,
         required_order: Order,
         out_fields: FixedBitSet,
         out_names: Vec<String>,
+    ) -> Self {
+        assert_eq!(plan.convention(), Convention::Batch);
+        Self::new_inner(
+            plan,
+            PlanPhase::Batch,
+            required_dist,
+            required_order,
+            out_fields,
+            out_names,
+        )
+    }
+
+    fn new_inner(
+        plan: PlanRef,
+        phase: PlanPhase,
+        required_dist: RequiredDist,
+        required_order: Order,
+        out_fields: FixedBitSet,
+        out_names: Vec<String>,
     ) -> Self {
         let input_schema = plan.schema();
         assert_eq!(input_schema.fields().len(), out_fields.len());
@@ -117,6 +171,7 @@ impl PlanRoot {
 
         Self {
             plan,
+            phase,
             required_dist,
             required_order,
             out_fields,
@@ -160,6 +215,8 @@ impl PlanRoot {
     /// example as insert source or subquery. This ignores Order but retains post-Order pruning
     /// (`out_fields`).
     pub fn into_unordered_subplan(self) -> PlanRef {
+        assert_eq!(self.phase, PlanPhase::Logical);
+        assert_eq!(self.plan.convention(), Convention::Logical);
         if self.out_fields.count_ones(..) == self.out_fields.len() {
             return self.plan;
         }
@@ -178,6 +235,8 @@ impl PlanRoot {
         use crate::expr::{ExprImpl, ExprType, FunctionCall, InputRef};
         use crate::utils::{Condition, IndexSet};
 
+        assert_eq!(self.phase, PlanPhase::Logical);
+        assert_eq!(self.plan.convention(), Convention::Logical);
         let Ok(select_idx) = self.out_fields.ones().exactly_one() else {
             bail!("subquery must return only one column");
         };
@@ -192,6 +251,7 @@ impl PlanRoot {
                 order_by: self.required_order.column_orders,
                 filter: Condition::true_cond(),
                 direct_args: vec![],
+                user_defined: None,
             }],
             IndexSet::empty(),
             self.plan,
@@ -211,19 +271,38 @@ impl PlanRoot {
     }
 
     /// Apply logical optimization to the plan for stream.
-    pub fn gen_optimized_logical_plan_for_stream(&self) -> Result<PlanRef> {
-        LogicalOptimizer::gen_optimized_logical_plan_for_stream(self.plan.clone())
+    pub fn gen_optimized_logical_plan_for_stream(&mut self) -> Result<PlanRef> {
+        assert_eq!(self.phase, PlanPhase::Logical);
+        assert_eq!(self.plan.convention(), Convention::Logical);
+        self.plan = LogicalOptimizer::gen_optimized_logical_plan_for_stream(self.plan.clone())?;
+        self.phase = PlanPhase::OptimizedLogicalForStream;
+        assert_eq!(self.plan.convention(), Convention::Logical);
+        Ok(self.plan.clone())
     }
 
     /// Apply logical optimization to the plan for batch.
-    pub fn gen_optimized_logical_plan_for_batch(&self) -> Result<PlanRef> {
-        LogicalOptimizer::gen_optimized_logical_plan_for_batch(self.plan.clone())
+    pub fn gen_optimized_logical_plan_for_batch(&mut self) -> Result<PlanRef> {
+        assert_eq!(self.phase, PlanPhase::Logical);
+        assert_eq!(self.plan.convention(), Convention::Logical);
+        self.plan = LogicalOptimizer::gen_optimized_logical_plan_for_batch(self.plan.clone())?;
+        self.phase = PlanPhase::OptimizedLogicalForBatch;
+        assert_eq!(self.plan.convention(), Convention::Logical);
+        Ok(self.plan.clone())
     }
 
     /// Optimize and generate a singleton batch physical plan without exchange nodes.
     pub fn gen_batch_plan(&mut self) -> Result<PlanRef> {
-        // Logical optimization
-        let mut plan = self.gen_optimized_logical_plan_for_batch()?;
+        assert_eq!(self.plan.convention(), Convention::Logical);
+        let mut plan = match self.phase {
+            PlanPhase::Logical => {
+                // Logical optimization
+                self.gen_optimized_logical_plan_for_batch()?
+            }
+            PlanPhase::OptimizedLogicalForBatch => self.plan.clone(),
+            PlanPhase::Batch | PlanPhase::OptimizedLogicalForStream | PlanPhase::Stream => {
+                panic!("unexpected phase")
+            }
+        };
 
         if TemporalJoinValidator::exist_dangling_temporal_scan(plan.clone()) {
             return Err(ErrorCode::NotSupported(
@@ -285,13 +364,18 @@ impl PlanRoot {
             ctx.trace(plan.explain_to_string());
         }
 
-        Ok(plan)
+        self.plan = plan;
+        self.phase = PlanPhase::Batch;
+        assert_eq!(self.plan.convention(), Convention::Batch);
+        Ok(self.plan.clone())
     }
 
     /// Optimize and generate a batch query plan for distributed execution.
-    pub fn gen_batch_distributed_plan(&mut self, batch_plan: PlanRef) -> Result<PlanRef> {
-        self.set_required_dist(RequiredDist::single());
-        let mut plan = batch_plan;
+    pub fn gen_batch_distributed_plan(mut self) -> Result<PlanRef> {
+        assert_eq!(self.phase, PlanPhase::Batch);
+        assert_eq!(self.plan.convention(), Convention::Batch);
+        self.required_dist = RequiredDist::single();
+        let mut plan = self.plan;
 
         // Convert to distributed plan
         plan = plan.to_distributed_with_required(&self.required_order, &self.required_dist)?;
@@ -319,12 +403,15 @@ impl PlanRoot {
             ApplyOrder::BottomUp,
         ));
 
+        assert_eq!(plan.convention(), Convention::Batch);
         Ok(plan)
     }
 
     /// Optimize and generate a batch query plan for local execution.
-    pub fn gen_batch_local_plan(&mut self, batch_plan: PlanRef) -> Result<PlanRef> {
-        let mut plan = batch_plan;
+    pub fn gen_batch_local_plan(self) -> Result<PlanRef> {
+        assert_eq!(self.phase, PlanPhase::Batch);
+        assert_eq!(self.plan.convention(), Convention::Batch);
+        let mut plan = self.plan;
 
         // Convert to local plan node
         plan = plan.to_local_with_order_required(&self.required_order)?;
@@ -359,11 +446,14 @@ impl PlanRoot {
             ApplyOrder::BottomUp,
         ));
 
+        assert_eq!(plan.convention(), Convention::Batch);
         Ok(plan)
     }
 
     /// Generate optimized stream plan
     fn gen_optimized_stream_plan(&mut self, emit_on_window_close: bool) -> Result<PlanRef> {
+        assert_eq!(self.phase, PlanPhase::Logical);
+        assert_eq!(self.plan.convention(), Convention::Logical);
         let stream_scan_type = if self.should_use_arrangement_backfill() {
             StreamScanType::ArrangementBackfill
         } else {
@@ -377,6 +467,8 @@ impl PlanRoot {
         emit_on_window_close: bool,
         stream_scan_type: StreamScanType,
     ) -> Result<PlanRef> {
+        assert_eq!(self.phase, PlanPhase::Logical);
+        assert_eq!(self.plan.convention(), Convention::Logical);
         let ctx = self.plan.ctx();
         let _explain_trace = ctx.is_explain_trace();
 
@@ -424,7 +516,10 @@ impl PlanRoot {
             ).into());
         }
 
-        Ok(plan)
+        self.plan = plan;
+        self.phase = PlanPhase::Stream;
+        assert_eq!(self.plan.convention(), Convention::Stream);
+        Ok(self.plan.clone())
     }
 
     /// Generate create index or create materialize view plan.
@@ -433,6 +528,8 @@ impl PlanRoot {
         emit_on_window_close: bool,
         stream_scan_type: StreamScanType,
     ) -> Result<PlanRef> {
+        assert_eq!(self.phase, PlanPhase::Logical);
+        assert_eq!(self.plan.convention(), Convention::Logical);
         let ctx = self.plan.ctx();
         let explain_trace = ctx.is_explain_trace();
 
@@ -523,7 +620,7 @@ impl PlanRoot {
     /// Optimize and generate a create table plan.
     #[allow(clippy::too_many_arguments)]
     pub fn gen_table_plan(
-        &mut self,
+        mut self,
         context: OptimizerContextRef,
         table_name: String,
         columns: Vec<ColumnCatalog>,
@@ -538,7 +635,11 @@ impl PlanRoot {
         with_external_source: bool,
         retention_seconds: Option<NonZeroU32>,
     ) -> Result<StreamMaterialize> {
+        assert_eq!(self.phase, PlanPhase::Logical);
+        assert_eq!(self.plan.convention(), Convention::Logical);
         let stream_plan = self.gen_optimized_stream_plan(false)?;
+        assert_eq!(self.phase, PlanPhase::Stream);
+        assert_eq!(stream_plan.convention(), Convention::Stream);
 
         assert!(!pk_column_ids.is_empty() || row_id_index.is_some());
 
@@ -767,13 +868,17 @@ impl PlanRoot {
 
     /// Optimize and generate a create materialized view plan.
     pub fn gen_materialize_plan(
-        &mut self,
+        mut self,
         mv_name: String,
         definition: String,
         emit_on_window_close: bool,
     ) -> Result<StreamMaterialize> {
         let cardinality = self.compute_cardinality();
+        assert_eq!(self.phase, PlanPhase::Logical);
+        assert_eq!(self.plan.convention(), Convention::Logical);
         let stream_plan = self.gen_optimized_stream_plan(emit_on_window_close)?;
+        assert_eq!(self.phase, PlanPhase::Stream);
+        assert_eq!(stream_plan.convention(), Convention::Stream);
 
         StreamMaterialize::create(
             stream_plan,
@@ -791,13 +896,17 @@ impl PlanRoot {
 
     /// Optimize and generate a create index plan.
     pub fn gen_index_plan(
-        &mut self,
+        mut self,
         index_name: String,
         definition: String,
         retention_seconds: Option<NonZeroU32>,
     ) -> Result<StreamMaterialize> {
         let cardinality = self.compute_cardinality();
+        assert_eq!(self.phase, PlanPhase::Logical);
+        assert_eq!(self.plan.convention(), Convention::Logical);
         let stream_plan = self.gen_optimized_stream_plan(false)?;
+        assert_eq!(self.phase, PlanPhase::Stream);
+        assert_eq!(stream_plan.convention(), Convention::Stream);
 
         StreamMaterialize::create(
             stream_plan,
@@ -835,9 +944,12 @@ impl PlanRoot {
         } else {
             StreamScanType::Backfill
         };
+        assert_eq!(self.phase, PlanPhase::Logical);
+        assert_eq!(self.plan.convention(), Convention::Logical);
         let stream_plan =
             self.gen_optimized_stream_plan_inner(emit_on_window_close, stream_scan_type)?;
-
+        assert_eq!(self.phase, PlanPhase::Stream);
+        assert_eq!(stream_plan.convention(), Convention::Stream);
         StreamSink::create(
             stream_plan,
             sink_name,
@@ -855,46 +967,6 @@ impl PlanRoot {
         )
     }
 
-    #[allow(clippy::too_many_arguments)]
-    /// Optimize and generate a create subscription plan.
-    pub fn gen_subscription_plan(
-        &mut self,
-        database_id: u32,
-        schema_id: u32,
-        dependent_relations: HashSet<TableId>,
-        subscription_name: String,
-        definition: String,
-        properties: WithOptions,
-        emit_on_window_close: bool,
-        subscription_from_table_name: String,
-        user_id: UserId,
-    ) -> Result<StreamSubscription> {
-        let stream_scan_type = StreamScanType::UpstreamOnly;
-        let stream_plan =
-            self.gen_optimized_stream_plan_inner(emit_on_window_close, stream_scan_type)?;
-
-        StreamSubscription::create(
-            database_id,
-            schema_id,
-            dependent_relations,
-            stream_plan,
-            subscription_name,
-            subscription_from_table_name,
-            self.required_dist.clone(),
-            self.required_order.clone(),
-            self.out_fields.clone(),
-            self.out_names.clone(),
-            definition,
-            properties,
-            user_id,
-        )
-    }
-
-    /// Set the plan root's required dist.
-    pub fn set_required_dist(&mut self, required_dist: RequiredDist) {
-        self.required_dist = required_dist;
-    }
-
     pub fn should_use_arrangement_backfill(&self) -> bool {
         let ctx = self.plan.ctx();
         let session_ctx = ctx.session_ctx();
@@ -972,6 +1044,10 @@ fn require_additional_exchange_on_root_in_distributed_mode(plan: PlanRef) -> boo
         plan.node_type() == PlanNodeType::BatchSeqScan
     }
 
+    fn is_log_table(plan: &PlanRef) -> bool {
+        plan.node_type() == PlanNodeType::BatchLogSeqScan
+    }
+
     fn is_source(plan: &PlanRef) -> bool {
         plan.node_type() == PlanNodeType::BatchSource
             || plan.node_type() == PlanNodeType::BatchKafkaScan
@@ -996,6 +1072,7 @@ fn require_additional_exchange_on_root_in_distributed_mode(plan: PlanRef) -> boo
         || exist_and_no_exchange_before(&plan, is_insert)
         || exist_and_no_exchange_before(&plan, is_update)
         || exist_and_no_exchange_before(&plan, is_delete)
+        || exist_and_no_exchange_before(&plan, is_log_table)
 }
 
 /// The purpose is same as `require_additional_exchange_on_root_in_distributed_mode`. We separate
@@ -1023,11 +1100,7 @@ fn require_additional_exchange_on_root_in_local_mode(plan: PlanRef) -> bool {
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::catalog::Field;
-    use risingwave_common::types::DataType;
-
     use super::*;
-    use crate::optimizer::optimizer_context::OptimizerContext;
     use crate::optimizer::plan_node::LogicalValues;
 
     #[tokio::test]
@@ -1044,7 +1117,7 @@ mod tests {
         .into();
         let out_fields = FixedBitSet::with_capacity_and_blocks(2, [1]);
         let out_names = vec!["v1".into()];
-        let root = PlanRoot::new(
+        let root = PlanRoot::new_with_logical_plan(
             values,
             RequiredDist::Any,
             Order::any(),
diff --git a/src/frontend/src/optimizer/optimizer_context.rs b/src/frontend/src/optimizer/optimizer_context.rs
index eb6e410378749..35c5a3ee9129b 100644
--- a/src/frontend/src/optimizer/optimizer_context.rs
+++ b/src/frontend/src/optimizer/optimizer_context.rs
@@ -12,20 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use core::convert::Into;
 use core::fmt::Formatter;
 use std::cell::{RefCell, RefMut};
+use std::collections::HashMap;
 use std::marker::PhantomData;
 use std::rc::Rc;
 use std::sync::Arc;
 
 use risingwave_sqlparser::ast::{ExplainOptions, ExplainType};
 
+use crate::binder::ShareId;
 use crate::expr::{CorrelatedId, SessionTimezone};
 use crate::handler::HandlerArgs;
 use crate::optimizer::plan_node::PlanNodeId;
 use crate::session::SessionImpl;
 use crate::utils::{OverwriteOptions, WithOptions};
+use crate::PlanRef;
 
 const RESERVED_ID_NUM: u16 = 10000;
 
@@ -58,6 +60,9 @@ pub struct OptimizerContext {
     /// Store the configs can be overwritten in with clause
     /// if not specified, use the value from session variable.
     overwrite_options: OverwriteOptions,
+    /// Store the mapping between `share_id` and the corresponding
+    /// `PlanRef`, used by rcte's planning. (e.g., in `LogicalCteRef`)
+    rcte_cache: RefCell<HashMap<ShareId, PlanRef>>,
 
     _phantom: PhantomUnsend,
 }
@@ -91,6 +96,7 @@ impl OptimizerContext {
             next_expr_display_id: RefCell::new(RESERVED_ID_NUM.into()),
             total_rule_applied: RefCell::new(0),
             overwrite_options,
+            rcte_cache: RefCell::new(HashMap::new()),
             _phantom: Default::default(),
         }
     }
@@ -113,6 +119,7 @@ impl OptimizerContext {
             next_expr_display_id: RefCell::new(0),
             total_rule_applied: RefCell::new(0),
             overwrite_options: OverwriteOptions::default(),
+            rcte_cache: RefCell::new(HashMap::new()),
             _phantom: Default::default(),
         }
         .into()
@@ -230,6 +237,14 @@ impl OptimizerContext {
     pub fn get_session_timezone(&self) -> String {
         self.session_timezone.borrow().timezone()
     }
+
+    pub fn get_rcte_cache_plan(&self, id: &ShareId) -> Option<PlanRef> {
+        self.rcte_cache.borrow().get(id).cloned()
+    }
+
+    pub fn insert_rcte_cache_plan(&self, id: ShareId, plan: PlanRef) {
+        self.rcte_cache.borrow_mut().insert(id, plan);
+    }
 }
 
 impl std::fmt::Debug for OptimizerContext {
diff --git a/src/frontend/src/optimizer/plan_expr_visitor/strong.rs b/src/frontend/src/optimizer/plan_expr_visitor/strong.rs
index e30cdb0b6e314..fefdd1e4547fc 100644
--- a/src/frontend/src/optimizer/plan_expr_visitor/strong.rs
+++ b/src/frontend/src/optimizer/plan_expr_visitor/strong.rs
@@ -81,6 +81,7 @@ impl Strong {
             | ExprType::IsDistinctFrom
             | ExprType::IsNotDistinctFrom
             | ExprType::IsTrue
+            | ExprType::QuoteNullable
             | ExprType::IsNotTrue
             | ExprType::IsFalse
             | ExprType::IsNotFalse => false,
@@ -180,6 +181,7 @@ impl Strong {
             | ExprType::ToAscii
             | ExprType::ToHex
             | ExprType::QuoteIdent
+            | ExprType::QuoteLiteral
             | ExprType::Sin
             | ExprType::Cos
             | ExprType::Tan
@@ -287,6 +289,7 @@ impl Strong {
             | ExprType::JsonbPathQueryFirst
             | ExprType::JsonbPopulateRecord
             | ExprType::JsonbToRecord
+            | ExprType::JsonbSet
             | ExprType::Vnode
             | ExprType::Proctime
             | ExprType::PgSleep
@@ -301,6 +304,9 @@ impl Strong {
             | ExprType::PgRelationSize
             | ExprType::PgGetSerialSequence
             | ExprType::IcebergTransform
+            | ExprType::HasTablePrivilege
+            | ExprType::HasAnyColumnPrivilege
+            | ExprType::HasSchemaPrivilege
             | ExprType::InetAton
             | ExprType::InetNtoa => false,
             ExprType::Unspecified => unreachable!(),
diff --git a/src/frontend/src/optimizer/plan_node/batch_delete.rs b/src/frontend/src/optimizer/plan_node/batch_delete.rs
index 62499a495d1de..8980f58a9e2c4 100644
--- a/src/frontend/src/optimizer/plan_node/batch_delete.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_delete.rs
@@ -22,7 +22,6 @@ use super::{
 };
 use crate::error::Result;
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::plan_node::generic::PhysicalPlanRef;
 use crate::optimizer::plan_node::{utils, ToLocalBatch};
 use crate::optimizer::plan_visitor::DistributedDmlVisitor;
 use crate::optimizer::property::{Distribution, Order, RequiredDist};
diff --git a/src/frontend/src/optimizer/plan_node/batch_hash_agg.rs b/src/frontend/src/optimizer/plan_node/batch_hash_agg.rs
index af5f4c4accbb6..341d1281009eb 100644
--- a/src/frontend/src/optimizer/plan_node/batch_hash_agg.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_hash_agg.rs
@@ -17,7 +17,7 @@ use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::HashAggNode;
 
 use super::batch::prelude::*;
-use super::generic::{self, GenericPlanRef, PlanAggCall};
+use super::generic::{self, PlanAggCall};
 use super::utils::impl_distill_by_unit;
 use super::{
     ExprRewritable, PlanBase, PlanNodeType, PlanRef, PlanTreeNodeUnary, ToBatchPb,
diff --git a/src/frontend/src/optimizer/plan_node/batch_hash_join.rs b/src/frontend/src/optimizer/plan_node/batch_hash_join.rs
index 8b92f722490d5..399817336e4ca 100644
--- a/src/frontend/src/optimizer/plan_node/batch_hash_join.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_hash_join.rs
@@ -18,10 +18,9 @@ use risingwave_pb::batch_plan::HashJoinNode;
 use risingwave_pb::plan_common::JoinType;
 
 use super::batch::prelude::*;
-use super::generic::{self, GenericPlanRef};
 use super::utils::{childless_record, Distill};
 use super::{
-    EqJoinPredicate, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeBinary, ToBatchPb,
+    generic, EqJoinPredicate, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeBinary, ToBatchPb,
     ToDistributedBatch,
 };
 use crate::error::Result;
diff --git a/src/frontend/src/optimizer/plan_node/batch_insert.rs b/src/frontend/src/optimizer/plan_node/batch_insert.rs
index 1dbe387bb708f..8a3e511efc83c 100644
--- a/src/frontend/src/optimizer/plan_node/batch_insert.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_insert.rs
@@ -18,7 +18,6 @@ use risingwave_pb::batch_plan::InsertNode;
 use risingwave_pb::plan_common::{DefaultColumns, IndexAndExpr};
 
 use super::batch::prelude::*;
-use super::generic::GenericPlanRef;
 use super::utils::{childless_record, Distill};
 use super::{generic, ExprRewritable, PlanRef, PlanTreeNodeUnary, ToBatchPb, ToDistributedBatch};
 use crate::error::Result;
diff --git a/src/frontend/src/optimizer/plan_node/batch_limit.rs b/src/frontend/src/optimizer/plan_node/batch_limit.rs
index 1dfe9215be1ea..ef717d179130e 100644
--- a/src/frontend/src/optimizer/plan_node/batch_limit.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_limit.rs
@@ -16,7 +16,6 @@ use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::LimitNode;
 
 use super::batch::prelude::*;
-use super::generic::PhysicalPlanRef;
 use super::utils::impl_distill_by_unit;
 use super::{
     generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, ToBatchPb, ToDistributedBatch,
diff --git a/src/frontend/src/optimizer/plan_node/batch_log_seq_scan.rs b/src/frontend/src/optimizer/plan_node/batch_log_seq_scan.rs
new file mode 100644
index 0000000000000..93132ce65e51c
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/batch_log_seq_scan.rs
@@ -0,0 +1,142 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use pretty_xmlish::{Pretty, XmlNode};
+use risingwave_pb::batch_plan::plan_node::NodeBody;
+use risingwave_pb::batch_plan::LogRowSeqScanNode;
+use risingwave_pb::common::BatchQueryEpoch;
+
+use super::batch::prelude::*;
+use super::utils::{childless_record, Distill};
+use super::{generic, ExprRewritable, PlanBase, PlanRef, ToDistributedBatch, TryToBatchPb};
+use crate::catalog::ColumnId;
+use crate::error::Result;
+use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::plan_node::ToLocalBatch;
+use crate::optimizer::property::{Distribution, DistributionDisplay, Order};
+use crate::scheduler::SchedulerResult;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct BatchLogSeqScan {
+    pub base: PlanBase<Batch>,
+    core: generic::LogScan,
+}
+
+impl BatchLogSeqScan {
+    fn new_inner(core: generic::LogScan, dist: Distribution) -> Self {
+        let order = Order::any();
+        let base = PlanBase::new_batch(core.ctx(), core.schema(), dist, order);
+
+        Self { base, core }
+    }
+
+    pub fn new(core: generic::LogScan) -> Self {
+        // Use `Single` by default, will be updated later with `clone_with_dist`.
+        Self::new_inner(core, Distribution::Single)
+    }
+
+    fn clone_with_dist(&self) -> Self {
+        Self::new_inner(
+            self.core.clone(),
+            match self.core.distribution_key() {
+                None => Distribution::SomeShard,
+                Some(distribution_key) => {
+                    if distribution_key.is_empty() {
+                        Distribution::Single
+                    } else {
+                        Distribution::UpstreamHashShard(
+                            distribution_key,
+                            self.core.table_desc.table_id,
+                        )
+                    }
+                }
+            },
+        )
+    }
+
+    /// Get a reference to the batch seq scan's logical.
+    #[must_use]
+    pub fn core(&self) -> &generic::LogScan {
+        &self.core
+    }
+}
+
+impl_plan_tree_node_for_leaf! { BatchLogSeqScan }
+
+impl Distill for BatchLogSeqScan {
+    fn distill<'a>(&self) -> XmlNode<'a> {
+        let verbose = self.base.ctx().is_explain_verbose();
+        let mut vec = Vec::with_capacity(3);
+        vec.push(("table", Pretty::from(self.core.table_name.clone())));
+        vec.push(("columns", self.core.columns_pretty(verbose)));
+
+        if verbose {
+            let dist = Pretty::display(&DistributionDisplay {
+                distribution: self.distribution(),
+                input_schema: self.base.schema(),
+            });
+            vec.push(("distribution", dist));
+        }
+
+        childless_record("BatchScan", vec)
+    }
+}
+
+impl ToDistributedBatch for BatchLogSeqScan {
+    fn to_distributed(&self) -> Result<PlanRef> {
+        Ok(self.clone_with_dist().into())
+    }
+}
+
+impl TryToBatchPb for BatchLogSeqScan {
+    fn try_to_batch_prost_body(&self) -> SchedulerResult<NodeBody> {
+        Ok(NodeBody::LogRowSeqScan(LogRowSeqScanNode {
+            table_desc: Some(self.core.table_desc.try_to_protobuf()?),
+            column_ids: self
+                .core
+                .output_column_ids()
+                .iter()
+                .map(ColumnId::get_id)
+                .collect(),
+            vnode_bitmap: None,
+            old_epoch: Some(BatchQueryEpoch {
+                epoch: Some(risingwave_pb::common::batch_query_epoch::Epoch::Committed(
+                    self.core.old_epoch,
+                )),
+            }),
+            new_epoch: Some(BatchQueryEpoch {
+                epoch: Some(risingwave_pb::common::batch_query_epoch::Epoch::Committed(
+                    self.core.new_epoch,
+                )),
+            }),
+        }))
+    }
+}
+
+impl ToLocalBatch for BatchLogSeqScan {
+    fn to_local(&self) -> Result<PlanRef> {
+        let dist = if let Some(distribution_key) = self.core.distribution_key()
+            && !distribution_key.is_empty()
+        {
+            Distribution::UpstreamHashShard(distribution_key, self.core.table_desc.table_id)
+        } else {
+            Distribution::SomeShard
+        };
+        Ok(Self::new_inner(self.core.clone(), dist).into())
+    }
+}
+
+impl ExprRewritable for BatchLogSeqScan {}
+
+impl ExprVisitable for BatchLogSeqScan {}
diff --git a/src/frontend/src/optimizer/plan_node/batch_lookup_join.rs b/src/frontend/src/optimizer/plan_node/batch_lookup_join.rs
index a20b4ad316cd7..a181c590e0e65 100644
--- a/src/frontend/src/optimizer/plan_node/batch_lookup_join.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_lookup_join.rs
@@ -18,9 +18,8 @@ use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::{DistributedLookupJoinNode, LocalLookupJoinNode};
 
 use super::batch::prelude::*;
-use super::generic::{self, GenericPlanRef};
 use super::utils::{childless_record, Distill};
-use super::ExprRewritable;
+use super::{generic, ExprRewritable};
 use crate::error::Result;
 use crate::expr::{Expr, ExprRewriter, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
diff --git a/src/frontend/src/optimizer/plan_node/batch_max_one_row.rs b/src/frontend/src/optimizer/plan_node/batch_max_one_row.rs
index 94b8b3e6e0483..e35d6976672c4 100644
--- a/src/frontend/src/optimizer/plan_node/batch_max_one_row.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_max_one_row.rs
@@ -17,7 +17,7 @@ use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::MaxOneRowNode;
 
 use super::batch::prelude::*;
-use super::generic::{DistillUnit, PhysicalPlanRef};
+use super::generic::DistillUnit;
 use super::utils::Distill;
 use super::{
     generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, ToBatchPb, ToDistributedBatch,
diff --git a/src/frontend/src/optimizer/plan_node/batch_nested_loop_join.rs b/src/frontend/src/optimizer/plan_node/batch_nested_loop_join.rs
index 401166b1a298a..5d07aad86017b 100644
--- a/src/frontend/src/optimizer/plan_node/batch_nested_loop_join.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_nested_loop_join.rs
@@ -17,9 +17,10 @@ use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::NestedLoopJoinNode;
 
 use super::batch::prelude::*;
-use super::generic::{self, GenericPlanRef};
 use super::utils::{childless_record, Distill};
-use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeBinary, ToBatchPb, ToDistributedBatch};
+use super::{
+    generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeBinary, ToBatchPb, ToDistributedBatch,
+};
 use crate::error::Result;
 use crate::expr::{Expr, ExprImpl, ExprRewriter, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
diff --git a/src/frontend/src/optimizer/plan_node/batch_over_window.rs b/src/frontend/src/optimizer/plan_node/batch_over_window.rs
index 1712e87bbabbf..b9355fb117098 100644
--- a/src/frontend/src/optimizer/plan_node/batch_over_window.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_over_window.rs
@@ -17,7 +17,6 @@ use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::SortOverWindowNode;
 
 use super::batch::prelude::*;
-use super::batch::BatchPlanRef;
 use super::generic::PlanWindowFunction;
 use super::utils::impl_distill_by_unit;
 use super::{
diff --git a/src/frontend/src/optimizer/plan_node/batch_simple_agg.rs b/src/frontend/src/optimizer/plan_node/batch_simple_agg.rs
index a112e36c3a98f..ff20df7a4d177 100644
--- a/src/frontend/src/optimizer/plan_node/batch_simple_agg.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_simple_agg.rs
@@ -16,7 +16,7 @@ use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::SortAggNode;
 
 use super::batch::prelude::*;
-use super::generic::{self, GenericPlanRef, PlanAggCall};
+use super::generic::{self, PlanAggCall};
 use super::utils::impl_distill_by_unit;
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, ToBatchPb, ToDistributedBatch};
 use crate::error::Result;
diff --git a/src/frontend/src/optimizer/plan_node/batch_sort.rs b/src/frontend/src/optimizer/plan_node/batch_sort.rs
index 413103ee98c89..9136e3f7c685a 100644
--- a/src/frontend/src/optimizer/plan_node/batch_sort.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_sort.rs
@@ -17,7 +17,6 @@ use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::SortNode;
 
 use super::batch::prelude::*;
-use super::batch::BatchPlanRef;
 use super::utils::{childless_record, Distill};
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, ToBatchPb, ToDistributedBatch};
 use crate::error::Result;
diff --git a/src/frontend/src/optimizer/plan_node/batch_sort_agg.rs b/src/frontend/src/optimizer/plan_node/batch_sort_agg.rs
index 7393c10edc8f0..f8ea17dcd837e 100644
--- a/src/frontend/src/optimizer/plan_node/batch_sort_agg.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_sort_agg.rs
@@ -17,7 +17,7 @@ use risingwave_pb::batch_plan::SortAggNode;
 use risingwave_pb::expr::ExprNode;
 
 use super::batch::prelude::*;
-use super::generic::{self, GenericPlanRef, PlanAggCall};
+use super::generic::{self, PlanAggCall};
 use super::utils::impl_distill_by_unit;
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, ToBatchPb, ToDistributedBatch};
 use crate::error::Result;
diff --git a/src/frontend/src/optimizer/plan_node/batch_topn.rs b/src/frontend/src/optimizer/plan_node/batch_topn.rs
index d8f96668f88a7..643ef0fcc01fc 100644
--- a/src/frontend/src/optimizer/plan_node/batch_topn.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_topn.rs
@@ -22,7 +22,6 @@ use super::{
     generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, ToBatchPb, ToDistributedBatch,
 };
 use crate::error::Result;
-use crate::optimizer::plan_node::batch::BatchPlanRef;
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::{BatchLimit, ToLocalBatch};
 use crate::optimizer::property::{Order, RequiredDist};
diff --git a/src/frontend/src/optimizer/plan_node/batch_update.rs b/src/frontend/src/optimizer/plan_node/batch_update.rs
index 34d1eb3db3d74..d0351e6fdec2e 100644
--- a/src/frontend/src/optimizer/plan_node/batch_update.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_update.rs
@@ -18,7 +18,6 @@ use risingwave_pb::batch_plan::plan_node::NodeBody;
 use risingwave_pb::batch_plan::UpdateNode;
 
 use super::batch::prelude::*;
-use super::generic::GenericPlanRef;
 use super::utils::impl_distill_by_unit;
 use super::{
     generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, ToBatchPb, ToDistributedBatch,
diff --git a/src/frontend/src/optimizer/plan_node/convert.rs b/src/frontend/src/optimizer/plan_node/convert.rs
index a39ed7d3b048d..db961b3b1e20a 100644
--- a/src/frontend/src/optimizer/plan_node/convert.rs
+++ b/src/frontend/src/optimizer/plan_node/convert.rs
@@ -19,8 +19,7 @@ use risingwave_common::catalog::FieldDisplay;
 use risingwave_pb::stream_plan::StreamScanType;
 
 use super::*;
-use crate::optimizer::property::{Order, RequiredDist};
-use crate::utils::ColIndexMapping;
+use crate::optimizer::property::RequiredDist;
 use crate::{for_batch_plan_nodes, for_logical_plan_nodes, for_stream_plan_nodes};
 
 /// `ToStream` converts a logical plan node to streaming physical node
diff --git a/src/frontend/src/optimizer/plan_node/expr_rewritable.rs b/src/frontend/src/optimizer/plan_node/expr_rewritable.rs
index 33deebde1cdf9..4a30a599742ed 100644
--- a/src/frontend/src/optimizer/plan_node/expr_rewritable.rs
+++ b/src/frontend/src/optimizer/plan_node/expr_rewritable.rs
@@ -12,10 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::ops::Deref;
-
 use super::*;
-use crate::expr::ExprRewriter;
 
 /// Rewrites expressions in a `PlanRef`. Due to `Share` operator,
 /// the `ExprRewriter` needs to be idempotent i.e., applying it more than once
diff --git a/src/frontend/src/optimizer/plan_node/expr_visitable.rs b/src/frontend/src/optimizer/plan_node/expr_visitable.rs
index e4454fa92a7e5..e5527dfa17785 100644
--- a/src/frontend/src/optimizer/plan_node/expr_visitable.rs
+++ b/src/frontend/src/optimizer/plan_node/expr_visitable.rs
@@ -12,10 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::ops::Deref;
-
 use super::*;
-use crate::expr::ExprVisitor;
 
 /// Vistis expressions in a `PlanRef`.
 /// To visit recursively, call `visit_exprs_recursive` on [`VisitExprsRecursive`].
diff --git a/src/frontend/src/optimizer/plan_node/generic/agg.rs b/src/frontend/src/optimizer/plan_node/generic/agg.rs
index faff0c415f925..07febf4d9b4c5 100644
--- a/src/frontend/src/optimizer/plan_node/generic/agg.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/agg.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::sync::Arc;
 use std::{fmt, vec};
 
 use fixedbitset::FixedBitSet;
@@ -30,6 +31,7 @@ use risingwave_pb::stream_plan::{agg_call_state, AggCallState as AggCallStatePb}
 
 use super::super::utils::TableCatalogBuilder;
 use super::{impl_distill_unit_from_fields, stream, GenericPlanNode, GenericPlanRef};
+use crate::catalog::function_catalog::FunctionCatalog;
 use crate::expr::{Expr, ExprRewriter, ExprVisitor, InputRef, InputRefDisplay, Literal};
 use crate::optimizer::optimizer_context::OptimizerContextRef;
 use crate::optimizer::plan_node::batch::BatchPlanRef;
@@ -520,6 +522,11 @@ impl<PlanRef: stream::StreamPlanRef> Agg<PlanRef> {
             .iter()
             .zip_eq_fast(&mut out_fields[self.group_key.len()..])
         {
+            if agg_call.agg_kind == AggKind::UserDefined {
+                // for user defined aggregate, the state type is always BYTEA
+                field.data_type = DataType::Bytea;
+                continue;
+            }
             let sig = FUNCTION_REGISTRY
                 .get(
                     agg_call.agg_kind,
@@ -703,6 +710,9 @@ pub struct PlanAggCall {
     /// `filter` evaluates to `true` will be fed to the aggregate function.
     pub filter: Condition,
     pub direct_args: Vec<Literal>,
+
+    /// Catalog of user defined aggregate function.
+    pub user_defined: Option<Arc<FunctionCatalog>>,
 }
 
 impl fmt::Debug for PlanAggCall {
@@ -771,6 +781,7 @@ impl PlanAggCall {
                     r#type: Some(x.return_type().to_protobuf()),
                 })
                 .collect(),
+            udf: self.user_defined.as_ref().map(|c| c.as_ref().into()),
         }
     }
 
@@ -797,6 +808,7 @@ impl PlanAggCall {
             order_by: vec![],
             filter: Condition::true_cond(),
             direct_args: vec![],
+            user_defined: None,
         }
     }
 
diff --git a/src/frontend/src/optimizer/plan_node/generic/cdc_scan.rs b/src/frontend/src/optimizer/plan_node/generic/cdc_scan.rs
index f742238de8912..2d7d708291e47 100644
--- a/src/frontend/src/optimizer/plan_node/generic/cdc_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/cdc_scan.rs
@@ -13,19 +13,28 @@
 // limitations under the License.
 
 use std::rc::Rc;
+use std::str::FromStr;
 
+use anyhow::anyhow;
 use educe::Educe;
 use fixedbitset::FixedBitSet;
 use pretty_xmlish::Pretty;
 use risingwave_common::catalog::{CdcTableDesc, ColumnDesc, Field, Schema};
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_common::util::sort_util::ColumnOrder;
+use risingwave_connector::source::cdc::{
+    CDC_BACKFILL_ENABLE_KEY, CDC_BACKFILL_SNAPSHOT_BATCH_SIZE_KEY,
+    CDC_BACKFILL_SNAPSHOT_INTERVAL_KEY,
+};
+use risingwave_pb::stream_plan::StreamCdcScanOptions;
 
 use super::GenericPlanNode;
 use crate::catalog::ColumnId;
+use crate::error::Result;
 use crate::expr::{ExprRewriter, ExprVisitor};
 use crate::optimizer::optimizer_context::OptimizerContextRef;
 use crate::optimizer::property::FunctionalDependencySet;
+use crate::WithOptions;
 
 /// [`CdcScan`] reads rows of a table from an external upstream database
 #[derive(Debug, Clone, Educe)]
@@ -40,7 +49,59 @@ pub struct CdcScan {
     #[educe(Hash(ignore))]
     pub ctx: OptimizerContextRef,
 
+    pub options: CdcScanOptions,
+}
+
+#[derive(Debug, Clone, Hash, PartialEq)]
+pub struct CdcScanOptions {
     pub disable_backfill: bool,
+    pub snapshot_barrier_interval: u32,
+    pub snapshot_batch_size: u32,
+}
+
+impl Default for CdcScanOptions {
+    fn default() -> Self {
+        Self {
+            disable_backfill: false,
+            snapshot_barrier_interval: 1,
+            snapshot_batch_size: 1000,
+        }
+    }
+}
+
+impl CdcScanOptions {
+    pub fn from_with_options(with_options: &WithOptions) -> Result<Self> {
+        // unspecified option will use default values
+        let mut scan_options = Self::default();
+
+        // disable backfill if 'snapshot=false'
+        if let Some(snapshot) = with_options.get(CDC_BACKFILL_ENABLE_KEY) {
+            scan_options.disable_backfill = !(bool::from_str(snapshot)
+                .map_err(|_| anyhow!("Invalid value for {}", CDC_BACKFILL_ENABLE_KEY))?);
+        };
+
+        if let Some(snapshot_interval) = with_options.get(CDC_BACKFILL_SNAPSHOT_INTERVAL_KEY) {
+            scan_options.snapshot_barrier_interval = u32::from_str(snapshot_interval)
+                .map_err(|_| anyhow!("Invalid value for {}", CDC_BACKFILL_SNAPSHOT_INTERVAL_KEY))?;
+        };
+
+        if let Some(snapshot_batch_size) = with_options.get(CDC_BACKFILL_SNAPSHOT_BATCH_SIZE_KEY) {
+            scan_options.snapshot_batch_size =
+                u32::from_str(snapshot_batch_size).map_err(|_| {
+                    anyhow!("Invalid value for {}", CDC_BACKFILL_SNAPSHOT_BATCH_SIZE_KEY)
+                })?;
+        };
+
+        Ok(scan_options)
+    }
+
+    pub fn to_proto(&self) -> StreamCdcScanOptions {
+        StreamCdcScanOptions {
+            disable_backfill: self.disable_backfill,
+            snapshot_barrier_interval: self.snapshot_barrier_interval,
+            snapshot_batch_size: self.snapshot_batch_size,
+        }
+    }
 }
 
 impl CdcScan {
@@ -104,14 +165,14 @@ impl CdcScan {
         output_col_idx: Vec<usize>, // the column index in the table
         cdc_table_desc: Rc<CdcTableDesc>,
         ctx: OptimizerContextRef,
-        disable_backfill: bool,
+        options: CdcScanOptions,
     ) -> Self {
         Self {
             table_name,
             output_col_idx,
             cdc_table_desc,
             ctx,
-            disable_backfill,
+            options,
         }
     }
 
diff --git a/src/frontend/src/optimizer/plan_node/generic/cte_ref.rs b/src/frontend/src/optimizer/plan_node/generic/cte_ref.rs
new file mode 100644
index 0000000000000..6414b8dfdc0b2
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/generic/cte_ref.rs
@@ -0,0 +1,100 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::hash::Hash;
+
+use itertools::Itertools;
+use pretty_xmlish::{Pretty, StrAssocArr};
+use risingwave_common::catalog::Schema;
+
+use super::{impl_distill_unit_from_fields, GenericPlanNode, GenericPlanRef};
+use crate::binder::ShareId;
+use crate::optimizer::property::FunctionalDependencySet;
+use crate::{optimizer, OptimizerContextRef};
+
+#[derive(Clone, Debug)]
+pub struct CteRef<PlanRef> {
+    share_id: ShareId,
+    base: PlanRef,
+}
+
+impl<PlanRef> PartialEq for CteRef<PlanRef> {
+    fn eq(&self, other: &Self) -> bool {
+        self.share_id == other.share_id
+    }
+}
+
+impl<PlanRef> Eq for CteRef<PlanRef> {}
+
+impl<PlanRef> Hash for CteRef<PlanRef> {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.share_id.hash(state);
+    }
+}
+
+impl<PlanRef> CteRef<PlanRef> {
+    pub fn new(share_id: ShareId, base: PlanRef) -> Self {
+        Self { share_id, base }
+    }
+}
+
+impl<PlanRef: GenericPlanRef> CteRef<PlanRef> {
+    pub fn get_cte_ref(&self) -> Option<optimizer::plan_node::PlanRef> {
+        self.ctx().get_rcte_cache_plan(&self.share_id)
+    }
+}
+
+impl<PlanRef: GenericPlanRef> GenericPlanNode for CteRef<PlanRef> {
+    fn schema(&self) -> Schema {
+        if let Some(plan_ref) = self.get_cte_ref() {
+            plan_ref.schema().clone()
+        } else {
+            self.base.schema().clone()
+        }
+    }
+
+    fn stream_key(&self) -> Option<Vec<usize>> {
+        if let Some(plan_ref) = self.get_cte_ref() {
+            plan_ref
+                .stream_key()
+                .map(|s| s.iter().map(|i| i.to_owned()).collect_vec())
+        } else {
+            self.base
+                .stream_key()
+                .map(|s| s.iter().map(|i| i.to_owned()).collect_vec())
+        }
+    }
+
+    fn ctx(&self) -> OptimizerContextRef {
+        // it does not matter where the context is coming from,
+        // since we are only getting a reference.
+        self.base.ctx()
+    }
+
+    fn functional_dependency(&self) -> FunctionalDependencySet {
+        if let Some(plan_ref) = self.get_cte_ref() {
+            plan_ref.functional_dependency().clone()
+        } else {
+            self.base.functional_dependency().clone()
+        }
+    }
+}
+
+impl<PlanRef: GenericPlanRef> CteRef<PlanRef> {
+    pub fn fields_pretty<'a>(&self) -> StrAssocArr<'a> {
+        vec![("share_id", Pretty::debug(&self.share_id))]
+    }
+}
+
+impl_distill_unit_from_fields! {CteRef, GenericPlanRef}
diff --git a/src/frontend/src/optimizer/plan_node/generic/log_scan.rs b/src/frontend/src/optimizer/plan_node/generic/log_scan.rs
new file mode 100644
index 0000000000000..498d4a44b0fcc
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/generic/log_scan.rs
@@ -0,0 +1,160 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashMap;
+use std::rc::Rc;
+
+use educe::Educe;
+use pretty_xmlish::Pretty;
+use risingwave_common::catalog::{Field, Schema, TableDesc};
+use risingwave_common::types::DataType;
+use risingwave_common::util::sort_util::ColumnOrder;
+
+use crate::catalog::ColumnId;
+use crate::optimizer::optimizer_context::OptimizerContextRef;
+
+const OP_NAME: &str = "op";
+const OP_TYPE: DataType = DataType::Int16;
+
+#[derive(Debug, Clone, Educe)]
+#[educe(PartialEq, Eq, Hash)]
+pub struct LogScan {
+    pub table_name: String,
+    /// Include `output_col_idx` and `op_column`
+    pub output_col_idx: Vec<usize>,
+    /// Descriptor of the table
+    pub table_desc: Rc<TableDesc>,
+    /// Help `RowSeqLogScan` executor use a better chunk size
+    pub chunk_size: Option<u32>,
+
+    #[educe(PartialEq(ignore))]
+    #[educe(Hash(ignore))]
+    pub ctx: OptimizerContextRef,
+
+    pub old_epoch: u64,
+    pub new_epoch: u64,
+}
+
+impl LogScan {
+    // Used for create batch exec, without op
+    pub fn output_column_ids(&self) -> Vec<ColumnId> {
+        self.output_col_idx
+            .iter()
+            .map(|i| self.table_desc.columns[*i].column_id)
+            .collect()
+    }
+
+    pub fn primary_key(&self) -> &[ColumnOrder] {
+        &self.table_desc.pk
+    }
+
+    fn column_names_with_table_prefix(&self) -> Vec<String> {
+        let mut out_column_names: Vec<_> = self
+            .output_col_idx
+            .iter()
+            .map(|&i| format!("{}.{}", self.table_name, self.table_desc.columns[i].name))
+            .collect();
+        out_column_names.push(format!("{}.{}", self.table_name, OP_NAME));
+        out_column_names
+    }
+
+    pub(crate) fn column_names(&self) -> Vec<String> {
+        let mut out_column_names: Vec<_> = self
+            .output_col_idx
+            .iter()
+            .map(|&i| self.table_desc.columns[i].name.clone())
+            .collect();
+        out_column_names.push(OP_NAME.to_string());
+        out_column_names
+    }
+
+    pub fn distribution_key(&self) -> Option<Vec<usize>> {
+        let tb_idx_to_op_idx = self
+            .output_col_idx
+            .iter()
+            .enumerate()
+            .map(|(op_idx, tb_idx)| (*tb_idx, op_idx))
+            .collect::<HashMap<_, _>>();
+        self.table_desc
+            .distribution_key
+            .iter()
+            .map(|&tb_idx| tb_idx_to_op_idx.get(&tb_idx).cloned())
+            .collect()
+    }
+
+    /// Create a logical scan node for log table scan
+    pub(crate) fn new(
+        table_name: String,
+        output_col_idx: Vec<usize>,
+        table_desc: Rc<TableDesc>,
+        ctx: OptimizerContextRef,
+        old_epoch: u64,
+        new_epoch: u64,
+    ) -> Self {
+        Self {
+            table_name,
+            output_col_idx,
+            table_desc,
+            chunk_size: None,
+            ctx,
+            old_epoch,
+            new_epoch,
+        }
+    }
+
+    pub(crate) fn columns_pretty<'a>(&self, verbose: bool) -> Pretty<'a> {
+        Pretty::Array(
+            match verbose {
+                true => self.column_names_with_table_prefix(),
+                false => self.column_names(),
+            }
+            .into_iter()
+            .map(Pretty::from)
+            .collect(),
+        )
+    }
+
+    pub(crate) fn schema(&self) -> Schema {
+        let mut fields: Vec<_> = self
+            .output_col_idx
+            .iter()
+            .map(|tb_idx| {
+                let col = &self.table_desc.columns[*tb_idx];
+                Field::from_with_table_name_prefix(col, &self.table_name)
+            })
+            .collect();
+        fields.push(Field::with_name(
+            OP_TYPE,
+            format!("{}.{}", &self.table_name, OP_NAME),
+        ));
+        Schema { fields }
+    }
+
+    pub(crate) fn schema_without_table_name(&self) -> Schema {
+        let mut fields: Vec<_> = self
+            .output_col_idx
+            .iter()
+            .map(|tb_idx| {
+                let col = &self.table_desc.columns[*tb_idx];
+                Field::from(col)
+            })
+            .collect();
+        fields.push(Field::with_name(OP_TYPE, OP_NAME));
+        Schema { fields }
+    }
+
+    pub(crate) fn ctx(&self) -> OptimizerContextRef {
+        self.ctx.clone()
+    }
+}
diff --git a/src/frontend/src/optimizer/plan_node/generic/mod.rs b/src/frontend/src/optimizer/plan_node/generic/mod.rs
index 5154c84017b87..00db5730e8038 100644
--- a/src/frontend/src/optimizer/plan_node/generic/mod.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/mod.rs
@@ -44,6 +44,8 @@ mod table_scan;
 pub use table_scan::*;
 mod sys_scan;
 pub use sys_scan::*;
+mod log_scan;
+pub use log_scan::*;
 
 mod cdc_scan;
 pub use cdc_scan::*;
@@ -72,6 +74,10 @@ mod limit;
 pub use limit::*;
 mod max_one_row;
 pub use max_one_row::*;
+mod cte_ref;
+pub use cte_ref::*;
+mod recursive_union;
+pub use recursive_union::*;
 
 pub trait DistillUnit {
     fn distill_with_name<'a>(&self, name: impl Into<Cow<'a, str>>) -> XmlNode<'a>;
diff --git a/src/frontend/src/optimizer/plan_node/generic/project.rs b/src/frontend/src/optimizer/plan_node/generic/project.rs
index 10329c9f8ca91..68c652f0e006f 100644
--- a/src/frontend/src/optimizer/plan_node/generic/project.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/project.rs
@@ -86,7 +86,7 @@ impl<PlanRef: GenericPlanRef> GenericPlanNode for Project<PlanRef> {
                     Some(input_idx) => {
                         let mut field = input_schema.fields()[input_idx].clone();
                         if let Some(name) = self.field_names.get(&i) {
-                            field.name = name.clone();
+                            field.name.clone_from(name);
                         }
                         (field.name, field.sub_fields, field.type_name)
                     }
diff --git a/src/frontend/src/optimizer/plan_node/generic/recursive_union.rs b/src/frontend/src/optimizer/plan_node/generic/recursive_union.rs
new file mode 100644
index 0000000000000..023718f1a9682
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/generic/recursive_union.rs
@@ -0,0 +1,64 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use pretty_xmlish::StrAssocArr;
+use risingwave_common::catalog::Schema;
+
+use super::{impl_distill_unit_from_fields, GenericPlanNode, GenericPlanRef};
+use crate::binder::ShareId;
+use crate::optimizer::property::FunctionalDependencySet;
+use crate::OptimizerContextRef;
+
+/// `RecursiveUnion` returns the union of the rows of its inputs.
+/// note: if `all` is false, it needs to eliminate duplicates.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct RecursiveUnion<PlanRef> {
+    pub id: ShareId,
+    pub base: PlanRef,
+    pub recursive: PlanRef,
+}
+
+impl<PlanRef: GenericPlanRef> GenericPlanNode for RecursiveUnion<PlanRef> {
+    fn functional_dependency(&self) -> FunctionalDependencySet {
+        self.recursive.functional_dependency().clone()
+    }
+
+    fn schema(&self) -> Schema {
+        self.recursive.schema().clone()
+    }
+
+    fn stream_key(&self) -> Option<Vec<usize>> {
+        let fields_len = self.base.schema().len();
+        let base = self.base.stream_key();
+        if let Some(base) = base {
+            let mut base = base.to_vec();
+            base.push(fields_len);
+            Some(base)
+        } else {
+            None
+        }
+    }
+
+    fn ctx(&self) -> OptimizerContextRef {
+        self.recursive.ctx()
+    }
+}
+
+impl<PlanRef: GenericPlanRef> RecursiveUnion<PlanRef> {
+    pub fn fields_pretty<'a>(&self) -> StrAssocArr<'a> {
+        vec![]
+    }
+}
+
+impl_distill_unit_from_fields!(RecursiveUnion, GenericPlanRef);
diff --git a/src/frontend/src/optimizer/plan_node/generic/source.rs b/src/frontend/src/optimizer/plan_node/generic/source.rs
index 40bd376dd46ca..af138f7dd80a5 100644
--- a/src/frontend/src/optimizer/plan_node/generic/source.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/source.rs
@@ -137,13 +137,27 @@ impl Source {
         (self, original_row_id_index)
     }
 
-    pub fn infer_internal_table_catalog(require_dist_key: bool) -> TableCatalog {
-        // note that source's internal table is to store partition_id -> offset mapping and its
-        // schema is irrelevant to input schema
-        // On the premise of ensuring that the materialized_source data can be cleaned up, keep the
-        // state in source.
-        // Source state doesn't maintain retention_seconds, internal_table_subset function only
-        // returns retention_seconds so default is used here
+    /// Source's state table is `partition_id -> offset_info`.
+    /// Its schema is irrelevant to the data's schema.
+    ///
+    /// ## Notes on the distribution of the state table (`is_distributed`)
+    ///
+    /// Source executors are always distributed, but their state tables are special.
+    ///
+    /// ### `StreamSourceExecutor`: singleton (only one vnode)
+    ///
+    /// Its states are not sharded by consistent hash.
+    ///
+    /// Each actor accesses (point get) some partitions (a.k.a splits).
+    /// They are assigned by `SourceManager` in meta,
+    /// instead of `vnode` computed from the `partition_id`.
+    ///
+    /// ### `StreamFsFetch`: distributed by `partition_id`
+    ///
+    /// Each actor accesses (range scan) splits according to the `vnode`
+    /// computed from `partition_id`.
+    /// This is a normal distributed table.
+    pub fn infer_internal_table_catalog(is_distributed: bool) -> TableCatalog {
         let mut builder = TableCatalogBuilder::default();
 
         let key = Field {
@@ -164,7 +178,7 @@ impl Source {
         builder.add_order_column(ordered_col_idx, OrderType::ascending());
 
         builder.build(
-            if require_dist_key {
+            if is_distributed {
                 vec![ordered_col_idx]
             } else {
                 vec![]
diff --git a/src/frontend/src/optimizer/plan_node/logical_agg.rs b/src/frontend/src/optimizer/plan_node/logical_agg.rs
index cad073386a42e..f32b37d250b5a 100644
--- a/src/frontend/src/optimizer/plan_node/logical_agg.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_agg.rs
@@ -18,7 +18,6 @@ use risingwave_common::types::{DataType, Datum, ScalarImpl};
 use risingwave_common::util::sort_util::ColumnOrder;
 use risingwave_common::{bail_not_implemented, not_implemented};
 use risingwave_expr::aggregate::{agg_kinds, AggKind};
-use risingwave_expr::sig::FUNCTION_REGISTRY;
 
 use super::generic::{self, Agg, GenericPlanRef, PlanAggCall, ProjectBuilder};
 use super::utils::impl_distill_by_unit;
@@ -27,7 +26,7 @@ use super::{
     PlanTreeNodeUnary, PredicatePushdown, StreamHashAgg, StreamProject, StreamSimpleAgg,
     StreamStatelessSimpleAgg, ToBatch, ToStream,
 };
-use crate::error::{ErrorCode, Result};
+use crate::error::{ErrorCode, Result, RwError};
 use crate::expr::{
     AggCall, Expr, ExprImpl, ExprRewriter, ExprType, ExprVisitor, FunctionCall, InputRef, Literal,
     OrderBy, WindowFunction,
@@ -262,7 +261,7 @@ pub struct LogicalAggBuilder {
     /// the agg calls
     agg_calls: Vec<PlanAggCall>,
     /// the error during the expression rewriting
-    error: Option<ErrorCode>,
+    error: Option<RwError>,
     /// If `is_in_filter_clause` is true, it means that
     /// we are processing filter clause.
     /// This field is needed because input refs in these clauses
@@ -354,7 +353,7 @@ impl LogicalAggBuilder {
     fn rewrite_with_error(&mut self, expr: ExprImpl) -> Result<ExprImpl> {
         let rewritten_expr = self.rewrite_expr(expr);
         if let Some(error) = self.error.take() {
-            return Err(error.into());
+            return Err(error);
         }
         Ok(rewritten_expr)
     }
@@ -377,51 +376,159 @@ impl LogicalAggBuilder {
         self.group_key.len()
     }
 
-    /// Push a new planned agg call into the builder.
-    /// Return an `InputRef` to that agg call.
-    /// For existing agg calls, return an `InputRef` to the existing one.
-    fn push_agg_call(&mut self, agg_call: PlanAggCall) -> InputRef {
-        if let Some((pos, existing)) = self.agg_calls.iter().find_position(|&c| c == &agg_call) {
-            return InputRef::new(
-                self.schema_agg_start_offset() + pos,
-                existing.return_type.clone(),
-            );
-        }
-        let index = self.schema_agg_start_offset() + self.agg_calls.len();
-        let data_type = agg_call.return_type.clone();
-        self.agg_calls.push(agg_call);
-        InputRef::new(index, data_type)
-    }
-
-    /// When there is an agg call, there are 3 things to do:
-    /// 1. eval its inputs via project;
-    /// 2. add a `PlanAggCall` to agg;
-    /// 3. rewrite it as an `InputRef` to the agg result in select list.
-    ///
-    /// Note that the rewriter does not traverse into inputs of agg calls.
-    fn try_rewrite_agg_call(
-        &mut self,
+    /// Rewrite [`AggCall`] if needed, and push it into the builder using `push_agg_call`.
+    /// This is shared by [`LogicalAggBuilder`] and `LogicalOverWindowBuilder`.
+    pub(crate) fn general_rewrite_agg_call(
         agg_call: AggCall,
-    ) -> std::result::Result<ExprImpl, ErrorCode> {
-        let return_type = agg_call.return_type();
-        let (agg_kind, inputs, mut distinct, mut order_by, filter, direct_args) =
-            agg_call.decompose();
+        mut push_agg_call: impl FnMut(AggCall) -> Result<InputRef>,
+    ) -> Result<ExprImpl> {
+        match agg_call.agg_kind {
+            // Rewrite avg to cast(sum as avg_return_type) / count.
+            AggKind::Avg => {
+                assert_eq!(agg_call.args.len(), 1);
+
+                let sum = ExprImpl::from(push_agg_call(AggCall::new(
+                    AggKind::Sum,
+                    agg_call.args.clone(),
+                    agg_call.distinct,
+                    agg_call.order_by.clone(),
+                    agg_call.filter.clone(),
+                    agg_call.direct_args.clone(),
+                )?)?)
+                .cast_explicit(agg_call.return_type())?;
+
+                let count = ExprImpl::from(push_agg_call(AggCall::new(
+                    AggKind::Count,
+                    agg_call.args.clone(),
+                    agg_call.distinct,
+                    agg_call.order_by.clone(),
+                    agg_call.filter.clone(),
+                    agg_call.direct_args.clone(),
+                )?)?);
+
+                Ok(FunctionCall::new(ExprType::Divide, Vec::from([sum, count]))?.into())
+            }
+            // We compute `var_samp` as
+            // (sum(sq) - sum * sum / count) / (count - 1)
+            // and `var_pop` as
+            // (sum(sq) - sum * sum / count) / count
+            // Since we don't have the square function, we use the plain Multiply for squaring,
+            // which is in a sense more general than the pow function, especially when calculating
+            // covariances in the future. Also we don't have the sqrt function for rooting, so we
+            // use pow(x, 0.5) to simulate
+            kind @ (AggKind::StddevPop
+            | AggKind::StddevSamp
+            | AggKind::VarPop
+            | AggKind::VarSamp) => {
+                let arg = agg_call.args().iter().exactly_one().unwrap();
+                let squared_arg = ExprImpl::from(FunctionCall::new(
+                    ExprType::Multiply,
+                    vec![arg.clone(), arg.clone()],
+                )?);
+
+                let sum_of_sq = ExprImpl::from(push_agg_call(AggCall::new(
+                    AggKind::Sum,
+                    vec![squared_arg],
+                    agg_call.distinct,
+                    agg_call.order_by.clone(),
+                    agg_call.filter.clone(),
+                    agg_call.direct_args.clone(),
+                )?)?)
+                .cast_explicit(agg_call.return_type())?;
+
+                let sum = ExprImpl::from(push_agg_call(AggCall::new(
+                    AggKind::Sum,
+                    agg_call.args.clone(),
+                    agg_call.distinct,
+                    agg_call.order_by.clone(),
+                    agg_call.filter.clone(),
+                    agg_call.direct_args.clone(),
+                )?)?)
+                .cast_explicit(agg_call.return_type())?;
+
+                let count = ExprImpl::from(push_agg_call(AggCall::new(
+                    AggKind::Count,
+                    agg_call.args.clone(),
+                    agg_call.distinct,
+                    agg_call.order_by.clone(),
+                    agg_call.filter.clone(),
+                    agg_call.direct_args.clone(),
+                )?)?);
+
+                let one = ExprImpl::from(Literal::new(
+                    Datum::from(ScalarImpl::Int64(1)),
+                    DataType::Int64,
+                ));
 
-        if matches!(agg_kind, agg_kinds::must_have_order_by!()) && order_by.sort_exprs.is_empty() {
-            return Err(ErrorCode::InvalidInputSyntax(format!(
-                "Aggregation function {} requires ORDER BY clause",
-                agg_kind
-            )));
-        }
+                let squared_sum = ExprImpl::from(FunctionCall::new(
+                    ExprType::Multiply,
+                    vec![sum.clone(), sum],
+                )?);
+
+                let numerator = ExprImpl::from(FunctionCall::new(
+                    ExprType::Subtract,
+                    vec![
+                        sum_of_sq,
+                        ExprImpl::from(FunctionCall::new(
+                            ExprType::Divide,
+                            vec![squared_sum, count.clone()],
+                        )?),
+                    ],
+                )?);
+
+                let denominator = match kind {
+                    AggKind::VarPop | AggKind::StddevPop => count.clone(),
+                    AggKind::VarSamp | AggKind::StddevSamp => ExprImpl::from(FunctionCall::new(
+                        ExprType::Subtract,
+                        vec![count.clone(), one.clone()],
+                    )?),
+                    _ => unreachable!(),
+                };
 
-        // try ignore ORDER BY if it doesn't affect the result
-        if matches!(agg_kind, agg_kinds::result_unaffected_by_order_by!()) {
-            order_by = OrderBy::any();
-        }
-        // try ignore DISTINCT if it doesn't affect the result
-        if matches!(agg_kind, agg_kinds::result_unaffected_by_distinct!()) {
-            distinct = false;
+                let mut target = ExprImpl::from(FunctionCall::new(
+                    ExprType::Divide,
+                    vec![numerator, denominator],
+                )?);
+
+                if matches!(kind, AggKind::StddevPop | AggKind::StddevSamp) {
+                    target = ExprImpl::from(FunctionCall::new(ExprType::Sqrt, vec![target])?);
+                }
+
+                match kind {
+                    AggKind::VarPop | AggKind::StddevPop => Ok(target),
+                    AggKind::StddevSamp | AggKind::VarSamp => {
+                        let case_cond = ExprImpl::from(FunctionCall::new(
+                            ExprType::LessThanOrEqual,
+                            vec![count, one],
+                        )?);
+                        let null = ExprImpl::from(Literal::new(None, agg_call.return_type()));
+
+                        Ok(ExprImpl::from(FunctionCall::new(
+                            ExprType::Case,
+                            vec![case_cond, null, target],
+                        )?))
+                    }
+                    _ => unreachable!(),
+                }
+            }
+            _ => Ok(push_agg_call(agg_call)?.into()),
         }
+    }
+
+    /// Push a new agg call into the builder.
+    /// Return an `InputRef` to that agg call.
+    /// For existing agg calls, return an `InputRef` to the existing one.
+    fn push_agg_call(&mut self, agg_call: AggCall) -> Result<InputRef> {
+        let AggCall {
+            agg_kind,
+            return_type,
+            args,
+            distinct,
+            order_by,
+            filter,
+            direct_args,
+            user_defined,
+        } = agg_call;
 
         self.is_in_filter_clause = true;
         // filter expr is not added to `input_proj_builder` as a whole. Special exprs incl
@@ -429,27 +536,7 @@ impl LogicalAggBuilder {
         let filter = filter.rewrite_expr(self);
         self.is_in_filter_clause = false;
 
-        if matches!(agg_kind, AggKind::Grouping) {
-            if self.grouping_sets.is_empty() {
-                return Err(ErrorCode::NotSupported(
-                    "GROUPING must be used in a query with grouping sets".into(),
-                    "try to use grouping sets instead".into(),
-                ));
-            }
-            if inputs.len() >= 32 {
-                return Err(ErrorCode::InvalidInputSyntax(
-                    "GROUPING must have fewer than 32 arguments".into(),
-                ));
-            }
-            if inputs.iter().any(|x| self.try_as_group_expr(x).is_none()) {
-                return Err(ErrorCode::InvalidInputSyntax(
-                    "arguments to GROUPING must be grouping expressions of the associated query level"
-                        .into(),
-                ));
-            }
-        }
-
-        let inputs: Vec<_> = inputs
+        let args: Vec<_> = args
             .iter()
             .map(|expr| {
                 let index = self.input_proj_builder.add_expr(expr)?;
@@ -470,220 +557,91 @@ impl LogicalAggBuilder {
                 not_implemented!("{err} inside aggregation calls order by")
             })?;
 
-        match agg_kind {
-            // Rewrite avg to cast(sum as avg_return_type) / count.
-            AggKind::Avg => {
-                assert_eq!(inputs.len(), 1);
-
-                let left_return_type = FUNCTION_REGISTRY
-                    .get_return_type(AggKind::Sum, &[inputs[0].return_type()])
-                    .unwrap();
-                let left_ref = self.push_agg_call(PlanAggCall {
-                    agg_kind: AggKind::Sum,
-                    return_type: left_return_type,
-                    inputs: inputs.clone(),
-                    distinct,
-                    order_by: order_by.clone(),
-                    filter: filter.clone(),
-                    direct_args: direct_args.clone(),
-                });
-                let left = ExprImpl::from(left_ref).cast_explicit(return_type).unwrap();
-
-                let right_return_type = FUNCTION_REGISTRY
-                    .get_return_type(AggKind::Count, &[inputs[0].return_type()])
-                    .unwrap();
-                let right_ref = self.push_agg_call(PlanAggCall {
-                    agg_kind: AggKind::Count,
-                    return_type: right_return_type,
-                    inputs,
-                    distinct,
-                    order_by,
-                    filter,
-                    direct_args,
-                });
-
-                Ok(ExprImpl::from(
-                    FunctionCall::new(ExprType::Divide, vec![left, right_ref.into()]).unwrap(),
-                ))
-            }
+        let plan_agg_call = PlanAggCall {
+            agg_kind,
+            return_type: return_type.clone(),
+            inputs: args,
+            distinct,
+            order_by,
+            filter,
+            direct_args,
+            user_defined,
+        };
 
-            // We compute `var_samp` as
-            // (sum(sq) - sum * sum / count) / (count - 1)
-            // and `var_pop` as
-            // (sum(sq) - sum * sum / count) / count
-            // Since we don't have the square function, we use the plain Multiply for squaring,
-            // which is in a sense more general than the pow function, especially when calculating
-            // covariances in the future. Also we don't have the sqrt function for rooting, so we
-            // use pow(x, 0.5) to simulate
-            AggKind::StddevPop | AggKind::StddevSamp | AggKind::VarPop | AggKind::VarSamp => {
-                let input = inputs.iter().exactly_one().unwrap();
-                let pre_proj_input = self.input_proj_builder.get_expr(input.index).unwrap();
-
-                // first, we compute sum of squared as sum_sq
-                let squared_input_expr = ExprImpl::from(
-                    FunctionCall::new(
-                        ExprType::Multiply,
-                        vec![pre_proj_input.clone(), pre_proj_input.clone()],
-                    )
-                    .unwrap(),
-                );
-
-                let squared_input_proj_index = self
-                    .input_proj_builder
-                    .add_expr(&squared_input_expr)
-                    .unwrap();
-
-                let sum_of_squares_return_type = FUNCTION_REGISTRY
-                    .get_return_type(AggKind::Sum, &[squared_input_expr.return_type()])
-                    .unwrap();
-
-                let sum_of_squares_expr = ExprImpl::from(self.push_agg_call(PlanAggCall {
-                    agg_kind: AggKind::Sum,
-                    return_type: sum_of_squares_return_type,
-                    inputs: vec![InputRef::new(
-                        squared_input_proj_index,
-                        squared_input_expr.return_type(),
-                    )],
-                    distinct,
-                    order_by: order_by.clone(),
-                    filter: filter.clone(),
-                    direct_args: direct_args.clone(),
-                }))
-                .cast_explicit(return_type.clone())
-                .unwrap();
-
-                // after that, we compute sum
-                let sum_return_type = FUNCTION_REGISTRY
-                    .get_return_type(AggKind::Sum, &[input.return_type()])
-                    .unwrap();
-
-                let sum_expr = ExprImpl::from(self.push_agg_call(PlanAggCall {
-                    agg_kind: AggKind::Sum,
-                    return_type: sum_return_type,
-                    inputs: inputs.clone(),
-                    distinct,
-                    order_by: order_by.clone(),
-                    filter: filter.clone(),
-                    direct_args: direct_args.clone(),
-                }))
-                .cast_explicit(return_type.clone())
-                .unwrap();
-
-                // then, we compute count
-                let count_return_type = FUNCTION_REGISTRY
-                    .get_return_type(AggKind::Count, &[input.return_type()])
-                    .unwrap();
-
-                let count_expr = ExprImpl::from(self.push_agg_call(PlanAggCall {
-                    agg_kind: AggKind::Count,
-                    return_type: count_return_type,
-                    inputs,
-                    distinct,
-                    order_by,
-                    filter,
-                    direct_args,
-                }));
-
-                // we start with variance
-
-                // sum * sum
-                let square_of_sum_expr = ExprImpl::from(
-                    FunctionCall::new(ExprType::Multiply, vec![sum_expr.clone(), sum_expr])
-                        .unwrap(),
-                );
-
-                // sum_sq - sum * sum / count
-                let numerator_expr = ExprImpl::from(
-                    FunctionCall::new(
-                        ExprType::Subtract,
-                        vec![
-                            sum_of_squares_expr,
-                            ExprImpl::from(
-                                FunctionCall::new(
-                                    ExprType::Divide,
-                                    vec![square_of_sum_expr, count_expr.clone()],
-                                )
-                                .unwrap(),
-                            ),
-                        ],
-                    )
-                    .unwrap(),
-                );
-
-                // count or count - 1
-                let denominator_expr = match agg_kind {
-                    AggKind::StddevPop | AggKind::VarPop => count_expr.clone(),
-                    AggKind::StddevSamp | AggKind::VarSamp => ExprImpl::from(
-                        FunctionCall::new(
-                            ExprType::Subtract,
-                            vec![
-                                count_expr.clone(),
-                                ExprImpl::from(Literal::new(
-                                    Datum::from(ScalarImpl::Int64(1)),
-                                    DataType::Int64,
-                                )),
-                            ],
-                        )
-                        .unwrap(),
-                    ),
-                    _ => unreachable!(),
-                };
+        if let Some((pos, existing)) = self
+            .agg_calls
+            .iter()
+            .find_position(|&c| c == &plan_agg_call)
+        {
+            return Ok(InputRef::new(
+                self.schema_agg_start_offset() + pos,
+                existing.return_type.clone(),
+            ));
+        }
+        let index = self.schema_agg_start_offset() + self.agg_calls.len();
+        self.agg_calls.push(plan_agg_call);
+        Ok(InputRef::new(index, return_type))
+    }
 
-                let mut target_expr = ExprImpl::from(
-                    FunctionCall::new(ExprType::Divide, vec![numerator_expr, denominator_expr])
-                        .unwrap(),
-                );
+    /// When there is an agg call, there are 3 things to do:
+    /// 1. Rewrite `avg`, `var_samp`, etc. into a combination of `sum`, `count`, etc.;
+    /// 2. Add exprs in arguments to input `Project`;
+    /// 2. Add the agg call to current `Agg`, and return an `InputRef` to it.
+    ///
+    /// Note that the rewriter does not traverse into inputs of agg calls.
+    fn try_rewrite_agg_call(&mut self, mut agg_call: AggCall) -> Result<ExprImpl> {
+        if matches!(agg_call.agg_kind, agg_kinds::must_have_order_by!())
+            && agg_call.order_by.sort_exprs.is_empty()
+        {
+            return Err(ErrorCode::InvalidInputSyntax(format!(
+                "Aggregation function {} requires ORDER BY clause",
+                agg_call.agg_kind
+            ))
+            .into());
+        }
 
-                // stddev = sqrt(variance)
-                if matches!(agg_kind, AggKind::StddevPop | AggKind::StddevSamp) {
-                    target_expr = ExprImpl::from(
-                        FunctionCall::new(ExprType::Sqrt, vec![target_expr]).unwrap(),
-                    );
-                }
+        // try ignore ORDER BY if it doesn't affect the result
+        if matches!(
+            agg_call.agg_kind,
+            agg_kinds::result_unaffected_by_order_by!()
+        ) {
+            agg_call.order_by = OrderBy::any();
+        }
+        // try ignore DISTINCT if it doesn't affect the result
+        if matches!(
+            agg_call.agg_kind,
+            agg_kinds::result_unaffected_by_distinct!()
+        ) {
+            agg_call.distinct = false;
+        }
 
-                match agg_kind {
-                    AggKind::VarPop | AggKind::StddevPop => Ok(target_expr),
-                    AggKind::StddevSamp | AggKind::VarSamp => {
-                        let less_than_expr = ExprImpl::from(
-                            FunctionCall::new(
-                                ExprType::LessThanOrEqual,
-                                vec![
-                                    count_expr,
-                                    ExprImpl::from(Literal::new(
-                                        Datum::from(ScalarImpl::Int64(1)),
-                                        DataType::Int64,
-                                    )),
-                                ],
-                            )
-                            .unwrap(),
-                        );
-                        let null_expr = ExprImpl::from(Literal::new(None, return_type));
-
-                        let case_expr = ExprImpl::from(
-                            FunctionCall::new(
-                                ExprType::Case,
-                                vec![less_than_expr, null_expr, target_expr],
-                            )
-                            .unwrap(),
-                        );
-
-                        Ok(case_expr)
-                    }
-                    _ => unreachable!(),
-                }
+        if matches!(agg_call.agg_kind, AggKind::Grouping) {
+            if self.grouping_sets.is_empty() {
+                return Err(ErrorCode::NotSupported(
+                    "GROUPING must be used in a query with grouping sets".into(),
+                    "try to use grouping sets instead".into(),
+                )
+                .into());
+            }
+            if agg_call.args.len() >= 32 {
+                return Err(ErrorCode::InvalidInputSyntax(
+                    "GROUPING must have fewer than 32 arguments".into(),
+                )
+                .into());
+            }
+            if agg_call
+                .args
+                .iter()
+                .any(|x| self.try_as_group_expr(x).is_none())
+            {
+                return Err(ErrorCode::InvalidInputSyntax(
+                    "arguments to GROUPING must be grouping expressions of the associated query level"
+                        .into(),
+                ).into());
             }
-            _ => Ok(self
-                .push_agg_call(PlanAggCall {
-                    agg_kind,
-                    return_type,
-                    inputs,
-                    distinct,
-                    order_by,
-                    filter,
-                    direct_args,
-                })
-                .into()),
         }
+
+        Self::general_rewrite_agg_call(agg_call, |agg_call| self.push_agg_call(agg_call))
     }
 }
 
@@ -754,10 +712,13 @@ impl ExprRewriter for LogicalAggBuilder {
             )
             .into()
         } else {
-            self.error = Some(ErrorCode::InvalidInputSyntax(
-                "column must appear in the GROUP BY clause or be used in an aggregate function"
-                    .into(),
-            ));
+            self.error = Some(
+                ErrorCode::InvalidInputSyntax(
+                    "column must appear in the GROUP BY clause or be used in an aggregate function"
+                        .into(),
+                )
+                .into(),
+            );
             expr
         }
     }
@@ -1179,10 +1140,16 @@ impl ToStream for LogicalAgg {
             Ok(plan)
         } else {
             // a `count(*)` is appended, should project the output
+            assert_eq!(self.agg_calls().len() + 1, n_final_agg_calls);
             Ok(StreamProject::new(generic::Project::with_out_col_idx(
                 plan,
                 0..self.schema().len(),
             ))
+            // If there's no agg call, then `count(*)` will be the only column in the output besides keys.
+            // Since it'll be pruned immediately in `StreamProject`, the update records are likely to be
+            // no-op. So we set the hint to instruct the executor to eliminate them.
+            // See https://github.com/risingwavelabs/risingwave/issues/17030.
+            .with_noop_update_hint(self.agg_calls().is_empty())
             .into())
         }
     }
@@ -1202,12 +1169,9 @@ impl ToStream for LogicalAgg {
 #[cfg(test)]
 mod tests {
     use risingwave_common::catalog::{Field, Schema};
-    use risingwave_common::types::DataType;
 
     use super::*;
-    use crate::expr::{
-        assert_eq_input_ref, input_ref_to_column_indices, AggCall, ExprType, FunctionCall, OrderBy,
-    };
+    use crate::expr::{assert_eq_input_ref, input_ref_to_column_indices};
     use crate::optimizer::optimizer_context::OptimizerContext;
     use crate::optimizer::plan_node::LogicalValues;
 
@@ -1378,6 +1342,7 @@ mod tests {
             order_by: vec![],
             filter: Condition::true_cond(),
             direct_args: vec![],
+            user_defined: None,
         };
         Agg::new(vec![agg_call], vec![1].into(), values.into()).into()
     }
@@ -1498,6 +1463,7 @@ mod tests {
             order_by: vec![],
             filter: Condition::true_cond(),
             direct_args: vec![],
+            user_defined: None,
         };
         let agg: PlanRef = Agg::new(vec![agg_call], vec![1].into(), values.into()).into();
 
@@ -1562,6 +1528,7 @@ mod tests {
                 order_by: vec![],
                 filter: Condition::true_cond(),
                 direct_args: vec![],
+                user_defined: None,
             },
             PlanAggCall {
                 agg_kind: AggKind::Max,
@@ -1571,6 +1538,7 @@ mod tests {
                 order_by: vec![],
                 filter: Condition::true_cond(),
                 direct_args: vec![],
+                user_defined: None,
             },
         ];
         let agg: PlanRef = Agg::new(agg_calls, vec![1, 2].into(), values.into()).into();
diff --git a/src/frontend/src/optimizer/plan_node/logical_apply.rs b/src/frontend/src/optimizer/plan_node/logical_apply.rs
index b7a564ee32982..9b11927a1c4e4 100644
--- a/src/frontend/src/optimizer/plan_node/logical_apply.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_apply.rs
@@ -53,6 +53,10 @@ pub struct LogicalApply {
     /// Whether we require the subquery to produce at most one row. If `true`, we have to report an
     /// error if the subquery produces more than one row.
     max_one_row: bool,
+
+    /// An apply has been translated by `translate_apply()`, so we should not translate it in `translate_apply_rule` again.
+    /// This flag is used to avoid infinite loop in General Unnesting(Translate Apply), since we use a top-down apply order instead of bottom-up to improve the multi-scalar subqueries optimization time.
+    translated: bool,
 }
 
 impl Distill for LogicalApply {
@@ -85,6 +89,7 @@ impl LogicalApply {
         correlated_id: CorrelatedId,
         correlated_indices: Vec<usize>,
         max_one_row: bool,
+        translated: bool,
     ) -> Self {
         let ctx = left.ctx();
         let join_core = generic::Join::with_full_output(left, right, join_type, on);
@@ -105,6 +110,7 @@ impl LogicalApply {
             correlated_id,
             correlated_indices,
             max_one_row,
+            translated,
         }
     }
 
@@ -125,6 +131,7 @@ impl LogicalApply {
             correlated_id,
             correlated_indices,
             max_one_row,
+            false,
         )
         .into()
     }
@@ -164,6 +171,10 @@ impl LogicalApply {
         self.correlated_indices.to_owned()
     }
 
+    pub fn translated(&self) -> bool {
+        self.translated
+    }
+
     pub fn max_one_row(&self) -> bool {
         self.max_one_row
     }
@@ -202,7 +213,7 @@ impl LogicalApply {
         let apply_left_len = apply_left.schema().len();
         let correlated_indices_len = correlated_indices.len();
 
-        let new_apply = LogicalApply::create(
+        let new_apply = LogicalApply::new(
             domain,
             apply_right,
             JoinType::Inner,
@@ -210,7 +221,9 @@ impl LogicalApply {
             correlated_id,
             correlated_indices,
             max_one_row,
-        );
+            true,
+        )
+        .into();
 
         let on = Self::rewrite_on(on, correlated_indices_len, apply_left_len).and(Condition {
             conjunctions: eq_predicates,
@@ -285,6 +298,7 @@ impl PlanTreeNodeBinary for LogicalApply {
             self.correlated_id,
             self.correlated_indices.clone(),
             self.max_one_row,
+            self.translated,
         )
     }
 }
diff --git a/src/frontend/src/optimizer/plan_node/logical_cdc_scan.rs b/src/frontend/src/optimizer/plan_node/logical_cdc_scan.rs
index 43ce91bd0dc21..5e9d6467656f1 100644
--- a/src/frontend/src/optimizer/plan_node/logical_cdc_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_cdc_scan.rs
@@ -28,6 +28,7 @@ use crate::error::Result;
 use crate::expr::{ExprRewriter, ExprVisitor};
 use crate::optimizer::optimizer_context::OptimizerContextRef;
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::plan_node::generic::CdcScanOptions;
 use crate::optimizer::plan_node::{
     ColumnPruningContext, PredicatePushdownContext, RewriteStreamContext, StreamCdcTableScan,
     ToStreamContext,
@@ -60,14 +61,14 @@ impl LogicalCdcScan {
         table_name: String, // explain-only
         cdc_table_desc: Rc<CdcTableDesc>,
         ctx: OptimizerContextRef,
-        disable_backfill: bool,
+        options: CdcScanOptions,
     ) -> Self {
         generic::CdcScan::new(
             table_name,
             (0..cdc_table_desc.columns.len()).collect(),
             cdc_table_desc,
             ctx,
-            disable_backfill,
+            options,
         )
         .into()
     }
@@ -96,7 +97,7 @@ impl LogicalCdcScan {
             output_col_idx,
             self.core.cdc_table_desc.clone(),
             self.base.ctx().clone(),
-            self.core.disable_backfill,
+            self.core.options.clone(),
         )
         .into()
     }
diff --git a/src/frontend/src/optimizer/plan_node/logical_cte_ref.rs b/src/frontend/src/optimizer/plan_node/logical_cte_ref.rs
new file mode 100644
index 0000000000000..473ed7d06f67d
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/logical_cte_ref.rs
@@ -0,0 +1,98 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_common::bail_not_implemented;
+use risingwave_common::util::column_index_mapping::ColIndexMapping;
+
+use super::expr_visitable::ExprVisitable;
+use super::utils::impl_distill_by_unit;
+use super::{
+    generic, ColPrunable, ColumnPruningContext, ExprRewritable, Logical, LogicalProject, PlanBase,
+    PredicatePushdown, PredicatePushdownContext, RewriteStreamContext, ToBatch, ToStream,
+    ToStreamContext,
+};
+use crate::binder::ShareId;
+use crate::error::Result;
+use crate::utils::Condition;
+use crate::PlanRef;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct LogicalCteRef {
+    pub base: PlanBase<Logical>,
+    core: generic::CteRef<PlanRef>,
+}
+
+impl LogicalCteRef {
+    pub fn new(share_id: ShareId, base_plan: PlanRef) -> Self {
+        let core = generic::CteRef::new(share_id, base_plan);
+        let base = PlanBase::new_logical_with_core(&core);
+        Self { base, core }
+    }
+
+    pub fn create(share_id: ShareId, base_plan: PlanRef) -> PlanRef {
+        Self::new(share_id, base_plan).into()
+    }
+}
+
+impl_plan_tree_node_for_leaf! {LogicalCteRef}
+
+impl_distill_by_unit! {LogicalCteRef, core, "LogicalCteRef"}
+
+impl ExprRewritable for LogicalCteRef {}
+
+impl ExprVisitable for LogicalCteRef {}
+
+impl ColPrunable for LogicalCteRef {
+    fn prune_col(&self, required_cols: &[usize], _ctx: &mut ColumnPruningContext) -> PlanRef {
+        LogicalProject::with_out_col_idx(self.clone().into(), required_cols.iter().copied()).into()
+    }
+}
+
+impl PredicatePushdown for LogicalCteRef {
+    fn predicate_pushdown(
+        &self,
+        _predicate: Condition,
+        _ctx: &mut PredicatePushdownContext,
+    ) -> PlanRef {
+        self.clone().into()
+    }
+}
+
+impl ToBatch for LogicalCteRef {
+    fn to_batch(&self) -> Result<PlanRef> {
+        bail_not_implemented!(
+            issue = 15135,
+            "recursive CTE not supported for to_batch of LogicalCteRef"
+        )
+    }
+}
+
+impl ToStream for LogicalCteRef {
+    fn to_stream(&self, _ctx: &mut ToStreamContext) -> Result<PlanRef> {
+        bail_not_implemented!(
+            issue = 15135,
+            "recursive CTE not supported for to_stream of LogicalCteRef"
+        )
+    }
+
+    fn logical_rewrite_for_stream(
+        &self,
+        _ctx: &mut RewriteStreamContext,
+    ) -> Result<(PlanRef, ColIndexMapping)> {
+        bail_not_implemented!(
+            issue = 15135,
+            "recursive CTE not supported for logical_rewrite_for_stream of LogicalCteRef"
+        )
+    }
+}
diff --git a/src/frontend/src/optimizer/plan_node/logical_filter.rs b/src/frontend/src/optimizer/plan_node/logical_filter.rs
index 6bae6cc7efb22..4ea9adf7aacac 100644
--- a/src/frontend/src/optimizer/plan_node/logical_filter.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_filter.rs
@@ -247,11 +247,11 @@ mod tests {
     use std::collections::HashSet;
 
     use risingwave_common::catalog::{Field, Schema};
-    use risingwave_common::types::{DataType, ScalarImpl};
+    use risingwave_common::types::ScalarImpl;
     use risingwave_pb::expr::expr_node::Type;
 
     use super::*;
-    use crate::expr::{assert_eq_input_ref, FunctionCall, InputRef, Literal};
+    use crate::expr::{assert_eq_input_ref, Literal};
     use crate::optimizer::optimizer_context::OptimizerContext;
     use crate::optimizer::plan_node::LogicalValues;
     use crate::optimizer::property::FunctionalDependency;
diff --git a/src/frontend/src/optimizer/plan_node/logical_hop_window.rs b/src/frontend/src/optimizer/plan_node/logical_hop_window.rs
index 37f1dba5c2720..b15424d6d892a 100644
--- a/src/frontend/src/optimizer/plan_node/logical_hop_window.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_hop_window.rs
@@ -366,7 +366,6 @@ mod test {
     use risingwave_common::types::DataType;
 
     use super::*;
-    use crate::expr::InputRef;
     use crate::optimizer::optimizer_context::OptimizerContext;
     use crate::optimizer::plan_node::LogicalValues;
     use crate::optimizer::property::FunctionalDependency;
diff --git a/src/frontend/src/optimizer/plan_node/logical_join.rs b/src/frontend/src/optimizer/plan_node/logical_join.rs
index 571efee542c2b..e9dac0de38b5a 100644
--- a/src/frontend/src/optimizer/plan_node/logical_join.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_join.rs
@@ -1339,10 +1339,9 @@ impl ToStream for LogicalJoin {
         } else {
             Err(RwError::from(ErrorCode::NotSupported(
                 "streaming nested-loop join".to_string(),
-                // TODO: replace the link with user doc
                 "The non-equal join in the query requires a nested-loop join executor, which could be very expensive to run. \
                  Consider rewriting the query to use dynamic filter as a substitute if possible.\n\
-                 See also: https://github.com/risingwavelabs/rfcs/blob/main/rfcs/0033-dynamic-filter.md".to_owned(),
+                 See also: https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/".to_owned(),
             )))
         }
     }
@@ -1472,7 +1471,7 @@ mod tests {
     use risingwave_pb::expr::expr_node::Type;
 
     use super::*;
-    use crate::expr::{assert_eq_input_ref, FunctionCall, InputRef, Literal};
+    use crate::expr::{assert_eq_input_ref, FunctionCall, Literal};
     use crate::optimizer::optimizer_context::OptimizerContext;
     use crate::optimizer::plan_node::LogicalValues;
     use crate::optimizer::property::FunctionalDependency;
diff --git a/src/frontend/src/optimizer/plan_node/logical_multi_join.rs b/src/frontend/src/optimizer/plan_node/logical_multi_join.rs
index a47c75d31bd27..a5ae569456336 100644
--- a/src/frontend/src/optimizer/plan_node/logical_multi_join.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_multi_join.rs
@@ -406,7 +406,7 @@ impl LogicalMultiJoin {
     ///        a. a filter with the non eq conditions
     ///        b. a projection which reorders the output column ordering to agree with the
     ///           original ordering of the joins.
-    ///   The filter will then be pushed down by another filter pushdown pass.
+    ///    The filter will then be pushed down by another filter pushdown pass.
     pub(crate) fn heuristic_ordering(&self) -> Result<Vec<usize>> {
         let mut labeller = ConnectedComponentLabeller::new(self.inputs.len());
 
@@ -494,9 +494,9 @@ impl LogicalMultiJoin {
     /// 2. Second, for every isolated node will create connection to every other nodes.
     /// 3. Third, select and merge one node for a iteration, and use a bfs policy for which node the
     ///    selected node merged with.
-    ///   i. The select node mentioned above is the node with least number of relations and the
+    ///    i. The select node mentioned above is the node with least number of relations and the
     ///      lowerst join tree.
-    ///   ii. nodes with a join tree higher than the temporal optimal join tree will be pruned.
+    ///    ii. nodes with a join tree higher than the temporal optimal join tree will be pruned.
     pub fn as_bushy_tree_join(&self) -> Result<PlanRef> {
         let (nodes, condition) = self.get_join_graph()?;
 
@@ -871,7 +871,7 @@ mod test {
     use risingwave_pb::expr::expr_node::Type;
 
     use super::*;
-    use crate::expr::{FunctionCall, InputRef};
+    use crate::expr::InputRef;
     use crate::optimizer::optimizer_context::OptimizerContext;
     use crate::optimizer::plan_node::generic::GenericPlanRef;
     use crate::optimizer::plan_node::LogicalValues;
diff --git a/src/frontend/src/optimizer/plan_node/logical_over_window.rs b/src/frontend/src/optimizer/plan_node/logical_over_window.rs
index af1fc50c9057f..bd38569397992 100644
--- a/src/frontend/src/optimizer/plan_node/logical_over_window.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_over_window.rs
@@ -14,7 +14,7 @@
 
 use fixedbitset::FixedBitSet;
 use itertools::Itertools;
-use risingwave_common::types::{DataType, Datum, ScalarImpl};
+use risingwave_common::types::DataType;
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use risingwave_common::{bail_not_implemented, not_implemented};
 use risingwave_expr::aggregate::AggKind;
@@ -29,9 +29,11 @@ use super::{
 };
 use crate::error::{ErrorCode, Result, RwError};
 use crate::expr::{
-    Expr, ExprImpl, ExprRewriter, ExprType, ExprVisitor, FunctionCall, InputRef, WindowFunction,
+    AggCall, Expr, ExprImpl, ExprRewriter, ExprType, ExprVisitor, FunctionCall, InputRef,
+    WindowFunction,
 };
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::plan_node::logical_agg::LogicalAggBuilder;
 use crate::optimizer::plan_node::{
     ColumnPruningContext, Literal, PredicatePushdownContext, RewriteStreamContext, ToStreamContext,
 };
@@ -103,7 +105,7 @@ impl<'a> LogicalOverWindowBuilder<'a> {
             window_func.frame,
         );
 
-        if let WindowFuncKind::Aggregate(agg_kind) = kind
+        let new_expr = if let WindowFuncKind::Aggregate(agg_kind) = kind
             && matches!(
                 agg_kind,
                 AggKind::Avg
@@ -111,148 +113,39 @@ impl<'a> LogicalOverWindowBuilder<'a> {
                     | AggKind::StddevSamp
                     | AggKind::VarPop
                     | AggKind::VarSamp
-            )
-        {
-            // Refer to `LogicalAggBuilder::try_rewrite_agg_call`
-            match agg_kind {
-                AggKind::Avg => {
-                    assert_eq!(args.len(), 1);
-                    let left_ref = ExprImpl::from(self.push_window_func(WindowFunction::new(
-                        WindowFuncKind::Aggregate(AggKind::Sum),
-                        partition_by.clone(),
-                        order_by.clone(),
-                        args.clone(),
-                        frame.clone(),
-                    )?))
-                    .cast_explicit(return_type)?;
-                    let right_ref = ExprImpl::from(self.push_window_func(WindowFunction::new(
-                        WindowFuncKind::Aggregate(AggKind::Count),
-                        partition_by,
-                        order_by,
-                        args,
-                        frame,
-                    )?));
-
-                    let new_expr = ExprImpl::from(FunctionCall::new(
-                        ExprType::Divide,
-                        vec![left_ref, right_ref],
-                    )?);
-                    Ok(new_expr)
-                }
-                AggKind::StddevPop | AggKind::StddevSamp | AggKind::VarPop | AggKind::VarSamp => {
-                    let input = args.first().unwrap();
-                    let squared_input_expr = ExprImpl::from(FunctionCall::new(
-                        ExprType::Multiply,
-                        vec![input.clone(), input.clone()],
-                    )?);
-
-                    let sum_of_squares_expr =
-                        ExprImpl::from(self.push_window_func(WindowFunction::new(
-                            WindowFuncKind::Aggregate(AggKind::Sum),
-                            partition_by.clone(),
-                            order_by.clone(),
-                            vec![squared_input_expr],
-                            frame.clone(),
-                        )?))
-                        .cast_explicit(return_type.clone())?;
-
-                    let sum_expr = ExprImpl::from(self.push_window_func(WindowFunction::new(
-                        WindowFuncKind::Aggregate(AggKind::Sum),
+            ) {
+            let agg_call = AggCall::new(
+                agg_kind,
+                args,
+                false,
+                order_by,
+                Condition::true_cond(),
+                vec![],
+            )?;
+            LogicalAggBuilder::general_rewrite_agg_call(agg_call, |agg_call| {
+                Ok(self.push_window_func(
+                    // AggCall -> WindowFunction
+                    WindowFunction::new(
+                        WindowFuncKind::Aggregate(agg_call.agg_kind),
                         partition_by.clone(),
-                        order_by.clone(),
-                        args.clone(),
+                        agg_call.order_by.clone(),
+                        agg_call.args.clone(),
                         frame.clone(),
-                    )?))
-                    .cast_explicit(return_type.clone())?;
-
-                    let count_expr = ExprImpl::from(self.push_window_func(WindowFunction::new(
-                        WindowFuncKind::Aggregate(AggKind::Count),
-                        partition_by,
-                        order_by,
-                        args.clone(),
-                        frame,
-                    )?));
-
-                    let square_of_sum_expr = ExprImpl::from(FunctionCall::new(
-                        ExprType::Multiply,
-                        vec![sum_expr.clone(), sum_expr],
-                    )?);
-
-                    let numerator_expr = ExprImpl::from(FunctionCall::new(
-                        ExprType::Subtract,
-                        vec![
-                            sum_of_squares_expr,
-                            ExprImpl::from(FunctionCall::new(
-                                ExprType::Divide,
-                                vec![square_of_sum_expr, count_expr.clone()],
-                            )?),
-                        ],
-                    )?);
-
-                    let denominator_expr = match agg_kind {
-                        AggKind::StddevPop | AggKind::VarPop => count_expr.clone(),
-                        AggKind::StddevSamp | AggKind::VarSamp => {
-                            ExprImpl::from(FunctionCall::new(
-                                ExprType::Subtract,
-                                vec![
-                                    count_expr.clone(),
-                                    ExprImpl::from(Literal::new(
-                                        Datum::from(ScalarImpl::Int64(1)),
-                                        DataType::Int64,
-                                    )),
-                                ],
-                            )?)
-                        }
-                        _ => unreachable!(),
-                    };
-
-                    let mut target_expr = ExprImpl::from(FunctionCall::new(
-                        ExprType::Divide,
-                        vec![numerator_expr, denominator_expr],
-                    )?);
-
-                    if matches!(agg_kind, AggKind::StddevPop | AggKind::StddevSamp) {
-                        target_expr = ExprImpl::from(
-                            FunctionCall::new(ExprType::Sqrt, vec![target_expr]).unwrap(),
-                        );
-                    }
-
-                    match agg_kind {
-                        AggKind::VarPop | AggKind::StddevPop => Ok(target_expr),
-                        AggKind::StddevSamp | AggKind::VarSamp => {
-                            let less_than_expr = ExprImpl::from(FunctionCall::new(
-                                ExprType::LessThanOrEqual,
-                                vec![
-                                    count_expr,
-                                    ExprImpl::from(Literal::new(
-                                        Datum::from(ScalarImpl::Int64(1)),
-                                        DataType::Int64,
-                                    )),
-                                ],
-                            )?);
-                            let null_expr = ExprImpl::from(Literal::new(None, return_type));
-
-                            let case_expr = ExprImpl::from(FunctionCall::new(
-                                ExprType::Case,
-                                vec![less_than_expr, null_expr, target_expr],
-                            )?);
-                            Ok(case_expr)
-                        }
-                        _ => unreachable!(),
-                    }
-                }
-                _ => unreachable!(),
-            }
+                    )?,
+                ))
+            })?
         } else {
-            let new_expr = ExprImpl::from(self.push_window_func(WindowFunction::new(
+            ExprImpl::from(self.push_window_func(WindowFunction::new(
                 kind,
                 partition_by,
                 order_by,
                 args,
                 frame,
-            )?));
-            Ok(new_expr)
-        }
+            )?))
+        };
+
+        assert_eq!(new_expr.return_type(), return_type);
+        Ok(new_expr)
     }
 }
 
@@ -711,6 +604,16 @@ impl PredicatePushdown for LogicalOverWindow {
     }
 }
 
+macro_rules! empty_partition_by_not_implemented {
+    () => {
+        bail_not_implemented!(
+            issue = 11505,
+            "Window function with empty PARTITION BY is not supported because of potential bad performance. \
+            If you really need this, please workaround with something like `PARTITION BY 1::int`."
+        )
+    };
+}
+
 impl ToBatch for LogicalOverWindow {
     fn to_batch(&self) -> Result<PlanRef> {
         assert!(
@@ -726,7 +629,7 @@ impl ToBatch for LogicalOverWindow {
             .map(|e| e.index())
             .collect_vec();
         if partition_key_indices.is_empty() {
-            bail_not_implemented!("Window function with empty PARTITION BY is not supported yet");
+            empty_partition_by_not_implemented!();
         }
 
         let input = self.input().to_batch()?;
@@ -777,9 +680,7 @@ impl ToStream for LogicalOverWindow {
                 .map(|e| e.index())
                 .collect_vec();
             if partition_key_indices.is_empty() {
-                bail_not_implemented!(
-                    "Window function with empty PARTITION BY is not supported yet"
-                );
+                empty_partition_by_not_implemented!();
             }
 
             let sort_input =
@@ -793,6 +694,17 @@ impl ToStream for LogicalOverWindow {
         } else {
             // General (Emit-On-Update) case
 
+            if self
+                .window_functions()
+                .iter()
+                .any(|f| f.frame.bounds.is_session())
+            {
+                bail_not_implemented!(
+                    "Session frame is not yet supported in general streaming mode. \
+                    Please consider using Emit-On-Window-Close mode."
+                );
+            }
+
             // TODO(rc): Let's not introduce too many cases at once. Later we may decide to support
             // empty PARTITION BY by simply removing the following check.
             let partition_key_indices = self.window_functions()[0]
@@ -801,9 +713,7 @@ impl ToStream for LogicalOverWindow {
                 .map(|e| e.index())
                 .collect_vec();
             if partition_key_indices.is_empty() {
-                bail_not_implemented!(
-                    "Window function with empty PARTITION BY is not supported yet"
-                );
+                empty_partition_by_not_implemented!();
             }
 
             let new_input =
diff --git a/src/frontend/src/optimizer/plan_node/logical_project.rs b/src/frontend/src/optimizer/plan_node/logical_project.rs
index a0f26548da2b1..78bd74969b2c1 100644
--- a/src/frontend/src/optimizer/plan_node/logical_project.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_project.rs
@@ -303,7 +303,7 @@ mod tests {
     use risingwave_pb::expr::expr_node::Type;
 
     use super::*;
-    use crate::expr::{assert_eq_input_ref, FunctionCall, InputRef, Literal};
+    use crate::expr::{assert_eq_input_ref, FunctionCall, Literal};
     use crate::optimizer::optimizer_context::OptimizerContext;
     use crate::optimizer::plan_node::LogicalValues;
 
diff --git a/src/frontend/src/optimizer/plan_node/logical_project_set.rs b/src/frontend/src/optimizer/plan_node/logical_project_set.rs
index 53e7995c1517e..6f31b35a21ee0 100644
--- a/src/frontend/src/optimizer/plan_node/logical_project_set.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_project_set.rs
@@ -104,7 +104,7 @@ impl LogicalProjectSet {
                         args,
                         return_type,
                         function_type,
-                        udtf_catalog,
+                        user_defined,
                     } = table_func;
                     let args = args
                         .into_iter()
@@ -116,7 +116,7 @@ impl LogicalProjectSet {
                         args,
                         return_type,
                         function_type,
-                        udtf_catalog,
+                        user_defined,
                     }
                     .into()
                 } else {
@@ -412,10 +412,8 @@ mod test {
     use std::collections::HashSet;
 
     use risingwave_common::catalog::{Field, Schema};
-    use risingwave_common::types::DataType;
 
     use super::*;
-    use crate::expr::{ExprImpl, InputRef, TableFunction};
     use crate::optimizer::optimizer_context::OptimizerContext;
     use crate::optimizer::plan_node::LogicalValues;
     use crate::optimizer::property::FunctionalDependency;
diff --git a/src/frontend/src/optimizer/plan_node/logical_recursive_union.rs b/src/frontend/src/optimizer/plan_node/logical_recursive_union.rs
new file mode 100644
index 0000000000000..04a3dc297d352
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/logical_recursive_union.rs
@@ -0,0 +1,141 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use itertools::Itertools;
+use pretty_xmlish::{Pretty, XmlNode};
+use risingwave_common::bail_not_implemented;
+use risingwave_common::util::column_index_mapping::ColIndexMapping;
+use smallvec::{smallvec, SmallVec};
+
+use super::expr_visitable::ExprVisitable;
+use super::generic::GenericPlanRef;
+use super::utils::{childless_record, Distill};
+use super::{
+    generic, ColPrunable, ColumnPruningContext, ExprRewritable, Logical, PlanBase, PlanTreeNode,
+    PredicatePushdown, PredicatePushdownContext, RewriteStreamContext, ToBatch, ToStream,
+    ToStreamContext,
+};
+use crate::binder::ShareId;
+use crate::error::Result;
+use crate::utils::Condition;
+use crate::PlanRef;
+
+/// `LogicalRecursiveUnion` returns the union of the rows of its inputs.
+/// note: if `all` is false, it needs to eliminate duplicates.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct LogicalRecursiveUnion {
+    pub base: PlanBase<Logical>,
+    core: generic::RecursiveUnion<PlanRef>,
+}
+
+impl LogicalRecursiveUnion {
+    pub fn new(base_plan: PlanRef, recursive: PlanRef, id: ShareId) -> Self {
+        let core = generic::RecursiveUnion {
+            base: base_plan,
+            recursive,
+            id,
+        };
+        let base = PlanBase::new_logical_with_core(&core);
+        LogicalRecursiveUnion { base, core }
+    }
+
+    pub fn create(base_plan: PlanRef, recursive: PlanRef, id: ShareId) -> PlanRef {
+        Self::new(base_plan, recursive, id).into()
+    }
+
+    pub(super) fn pretty_fields(base: impl GenericPlanRef, name: &str) -> XmlNode<'_> {
+        childless_record(name, vec![("id", Pretty::debug(&base.id().0))])
+    }
+}
+
+impl PlanTreeNode for LogicalRecursiveUnion {
+    fn inputs(&self) -> SmallVec<[PlanRef; 2]> {
+        smallvec![self.core.base.clone(), self.core.recursive.clone()]
+    }
+
+    fn clone_with_inputs(&self, inputs: &[PlanRef]) -> PlanRef {
+        let mut inputs = inputs.iter().cloned();
+        Self::create(inputs.next().unwrap(), inputs.next().unwrap(), self.core.id)
+    }
+}
+
+impl Distill for LogicalRecursiveUnion {
+    fn distill<'a>(&self) -> XmlNode<'a> {
+        Self::pretty_fields(&self.base, "LogicalRecursiveUnion")
+    }
+}
+
+impl ColPrunable for LogicalRecursiveUnion {
+    fn prune_col(&self, required_cols: &[usize], ctx: &mut ColumnPruningContext) -> PlanRef {
+        let new_inputs = self
+            .inputs()
+            .iter()
+            .map(|input| input.prune_col(required_cols, ctx))
+            .collect_vec();
+        let new_plan = self.clone_with_inputs(&new_inputs);
+        self.ctx()
+            .insert_rcte_cache_plan(self.core.id, new_plan.clone());
+        new_plan
+    }
+}
+
+impl ExprRewritable for LogicalRecursiveUnion {}
+
+impl ExprVisitable for LogicalRecursiveUnion {}
+
+impl PredicatePushdown for LogicalRecursiveUnion {
+    fn predicate_pushdown(
+        &self,
+        predicate: Condition,
+        ctx: &mut PredicatePushdownContext,
+    ) -> PlanRef {
+        let new_inputs = self
+            .inputs()
+            .iter()
+            .map(|input| input.predicate_pushdown(predicate.clone(), ctx))
+            .collect_vec();
+        let new_plan = self.clone_with_inputs(&new_inputs);
+        self.ctx()
+            .insert_rcte_cache_plan(self.core.id, new_plan.clone());
+        new_plan
+    }
+}
+
+impl ToBatch for LogicalRecursiveUnion {
+    fn to_batch(&self) -> Result<PlanRef> {
+        bail_not_implemented!(
+            issue = 15135,
+            "recursive CTE not supported for to_batch of LogicalRecursiveUnion"
+        )
+    }
+}
+
+impl ToStream for LogicalRecursiveUnion {
+    fn to_stream(&self, _ctx: &mut ToStreamContext) -> Result<PlanRef> {
+        bail_not_implemented!(
+            issue = 15135,
+            "recursive CTE not supported for to_stream of LogicalRecursiveUnion"
+        )
+    }
+
+    fn logical_rewrite_for_stream(
+        &self,
+        _ctx: &mut RewriteStreamContext,
+    ) -> Result<(PlanRef, ColIndexMapping)> {
+        bail_not_implemented!(
+            issue = 15135,
+            "recursive CTE not supported for logical_rewrite_for_stream of LogicalRecursiveUnion"
+        )
+    }
+}
diff --git a/src/frontend/src/optimizer/plan_node/logical_scan.rs b/src/frontend/src/optimizer/plan_node/logical_scan.rs
index 77f26d0adc45e..e2aeaa6b9517a 100644
--- a/src/frontend/src/optimizer/plan_node/logical_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_scan.rs
@@ -164,16 +164,14 @@ impl LogicalScan {
                         .pk()
                         .iter()
                         .map(|idx_item| {
-                            ColumnOrder::new(
-                                *output_col_map
-                                    .get(
-                                        s2p_mapping
-                                            .get(&idx_item.column_index)
-                                            .expect("should be in s2p mapping"),
-                                    )
-                                    .unwrap_or(&unmatched_idx),
-                                idx_item.order_type,
-                            )
+                            let idx = match s2p_mapping.get(&idx_item.column_index) {
+                                Some(col_idx) => {
+                                    *output_col_map.get(col_idx).unwrap_or(&unmatched_idx)
+                                }
+                                // After we support index on expressions, we need to handle the case where the column is not in the `s2p_mapping`.
+                                None => unmatched_idx,
+                            };
+                            ColumnOrder::new(idx, idx_item.order_type)
                         })
                         .collect(),
                 }
diff --git a/src/frontend/src/optimizer/plan_node/logical_source.rs b/src/frontend/src/optimizer/plan_node/logical_source.rs
index 50a28494e81df..0310fdbbd439b 100644
--- a/src/frontend/src/optimizer/plan_node/logical_source.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_source.rs
@@ -347,8 +347,9 @@ impl ToStream for LogicalSource {
             }
             SourceNodeKind::CreateMViewOrBatch => {
                 // Create MV on source.
-                let use_shared_source = self.source_catalog().is_some_and(|c| c.info.is_shared())
-                    && self.ctx().session_ctx().config().rw_enable_shared_source();
+                // We only check rw_enable_shared_source is true when `CREATE SOURCE`.
+                // The value does not affect the behavior of `CREATE MATERIALIZED VIEW` here.
+                let use_shared_source = self.source_catalog().is_some_and(|c| c.info.is_shared());
                 if use_shared_source {
                     plan = StreamSourceScan::new(self.core.clone()).into();
                 } else {
diff --git a/src/frontend/src/optimizer/plan_node/logical_union.rs b/src/frontend/src/optimizer/plan_node/logical_union.rs
index 99de2748dca4e..6238e63fd9f51 100644
--- a/src/frontend/src/optimizer/plan_node/logical_union.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_union.rs
@@ -327,8 +327,7 @@ impl ToStream for LogicalUnion {
 #[cfg(test)]
 mod tests {
 
-    use risingwave_common::catalog::{Field, Schema};
-    use risingwave_common::types::DataType;
+    use risingwave_common::catalog::Field;
 
     use super::*;
     use crate::optimizer::optimizer_context::OptimizerContext;
diff --git a/src/frontend/src/optimizer/plan_node/logical_values.rs b/src/frontend/src/optimizer/plan_node/logical_values.rs
index cdf9af3c93541..7a585f7e0801f 100644
--- a/src/frontend/src/optimizer/plan_node/logical_values.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_values.rs
@@ -205,11 +205,9 @@ impl ToStream for LogicalValues {
 #[cfg(test)]
 mod tests {
 
-    use risingwave_common::catalog::Field;
-    use risingwave_common::types::{DataType, Datum};
+    use risingwave_common::types::Datum;
 
     use super::*;
-    use crate::expr::Literal;
     use crate::optimizer::optimizer_context::OptimizerContext;
 
     fn literal(val: i32) -> ExprImpl {
diff --git a/src/frontend/src/optimizer/plan_node/mod.rs b/src/frontend/src/optimizer/plan_node/mod.rs
index e7ad78f373bac..d7187357e3fad 100644
--- a/src/frontend/src/optimizer/plan_node/mod.rs
+++ b/src/frontend/src/optimizer/plan_node/mod.rs
@@ -33,7 +33,7 @@ use std::ops::Deref;
 use std::rc::Rc;
 
 use downcast_rs::{impl_downcast, Downcast};
-use dyn_clone::{self, DynClone};
+use dyn_clone::DynClone;
 use fixedbitset::FixedBitSet;
 use itertools::Itertools;
 use paste::paste;
@@ -827,6 +827,7 @@ mod batch_hash_join;
 mod batch_hop_window;
 mod batch_insert;
 mod batch_limit;
+mod batch_log_seq_scan;
 mod batch_lookup_join;
 mod batch_max_one_row;
 mod batch_nested_loop_join;
@@ -847,6 +848,7 @@ mod batch_values;
 mod logical_agg;
 mod logical_apply;
 mod logical_cdc_scan;
+mod logical_cte_ref;
 mod logical_dedup;
 mod logical_delete;
 mod logical_except;
@@ -864,6 +866,7 @@ mod logical_now;
 mod logical_over_window;
 mod logical_project;
 mod logical_project_set;
+mod logical_recursive_union;
 mod logical_scan;
 mod logical_share;
 mod logical_source;
@@ -898,7 +901,6 @@ mod stream_sort;
 mod stream_source;
 mod stream_source_scan;
 mod stream_stateless_simple_agg;
-mod stream_subscription;
 mod stream_table_scan;
 mod stream_topn;
 mod stream_values;
@@ -926,6 +928,7 @@ pub use batch_iceberg_scan::BatchIcebergScan;
 pub use batch_insert::BatchInsert;
 pub use batch_kafka_scan::BatchKafkaScan;
 pub use batch_limit::BatchLimit;
+pub use batch_log_seq_scan::BatchLogSeqScan;
 pub use batch_lookup_join::BatchLookupJoin;
 pub use batch_max_one_row::BatchMaxOneRow;
 pub use batch_nested_loop_join::BatchNestedLoopJoin;
@@ -946,6 +949,7 @@ pub use batch_values::BatchValues;
 pub use logical_agg::LogicalAgg;
 pub use logical_apply::LogicalApply;
 pub use logical_cdc_scan::LogicalCdcScan;
+pub use logical_cte_ref::LogicalCteRef;
 pub use logical_dedup::LogicalDedup;
 pub use logical_delete::LogicalDelete;
 pub use logical_except::LogicalExcept;
@@ -964,6 +968,7 @@ pub use logical_now::LogicalNow;
 pub use logical_over_window::LogicalOverWindow;
 pub use logical_project::LogicalProject;
 pub use logical_project_set::LogicalProjectSet;
+pub use logical_recursive_union::LogicalRecursiveUnion;
 pub use logical_scan::LogicalScan;
 pub use logical_share::LogicalShare;
 pub use logical_source::LogicalSource;
@@ -1000,7 +1005,6 @@ pub use stream_sort::StreamEowcSort;
 pub use stream_source::StreamSource;
 pub use stream_source_scan::StreamSourceScan;
 pub use stream_stateless_simple_agg::StreamStatelessSimpleAgg;
-pub use stream_subscription::StreamSubscription;
 pub use stream_table_scan::StreamTableScan;
 pub use stream_temporal_join::StreamTemporalJoin;
 pub use stream_topn::StreamTopN;
@@ -1063,6 +1067,8 @@ macro_rules! for_all_plan_nodes {
             , { Logical, MaxOneRow }
             , { Logical, KafkaScan }
             , { Logical, IcebergScan }
+            , { Logical, RecursiveUnion }
+            , { Logical, CteRef }
             , { Batch, SimpleAgg }
             , { Batch, HashAgg }
             , { Batch, SortAgg }
@@ -1073,6 +1079,7 @@ macro_rules! for_all_plan_nodes {
             , { Batch, Update }
             , { Batch, SeqScan }
             , { Batch, SysSeqScan }
+            , { Batch, LogSeqScan }
             , { Batch, HashJoin }
             , { Batch, NestedLoopJoin }
             , { Batch, Values }
@@ -1097,7 +1104,6 @@ macro_rules! for_all_plan_nodes {
             , { Stream, TableScan }
             , { Stream, CdcTableScan }
             , { Stream, Sink }
-            , { Stream, Subscription }
             , { Stream, Source }
             , { Stream, SourceScan }
             , { Stream, HashJoin }
@@ -1165,6 +1171,8 @@ macro_rules! for_logical_plan_nodes {
             , { Logical, MaxOneRow }
             , { Logical, KafkaScan }
             , { Logical, IcebergScan }
+            , { Logical, RecursiveUnion }
+            , { Logical, CteRef }
         }
     };
 }
@@ -1181,6 +1189,7 @@ macro_rules! for_batch_plan_nodes {
             , { Batch, Filter }
             , { Batch, SeqScan }
             , { Batch, SysSeqScan }
+            , { Batch, LogSeqScan }
             , { Batch, HashJoin }
             , { Batch, NestedLoopJoin }
             , { Batch, Values }
@@ -1219,7 +1228,6 @@ macro_rules! for_stream_plan_nodes {
             , { Stream, TableScan }
             , { Stream, CdcTableScan }
             , { Stream, Sink }
-            , { Stream, Subscription }
             , { Stream, Source }
             , { Stream, SourceScan }
             , { Stream, HashAgg }
diff --git a/src/frontend/src/optimizer/plan_node/plan_base.rs b/src/frontend/src/optimizer/plan_node/plan_base.rs
index 3e9783894d40f..12fba475241c5 100644
--- a/src/frontend/src/optimizer/plan_node/plan_base.rs
+++ b/src/frontend/src/optimizer/plan_node/plan_base.rs
@@ -13,13 +13,10 @@
 // limitations under the License.
 
 use educe::Educe;
-use fixedbitset::FixedBitSet;
-use risingwave_common::catalog::Schema;
 
 use super::generic::GenericPlanNode;
 use super::*;
-use crate::optimizer::optimizer_context::OptimizerContextRef;
-use crate::optimizer::property::{Distribution, FunctionalDependencySet, Order};
+use crate::optimizer::property::Distribution;
 
 /// No extra fields for logical plan nodes.
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
diff --git a/src/frontend/src/optimizer/plan_node/predicate_pushdown.rs b/src/frontend/src/optimizer/plan_node/predicate_pushdown.rs
index 64abc6bcd2de8..b7c429f4f6399 100644
--- a/src/frontend/src/optimizer/plan_node/predicate_pushdown.rs
+++ b/src/frontend/src/optimizer/plan_node/predicate_pushdown.rs
@@ -19,7 +19,6 @@ use paste::paste;
 use super::*;
 use crate::optimizer::plan_visitor::ShareParentCounter;
 use crate::optimizer::PlanVisitor;
-use crate::utils::Condition;
 use crate::{for_batch_plan_nodes, for_stream_plan_nodes};
 
 /// The trait for predicate pushdown, only logical plan node will use it, though all plan node impl
@@ -30,14 +29,14 @@ pub trait PredicatePushdown {
     /// There are three kinds of predicates:
     ///
     /// 1. those can't be pushed down. We just create a `LogicalFilter` for them above the current
-    /// `PlanNode`. i.e.,
+    ///    `PlanNode`. i.e.,
     ///
     ///     ```ignore
     ///     LogicalFilter::create(self.clone().into(), predicate)
     ///     ```
     ///
     /// 2. those can be merged with current `PlanNode` (e.g., `LogicalJoin`). We just merge
-    /// the predicates with the `Condition` of it.
+    ///    the predicates with the `Condition` of it.
     ///
     /// 3. those can be pushed down. We pass them to current `PlanNode`'s input.
     fn predicate_pushdown(
diff --git a/src/frontend/src/optimizer/plan_node/stream_cdc_table_scan.rs b/src/frontend/src/optimizer/plan_node/stream_cdc_table_scan.rs
index 9df1a94879b22..9fe3347171453 100644
--- a/src/frontend/src/optimizer/plan_node/stream_cdc_table_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_cdc_table_scan.rs
@@ -247,6 +247,7 @@ impl StreamCdcTableScan {
             "stream cdc table scan output indices"
         );
 
+        let options = self.core.options.to_proto();
         let stream_scan_body = PbNodeBody::StreamCdcScan(StreamCdcScanNode {
             table_id: upstream_source_id,
             upstream_column_ids,
@@ -255,7 +256,8 @@ impl StreamCdcTableScan {
             state_table: Some(catalog),
             cdc_table_desc: Some(self.core.cdc_table_desc.to_protobuf()),
             rate_limit: self.base.ctx().overwrite_options().streaming_rate_limit,
-            disable_backfill: self.core.disable_backfill,
+            disable_backfill: options.disable_backfill,
+            options: Some(options),
         });
 
         // plan: merge -> filter -> exchange(simple) -> stream_scan
diff --git a/src/frontend/src/optimizer/plan_node/stream_delta_join.rs b/src/frontend/src/optimizer/plan_node/stream_delta_join.rs
index 89b1402cba919..49257676bc004 100644
--- a/src/frontend/src/optimizer/plan_node/stream_delta_join.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_delta_join.rs
@@ -20,10 +20,9 @@ use risingwave_pb::plan_common::JoinType;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
 use risingwave_pb::stream_plan::{ArrangementInfo, DeltaIndexJoinNode};
 
-use super::generic::{self, GenericPlanRef};
 use super::stream::prelude::*;
 use super::utils::{childless_record, Distill};
-use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeBinary};
+use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeBinary};
 use crate::expr::{Expr, ExprRewriter, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
 use crate::optimizer::plan_node::utils::IndicesDisplay;
diff --git a/src/frontend/src/optimizer/plan_node/stream_dml.rs b/src/frontend/src/optimizer/plan_node/stream_dml.rs
index bdda75d061012..e777041718291 100644
--- a/src/frontend/src/optimizer/plan_node/stream_dml.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_dml.rs
@@ -18,7 +18,6 @@ use risingwave_common::catalog::{ColumnDesc, INITIAL_TABLE_VERSION_ID};
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 
 use super::stream::prelude::*;
-use super::stream::StreamPlanRef;
 use super::utils::{childless_record, Distill};
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
diff --git a/src/frontend/src/optimizer/plan_node/stream_dynamic_filter.rs b/src/frontend/src/optimizer/plan_node/stream_dynamic_filter.rs
index 974c6fca5df1a..a90cd4b77c669 100644
--- a/src/frontend/src/optimizer/plan_node/stream_dynamic_filter.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_dynamic_filter.rs
@@ -17,9 +17,8 @@ pub use risingwave_pb::expr::expr_node::Type as ExprType;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
 use risingwave_pb::stream_plan::DynamicFilterNode;
 
-use super::generic::{DynamicFilter, GenericPlanRef};
+use super::generic::DynamicFilter;
 use super::stream::prelude::*;
-use super::stream::StreamPlanRef;
 use super::utils::{
     childless_record, column_names_pretty, plan_node_name, watermark_pretty, Distill,
 };
diff --git a/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs b/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs
index 7cd9bb533a807..78cb0e3b9d605 100644
--- a/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_eowc_over_window.rs
@@ -18,7 +18,7 @@ use fixedbitset::FixedBitSet;
 use risingwave_common::util::sort_util::OrderType;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 
-use super::generic::{self, GenericPlanRef, PlanWindowFunction};
+use super::generic::{self, PlanWindowFunction};
 use super::stream::prelude::*;
 use super::utils::{impl_distill_by_unit, TableCatalogBuilder};
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
diff --git a/src/frontend/src/optimizer/plan_node/stream_fs_fetch.rs b/src/frontend/src/optimizer/plan_node/stream_fs_fetch.rs
index 82f2bd208b548..931fbe045b78d 100644
--- a/src/frontend/src/optimizer/plan_node/stream_fs_fetch.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_fs_fetch.rs
@@ -101,8 +101,6 @@ impl StreamNode for StreamFsFetch {
             source_id: source_catalog.id,
             source_name: source_catalog.name.clone(),
             state_table: Some(
-                // `StreamFsSource` will do range scan according to assigned vnodes, so we need to set
-                // the key for distributing data to different vnodes.
                 generic::Source::infer_internal_table_catalog(true)
                     .with_id(state.gen_table_id_wrapped())
                     .to_internal_table_prost(),
diff --git a/src/frontend/src/optimizer/plan_node/stream_group_topn.rs b/src/frontend/src/optimizer/plan_node/stream_group_topn.rs
index e48d23d14e04e..8500e24b0fd9f 100644
--- a/src/frontend/src/optimizer/plan_node/stream_group_topn.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_group_topn.rs
@@ -16,9 +16,8 @@ use fixedbitset::FixedBitSet;
 use pretty_xmlish::XmlNode;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 
-use super::generic::{DistillUnit, GenericPlanRef, TopNLimit};
+use super::generic::{DistillUnit, TopNLimit};
 use super::stream::prelude::*;
-use super::stream::StreamPlanRef;
 use super::utils::{plan_node_name, watermark_pretty, Distill};
 use super::{generic, ExprRewritable, PlanBase, PlanTreeNodeUnary, StreamNode};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
diff --git a/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs b/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs
index d5d7bde249a17..eb69d5c259bb6 100644
--- a/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_hash_agg.rs
@@ -17,14 +17,13 @@ use itertools::Itertools;
 use pretty_xmlish::XmlNode;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 
-use super::generic::{self, GenericPlanRef, PlanAggCall};
+use super::generic::{self, PlanAggCall};
 use super::stream::prelude::*;
 use super::utils::{childless_record, plan_node_name, watermark_pretty, Distill};
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::error::{ErrorCode, Result};
 use crate::expr::{ExprRewriter, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::plan_node::stream::StreamPlanRef;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::utils::{ColIndexMapping, ColIndexMappingRewriteExt, IndexSet};
 
diff --git a/src/frontend/src/optimizer/plan_node/stream_hash_join.rs b/src/frontend/src/optimizer/plan_node/stream_hash_join.rs
index d5e5e9838ac66..b803fccef7b07 100644
--- a/src/frontend/src/optimizer/plan_node/stream_hash_join.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_hash_join.rs
@@ -20,9 +20,8 @@ use risingwave_pb::plan_common::JoinType;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
 use risingwave_pb::stream_plan::{DeltaExpression, HashJoinNode, PbInequalityPair};
 
-use super::generic::{GenericPlanRef, Join};
+use super::generic::Join;
 use super::stream::prelude::*;
-use super::stream::StreamPlanRef;
 use super::utils::{childless_record, plan_node_name, watermark_pretty, Distill};
 use super::{
     generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeBinary, StreamDeltaJoin, StreamNode,
diff --git a/src/frontend/src/optimizer/plan_node/stream_hop_window.rs b/src/frontend/src/optimizer/plan_node/stream_hop_window.rs
index 9cb5800cc7220..0cdddf77ed0e5 100644
--- a/src/frontend/src/optimizer/plan_node/stream_hop_window.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_hop_window.rs
@@ -17,9 +17,7 @@ use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 use risingwave_pb::stream_plan::HopWindowNode;
 
-use super::generic::GenericPlanRef;
 use super::stream::prelude::*;
-use super::stream::StreamPlanRef;
 use super::utils::{childless_record, watermark_pretty, Distill};
 use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::expr::{Expr, ExprImpl, ExprRewriter, ExprVisitor};
diff --git a/src/frontend/src/optimizer/plan_node/stream_materialize.rs b/src/frontend/src/optimizer/plan_node/stream_materialize.rs
index 3da52ef4f55a2..e5a2496916adc 100644
--- a/src/frontend/src/optimizer/plan_node/stream_materialize.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_materialize.rs
@@ -27,14 +27,13 @@ use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 
 use super::derive::derive_columns;
 use super::stream::prelude::*;
-use super::stream::StreamPlanRef;
 use super::utils::{childless_record, Distill};
 use super::{reorganize_elements_id, ExprRewritable, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::catalog::table_catalog::{TableCatalog, TableType, TableVersion};
 use crate::error::Result;
 use crate::optimizer::plan_node::derive::derive_pk;
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::plan_node::generic::GenericPlanRef;
+use crate::optimizer::plan_node::utils::plan_has_backfill_leaf_nodes;
 use crate::optimizer::plan_node::{PlanBase, PlanNodeMeta};
 use crate::optimizer::property::{Cardinality, Distribution, Order, RequiredDist};
 use crate::stream_fragmenter::BuildFragmentGraphState;
@@ -86,6 +85,14 @@ impl StreamMaterialize {
         let input = reorganize_elements_id(input);
         let columns = derive_columns(input.schema(), out_names, &user_cols)?;
 
+        let create_type = if matches!(table_type, TableType::MaterializedView)
+            && input.ctx().session_ctx().config().background_ddl()
+            && plan_has_backfill_leaf_nodes(&input)
+        {
+            CreateType::Background
+        } else {
+            CreateType::Foreground
+        };
         let table = Self::derive_table_catalog(
             input.clone(),
             name,
@@ -100,6 +107,7 @@ impl StreamMaterialize {
             None,
             cardinality,
             retention_seconds,
+            create_type,
         )?;
 
         Ok(Self::new(input, table))
@@ -141,6 +149,7 @@ impl StreamMaterialize {
             version,
             Cardinality::unknown(), // unknown cardinality for tables
             retention_seconds,
+            CreateType::Foreground,
         )?;
 
         Ok(Self::new(input, table))
@@ -212,6 +221,7 @@ impl StreamMaterialize {
         version: Option<TableVersion>,
         cardinality: Cardinality,
         retention_seconds: Option<NonZeroU32>,
+        create_type: CreateType,
     ) -> Result<TableCatalog> {
         let input = rewritten_input;
 
@@ -261,7 +271,7 @@ impl StreamMaterialize {
             created_at_epoch: None,
             initialized_at_epoch: None,
             cleaned_by_watermark: false,
-            create_type: CreateType::Foreground, // Will be updated in the handler itself.
+            create_type,
             stream_job_status: StreamJobStatus::Creating,
             description: None,
             incoming_sinks: vec![],
diff --git a/src/frontend/src/optimizer/plan_node/stream_over_window.rs b/src/frontend/src/optimizer/plan_node/stream_over_window.rs
index 7173e3a46e0cd..be6c63bcb50de 100644
--- a/src/frontend/src/optimizer/plan_node/stream_over_window.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_over_window.rs
@@ -23,7 +23,6 @@ use super::stream::prelude::*;
 use super::utils::{impl_distill_by_unit, TableCatalogBuilder};
 use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::plan_node::generic::GenericPlanRef;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 use crate::TableCatalog;
 
diff --git a/src/frontend/src/optimizer/plan_node/stream_project.rs b/src/frontend/src/optimizer/plan_node/stream_project.rs
index 1b97835e9896f..e8ff1df6e82db 100644
--- a/src/frontend/src/optimizer/plan_node/stream_project.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_project.rs
@@ -17,9 +17,7 @@ use pretty_xmlish::XmlNode;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 use risingwave_pb::stream_plan::ProjectNode;
 
-use super::generic::GenericPlanRef;
 use super::stream::prelude::*;
-use super::stream::StreamPlanRef;
 use super::utils::{childless_record, watermark_pretty, Distill};
 use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::expr::{
@@ -40,10 +38,15 @@ pub struct StreamProject {
     watermark_derivations: Vec<(usize, usize)>,
     /// Nondecreasing expression indices. `Project` can produce watermarks for these expressions.
     nondecreasing_exprs: Vec<usize>,
+    /// Whether there are likely no-op updates in the output chunks, so that eliminating them with
+    /// `StreamChunk::eliminate_adjacent_noop_update` could be beneficial.
+    noop_update_hint: bool,
 }
 
 impl Distill for StreamProject {
     fn distill<'a>(&self) -> XmlNode<'a> {
+        let verbose = self.base.ctx().is_explain_verbose();
+
         let schema = self.schema();
         let mut vec = self.core.fields_pretty(schema);
         if let Some(display_output_watermarks) =
@@ -51,12 +54,27 @@ impl Distill for StreamProject {
         {
             vec.push(("output_watermarks", display_output_watermarks));
         }
+        if verbose && self.noop_update_hint {
+            vec.push(("noop_update_hint", "true".into()));
+        }
         childless_record("StreamProject", vec)
     }
 }
 
 impl StreamProject {
     pub fn new(core: generic::Project<PlanRef>) -> Self {
+        Self::new_inner(core, false)
+    }
+
+    /// Set the `noop_update_hint` flag to the given value.
+    pub fn with_noop_update_hint(self, noop_update_hint: bool) -> Self {
+        Self {
+            noop_update_hint,
+            ..self
+        }
+    }
+
+    fn new_inner(core: generic::Project<PlanRef>, noop_update_hint: bool) -> Self {
         let input = core.input.clone();
         let distribution = core
             .i2o_col_mapping()
@@ -92,11 +110,13 @@ impl StreamProject {
             input.emit_on_window_close(),
             watermark_columns,
         );
+
         StreamProject {
             base,
             core,
             watermark_derivations,
             nondecreasing_exprs,
+            noop_update_hint,
         }
     }
 
@@ -107,6 +127,10 @@ impl StreamProject {
     pub fn exprs(&self) -> &Vec<ExprImpl> {
         &self.core.exprs
     }
+
+    pub fn noop_update_hint(&self) -> bool {
+        self.noop_update_hint
+    }
 }
 
 impl PlanTreeNodeUnary for StreamProject {
@@ -117,7 +141,7 @@ impl PlanTreeNodeUnary for StreamProject {
     fn clone_with_input(&self, input: PlanRef) -> Self {
         let mut core = self.core.clone();
         core.input = input;
-        Self::new(core)
+        Self::new_inner(core, self.noop_update_hint)
     }
 }
 impl_plan_tree_node_for_unary! {StreamProject}
@@ -134,6 +158,7 @@ impl StreamNode for StreamProject {
             watermark_input_cols,
             watermark_output_cols,
             nondecreasing_exprs: self.nondecreasing_exprs.iter().map(|i| *i as _).collect(),
+            noop_update_hint: self.noop_update_hint,
         })
     }
 }
@@ -146,7 +171,7 @@ impl ExprRewritable for StreamProject {
     fn rewrite_exprs(&self, r: &mut dyn ExprRewriter) -> PlanRef {
         let mut core = self.core.clone();
         core.rewrite_exprs(r);
-        Self::new(core).into()
+        Self::new_inner(core, self.noop_update_hint).into()
     }
 }
 
diff --git a/src/frontend/src/optimizer/plan_node/stream_share.rs b/src/frontend/src/optimizer/plan_node/stream_share.rs
index 395a32bc9be8a..5bf575f622bce 100644
--- a/src/frontend/src/optimizer/plan_node/stream_share.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_share.rs
@@ -16,7 +16,6 @@ use pretty_xmlish::XmlNode;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 use risingwave_pb::stream_plan::PbStreamNode;
 
-use super::generic::GenericPlanRef;
 use super::stream::prelude::*;
 use super::utils::Distill;
 use super::{generic, ExprRewritable, PlanRef, PlanTreeNodeUnary, StreamExchange, StreamNode};
diff --git a/src/frontend/src/optimizer/plan_node/stream_sink.rs b/src/frontend/src/optimizer/plan_node/stream_sink.rs
index e0bd1eb225e07..49378ab0a53ed 100644
--- a/src/frontend/src/optimizer/plan_node/stream_sink.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_sink.rs
@@ -37,15 +37,15 @@ use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 use risingwave_pb::stream_plan::SinkLogStoreType;
 
 use super::derive::{derive_columns, derive_pk};
-use super::generic::{self, GenericPlanRef};
 use super::stream::prelude::*;
 use super::utils::{
     childless_record, infer_kv_log_store_table_catalog_inner, Distill, IndicesDisplay,
 };
-use super::{ExprRewritable, PlanBase, PlanRef, StreamNode, StreamProject};
+use super::{generic, ExprRewritable, PlanBase, PlanRef, StreamNode, StreamProject};
 use crate::error::{ErrorCode, Result};
 use crate::expr::{ExprImpl, FunctionCall, InputRef};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::plan_node::utils::plan_has_backfill_leaf_nodes;
 use crate::optimizer::plan_node::PlanTreeNodeUnary;
 use crate::optimizer::property::{Distribution, Order, RequiredDist};
 use crate::stream_fragmenter::BuildFragmentGraphState;
@@ -148,18 +148,19 @@ impl IcebergPartitionInfo {
         ))
     }
 }
+
 #[inline]
 fn find_column_idx_by_name(columns: &[ColumnCatalog], col_name: &str) -> Result<usize> {
     columns
-            .iter()
-            .position(|col| col.column_desc.name == col_name)
-            .ok_or_else(|| {
-                ErrorCode::SinkError(Box::new(Error::new(
-                    ErrorKind::InvalidInput,
-                    format!("Sink primary key column not found: {}. Please use ',' as the delimiter for different primary key columns.", col_name)
-                )))
+        .iter()
+        .position(|col| col.column_desc.name == col_name)
+        .ok_or_else(|| {
+            ErrorCode::SinkError(Box::new(Error::new(
+                ErrorKind::InvalidInput,
+                format!("Sink primary key column not found: {}. Please use ',' as the delimiter for different primary key columns.", col_name),
+            )))
                 .into()
-            })
+        })
 }
 
 /// [`StreamSink`] represents a table/connector sink at the very end of the graph.
@@ -179,6 +180,7 @@ impl StreamSink {
             .into_stream()
             .expect("input should be stream plan")
             .clone_with_new_plan_id();
+
         Self {
             base,
             input,
@@ -315,6 +317,7 @@ impl StreamSink {
         let (pk, _) = derive_pk(input.clone(), user_order_by, &columns);
         let mut downstream_pk =
             Self::parse_downstream_pk(&columns, properties.get(DOWNSTREAM_PK_KEY))?;
+
         let mut extra_partition_col_idx = None;
         let required_dist = match input.distribution() {
             Distribution::Single => RequiredDist::single(),
@@ -371,7 +374,9 @@ impl StreamSink {
         };
         let input = required_dist.enforce_if_not_satisfies(input, &Order::any())?;
         let distribution_key = input.distribution().dist_column_indices().to_vec();
-        let create_type = if input.ctx().session_ctx().config().background_ddl() {
+        let create_type = if input.ctx().session_ctx().config().background_ddl()
+            && plan_has_backfill_leaf_nodes(&input)
+        {
             CreateType::Background
         } else {
             CreateType::Foreground
@@ -444,7 +449,7 @@ impl StreamSink {
         let (user_defined_append_only, user_force_append_only, syntax_legacy) = match format_desc {
             Some(f) => (
                 f.format == SinkFormat::AppendOnly,
-                Self::is_user_force_append_only(&WithOptions::from_inner(f.options.clone()))?,
+                Self::is_user_force_append_only(&WithOptions::new(f.options.clone()))?,
                 false,
             ),
             None => (
@@ -465,16 +470,20 @@ impl StreamSink {
             (false, true, false) => {
                 Err(ErrorCode::SinkError(Box::new(Error::new(
                     ErrorKind::InvalidInput,
-                        format!("The sink cannot be append-only. Please add \"force_append_only='true'\" in {} options to force the sink to be append-only. Notice that this will cause the sink executor to drop any UPDATE or DELETE message.", if syntax_legacy {"WITH"} else {"FORMAT ENCODE"}),
+                    format!(
+                        "The sink cannot be append-only. Please add \"force_append_only='true'\" in {} options to force the sink to be append-only. \
+                        Notice that this will cause the sink executor to drop DELETE messages and convert UPDATE messages to INSERT.",
+                        if syntax_legacy { "WITH" } else { "FORMAT ENCODE" }
+                    ),
                 )))
-                .into())
+                    .into())
             }
             (_, false, true) => {
                 Err(ErrorCode::SinkError(Box::new(Error::new(
                     ErrorKind::InvalidInput,
-                    format!("Cannot force the sink to be append-only without \"{}\".", if syntax_legacy {"type='append-only'"} else {"FORMAT PLAIN"}),
+                    format!("Cannot force the sink to be append-only without \"{}\".", if syntax_legacy { "type='append-only'" } else { "FORMAT PLAIN" }),
                 )))
-                .into())
+                    .into())
             }
         }
     }
@@ -612,6 +621,7 @@ mod test {
             },
         ]
     }
+
     #[test]
     fn test_iceberg_convert_to_expression() {
         let partition_type = StructType::new(vec![
diff --git a/src/frontend/src/optimizer/plan_node/stream_source.rs b/src/frontend/src/optimizer/plan_node/stream_source.rs
index 7b918a6312a63..865d444f8c154 100644
--- a/src/frontend/src/optimizer/plan_node/stream_source.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_source.rs
@@ -18,7 +18,7 @@ use fixedbitset::FixedBitSet;
 use itertools::Itertools;
 use pretty_xmlish::{Pretty, XmlNode};
 use risingwave_common::util::iter_util::ZipEqFast;
-use risingwave_connector::parser::additional_columns::add_partition_offset_cols;
+use risingwave_connector::parser::additional_columns::source_add_partition_offset_cols;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 use risingwave_pb::stream_plan::{PbStreamSource, SourceNode};
 
@@ -46,8 +46,10 @@ impl StreamSource {
         if let Some(source_catalog) = &core.catalog
             && source_catalog.info.is_shared()
         {
-            let (columns_exist, additional_columns) =
-                add_partition_offset_cols(&core.column_catalog, &source_catalog.connector_name());
+            let (columns_exist, additional_columns) = source_add_partition_offset_cols(
+                &core.column_catalog,
+                &source_catalog.connector_name(),
+            );
             for (existed, mut c) in columns_exist.into_iter().zip_eq_fast(additional_columns) {
                 c.is_hidden = true;
                 if !existed {
@@ -93,8 +95,6 @@ impl StreamNode for StreamSource {
             source_id: source_catalog.id,
             source_name: source_catalog.name.clone(),
             state_table: Some(
-                // `StreamSource` can write all data to the same vnode
-                // but it is ok because we only do point get on each key rather than range scan.
                 generic::Source::infer_internal_table_catalog(false)
                     .with_id(state.gen_table_id_wrapped())
                     .to_internal_table_prost(),
diff --git a/src/frontend/src/optimizer/plan_node/stream_source_scan.rs b/src/frontend/src/optimizer/plan_node/stream_source_scan.rs
index 597517ce1e3f8..02f794fe55a9f 100644
--- a/src/frontend/src/optimizer/plan_node/stream_source_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_source_scan.rs
@@ -21,7 +21,7 @@ use risingwave_common::catalog::Field;
 use risingwave_common::types::DataType;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::sort_util::OrderType;
-use risingwave_connector::parser::additional_columns::add_partition_offset_cols;
+use risingwave_connector::parser::additional_columns::source_add_partition_offset_cols;
 use risingwave_pb::stream_plan::stream_node::{NodeBody, PbNodeBody};
 use risingwave_pb::stream_plan::PbStreamNode;
 
@@ -58,8 +58,10 @@ impl StreamSourceScan {
         if let Some(source_catalog) = &core.catalog
             && source_catalog.info.is_shared()
         {
-            let (columns_exist, additional_columns) =
-                add_partition_offset_cols(&core.column_catalog, &source_catalog.connector_name());
+            let (columns_exist, additional_columns) = source_add_partition_offset_cols(
+                &core.column_catalog,
+                &source_catalog.connector_name(),
+            );
             for (existed, mut c) in columns_exist.into_iter().zip_eq_fast(additional_columns) {
                 c.is_hidden = true;
                 if !existed {
@@ -94,13 +96,9 @@ impl StreamSourceScan {
             .expect("source scan should have source cataglog")
     }
 
+    /// The state is different from but similar to `StreamSource`.
+    /// Refer to [`generic::Source::infer_internal_table_catalog`] for more details.
     pub fn infer_internal_table_catalog() -> TableCatalog {
-        // note that source's internal table is to store partition_id -> offset mapping and its
-        // schema is irrelevant to input schema
-        // On the premise of ensuring that the materialized_source data can be cleaned up, keep the
-        // state in source.
-        // Source state doesn't maintain retention_seconds, internal_table_subset function only
-        // returns retention_seconds so default is used here
         let mut builder = TableCatalogBuilder::default();
 
         let key = Field {
@@ -119,7 +117,7 @@ impl StreamSourceScan {
         let ordered_col_idx = builder.add_column(&key);
         builder.add_column(&value);
         builder.add_order_column(ordered_col_idx, OrderType::ascending());
-        // read prefix hint is 0. We need to scan all data in the state table.
+        // Hacky: read prefix hint is 0, because we need to scan all data in the state table.
         builder.build(vec![], 0)
     }
 
diff --git a/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs b/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs
index d5fce4eecdc1c..8ce0997b7fe1f 100644
--- a/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_stateless_simple_agg.rs
@@ -22,7 +22,6 @@ use super::utils::impl_distill_by_unit;
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::expr::{ExprRewriter, ExprVisitor};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::plan_node::generic::PhysicalPlanRef;
 use crate::optimizer::property::RequiredDist;
 use crate::stream_fragmenter::BuildFragmentGraphState;
 
diff --git a/src/frontend/src/optimizer/plan_node/stream_subscription.rs b/src/frontend/src/optimizer/plan_node/stream_subscription.rs
deleted file mode 100644
index 8b165d5bbacbc..0000000000000
--- a/src/frontend/src/optimizer/plan_node/stream_subscription.rs
+++ /dev/null
@@ -1,209 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::assert_matches::assert_matches;
-use std::collections::HashSet;
-
-use fixedbitset::FixedBitSet;
-use itertools::Itertools;
-use pretty_xmlish::{Pretty, XmlNode};
-use risingwave_common::catalog::{ColumnCatalog, TableId, UserId};
-use risingwave_pb::stream_plan::stream_node::PbNodeBody;
-
-use super::derive::{derive_columns, derive_pk};
-use super::expr_visitable::ExprVisitable;
-use super::stream::prelude::{GenericPlanRef, PhysicalPlanRef};
-use super::utils::{
-    childless_record, infer_kv_log_store_table_catalog_inner, Distill, IndicesDisplay,
-};
-use super::{ExprRewritable, PlanBase, PlanTreeNodeUnary, Stream, StreamNode};
-use crate::catalog::subscription_catalog::{SubscriptionCatalog, SubscriptionId};
-use crate::error::Result;
-use crate::optimizer::property::{Distribution, Order, RequiredDist};
-use crate::stream_fragmenter::BuildFragmentGraphState;
-use crate::{PlanRef, TableCatalog, WithOptions};
-
-/// [`StreamSubscription`] represents a subscription at the very end of the graph.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct StreamSubscription {
-    pub base: PlanBase<Stream>,
-    input: PlanRef,
-    subscription_catalog: SubscriptionCatalog,
-}
-
-impl StreamSubscription {
-    #[must_use]
-    pub fn new(input: PlanRef, subscription_catalog: SubscriptionCatalog) -> Self {
-        let base = input
-            .plan_base()
-            .into_stream()
-            .expect("input should be stream plan")
-            .clone_with_new_plan_id();
-        Self {
-            base,
-            input,
-            subscription_catalog,
-        }
-    }
-
-    pub fn subscription_catalog(&self) -> SubscriptionCatalog {
-        self.subscription_catalog.clone()
-    }
-
-    #[allow(clippy::too_many_arguments)]
-    pub fn create(
-        database_id: u32,
-        schema_id: u32,
-        dependent_relations: HashSet<TableId>,
-        input: PlanRef,
-        name: String,
-        subscription_from_name: String,
-        user_distributed_by: RequiredDist,
-        user_order_by: Order,
-        user_cols: FixedBitSet,
-        out_names: Vec<String>,
-        definition: String,
-        properties: WithOptions,
-        user_id: UserId,
-    ) -> Result<Self> {
-        let columns = derive_columns(input.schema(), out_names, &user_cols)?;
-        let (input, subscription) = Self::derive_subscription_catalog(
-            database_id,
-            schema_id,
-            dependent_relations,
-            input,
-            user_distributed_by,
-            name,
-            subscription_from_name,
-            user_order_by,
-            columns,
-            definition,
-            properties,
-            user_id,
-        )?;
-        Ok(Self::new(input, subscription))
-    }
-
-    #[allow(clippy::too_many_arguments)]
-    fn derive_subscription_catalog(
-        database_id: u32,
-        schema_id: u32,
-        dependent_relations: HashSet<TableId>,
-        input: PlanRef,
-        user_distributed_by: RequiredDist,
-        name: String,
-        subscription_from_name: String,
-        user_order_by: Order,
-        columns: Vec<ColumnCatalog>,
-        definition: String,
-        properties: WithOptions,
-        user_id: UserId,
-    ) -> Result<(PlanRef, SubscriptionCatalog)> {
-        let (pk, _) = derive_pk(input.clone(), user_order_by, &columns);
-        let required_dist = match input.distribution() {
-            Distribution::Single => RequiredDist::single(),
-            _ => {
-                assert_matches!(user_distributed_by, RequiredDist::Any);
-                RequiredDist::shard_by_key(input.schema().len(), input.expect_stream_key())
-            }
-        };
-        let input = required_dist.enforce_if_not_satisfies(input, &Order::any())?;
-        let distribution_key = input.distribution().dist_column_indices().to_vec();
-        let subscription_desc = SubscriptionCatalog {
-            database_id,
-            schema_id,
-            dependent_relations: dependent_relations.into_iter().collect(),
-            id: SubscriptionId::placeholder(),
-            name,
-            subscription_from_name,
-            definition,
-            columns,
-            plan_pk: pk,
-            distribution_key,
-            properties: properties.into_inner(),
-            owner: user_id,
-            initialized_at_epoch: None,
-            created_at_epoch: None,
-            created_at_cluster_version: None,
-            initialized_at_cluster_version: None,
-            subscription_internal_table_name: None,
-        };
-        Ok((input, subscription_desc))
-    }
-
-    /// The table schema is: | epoch | seq id | row op | subscription columns |
-    /// Pk is: | epoch | seq id |
-    fn infer_kv_log_store_table_catalog(&self) -> TableCatalog {
-        infer_kv_log_store_table_catalog_inner(&self.input, &self.subscription_catalog.columns)
-    }
-}
-
-impl PlanTreeNodeUnary for StreamSubscription {
-    fn input(&self) -> PlanRef {
-        self.input.clone()
-    }
-
-    fn clone_with_input(&self, input: PlanRef) -> Self {
-        Self::new(input, self.subscription_catalog.clone())
-        // TODO(nanderstabel): Add assertions (assert_eq!)
-    }
-}
-
-impl_plan_tree_node_for_unary! { StreamSubscription }
-
-impl Distill for StreamSubscription {
-    fn distill<'a>(&self) -> XmlNode<'a> {
-        let column_names = self
-            .subscription_catalog
-            .columns
-            .iter()
-            .map(|col| col.name_with_hidden().to_string())
-            .map(Pretty::from)
-            .collect();
-        let column_names = Pretty::Array(column_names);
-        let mut vec = Vec::with_capacity(2);
-        vec.push(("columns", column_names));
-        let pk = IndicesDisplay {
-            indices: &self
-                .subscription_catalog
-                .plan_pk
-                .iter()
-                .map(|k| k.column_index)
-                .collect_vec(),
-            schema: self.base.schema(),
-        };
-        vec.push(("pk", pk.distill()));
-        childless_record("Streamsubscription", vec)
-    }
-}
-
-impl StreamNode for StreamSubscription {
-    fn to_stream_prost_body(&self, state: &mut BuildFragmentGraphState) -> PbNodeBody {
-        use risingwave_pb::stream_plan::*;
-
-        // We need to create a table for subscription with a kv log store.
-        let table = self
-            .infer_kv_log_store_table_catalog()
-            .with_id(state.gen_table_id_wrapped());
-
-        PbNodeBody::Subscription(SubscriptionNode {
-            subscription_catalog: Some(self.subscription_catalog.to_proto()),
-            log_store_table: Some(table.to_internal_table_prost()),
-        })
-    }
-}
-
-impl ExprRewritable for StreamSubscription {}
-
-impl ExprVisitable for StreamSubscription {}
diff --git a/src/frontend/src/optimizer/plan_node/stream_table_scan.rs b/src/frontend/src/optimizer/plan_node/stream_table_scan.rs
index 458707910c089..35991349f17cc 100644
--- a/src/frontend/src/optimizer/plan_node/stream_table_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_table_scan.rs
@@ -147,6 +147,7 @@ impl StreamTableScan {
     ///        | 1002 | Int64(1) | t                   | 10          |
     ///        | 1003 | Int64(1) | t                   | 10          |
     ///        | 1003 | Int64(1) | t                   | 10          |
+    ///
     /// Eventually we should track progress per vnode, to support scaling with both mview and
     /// the corresponding `no_shuffle_backfill`.
     /// However this is not high priority, since we are working on supporting arrangement backfill,
diff --git a/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs b/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs
index ce8753b9ddbc8..f94dbba36cb79 100644
--- a/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs
@@ -20,9 +20,7 @@ use risingwave_pb::stream_plan::stream_node::NodeBody;
 use risingwave_pb::stream_plan::TemporalJoinNode;
 use risingwave_sqlparser::ast::AsOf;
 
-use super::generic::GenericPlanRef;
 use super::stream::prelude::*;
-use super::stream::StreamPlanRef;
 use super::utils::{childless_record, watermark_pretty, Distill};
 use super::{generic, ExprRewritable, PlanBase, PlanRef, PlanTreeNodeBinary};
 use crate::expr::{Expr, ExprRewriter, ExprVisitor};
@@ -74,7 +72,7 @@ impl StreamTemporalJoin {
         let base = PlanBase::new_stream_with_core(
             &core,
             dist,
-            true,
+            append_only,
             false, // TODO(rc): derive EOWC property from input
             watermark_columns,
         );
diff --git a/src/frontend/src/optimizer/plan_node/stream_union.rs b/src/frontend/src/optimizer/plan_node/stream_union.rs
index 7a317a2677495..1c269ec0c5ad2 100644
--- a/src/frontend/src/optimizer/plan_node/stream_union.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_union.rs
@@ -19,9 +19,7 @@ use pretty_xmlish::XmlNode;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 use risingwave_pb::stream_plan::UnionNode;
 
-use super::generic::GenericPlanRef;
 use super::stream::prelude::*;
-use super::stream::StreamPlanRef;
 use super::utils::{childless_record, watermark_pretty, Distill};
 use super::{generic, ExprRewritable, PlanRef};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
diff --git a/src/frontend/src/optimizer/plan_node/stream_watermark_filter.rs b/src/frontend/src/optimizer/plan_node/stream_watermark_filter.rs
index ed7e367f93715..ffb08776b3fe5 100644
--- a/src/frontend/src/optimizer/plan_node/stream_watermark_filter.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_watermark_filter.rs
@@ -20,7 +20,6 @@ use risingwave_pb::catalog::WatermarkDesc;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 
 use super::stream::prelude::*;
-use super::stream::StreamPlanRef;
 use super::utils::{childless_record, watermark_pretty, Distill, TableCatalogBuilder};
 use super::{ExprRewritable, PlanBase, PlanRef, PlanTreeNodeUnary, StreamNode};
 use crate::expr::{ExprDisplay, ExprImpl};
diff --git a/src/frontend/src/optimizer/plan_node/utils.rs b/src/frontend/src/optimizer/plan_node/utils.rs
index f16aed4521180..9ac61ed759c6d 100644
--- a/src/frontend/src/optimizer/plan_node/utils.rs
+++ b/src/frontend/src/optimizer/plan_node/utils.rs
@@ -31,6 +31,7 @@ use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use crate::catalog::table_catalog::TableType;
 use crate::catalog::{ColumnId, TableCatalog, TableId};
 use crate::optimizer::property::{Cardinality, Order, RequiredDist};
+use crate::optimizer::StreamScanType;
 use crate::utils::{Condition, IndexSet};
 
 #[derive(Default)]
@@ -41,7 +42,6 @@ pub struct TableCatalogBuilder {
     value_indices: Option<Vec<usize>>,
     vnode_col_idx: Option<usize>,
     column_names: HashMap<String, i32>,
-    read_prefix_len_hint: usize,
     watermark_columns: Option<FixedBitSet>,
     dist_key_in_pk: Option<Vec<usize>>,
 }
@@ -136,7 +136,7 @@ impl TableCatalogBuilder {
     /// anticipated read prefix pattern (number of fields) for the table, which can be utilized for
     /// implementing the table's bloom filter or other storage optimization techniques.
     pub fn build(self, distribution_key: Vec<usize>, read_prefix_len_hint: usize) -> TableCatalog {
-        assert!(self.read_prefix_len_hint <= self.pk.len());
+        assert!(read_prefix_len_hint <= self.pk.len());
         let watermark_columns = match self.watermark_columns {
             Some(w) => w,
             None => FixedBitSet::with_capacity(self.columns.len()),
@@ -297,6 +297,7 @@ pub(crate) fn sum_affected_row(dml: PlanRef) -> Result<PlanRef> {
         order_by: vec![],
         filter: Condition::true_cond(),
         direct_args: vec![],
+        user_defined: None,
     };
     let agg = Agg::new(vec![sum_agg], IndexSet::empty(), dml);
     let batch_agg = BatchSimpleAgg::new(agg);
@@ -371,3 +372,19 @@ pub fn infer_kv_log_store_table_catalog_inner(
 
     table_catalog_builder.build(dist_key, read_prefix_len_hint)
 }
+
+/// Check that all leaf nodes must be stream table scan,
+/// since that plan node maps to `backfill` executor, which supports recovery.
+pub(crate) fn plan_has_backfill_leaf_nodes(plan: &PlanRef) -> bool {
+    if plan.inputs().is_empty() {
+        if let Some(scan) = plan.as_stream_table_scan() {
+            scan.stream_scan_type() == StreamScanType::Backfill
+                || scan.stream_scan_type() == StreamScanType::ArrangementBackfill
+        } else {
+            false
+        }
+    } else {
+        assert!(!plan.inputs().is_empty());
+        plan.inputs().iter().all(plan_has_backfill_leaf_nodes)
+    }
+}
diff --git a/src/frontend/src/optimizer/plan_visitor/jsonb_stream_key_checker.rs b/src/frontend/src/optimizer/plan_visitor/jsonb_stream_key_checker.rs
index d0a4358b4582a..c41e1c5b36463 100644
--- a/src/frontend/src/optimizer/plan_visitor/jsonb_stream_key_checker.rs
+++ b/src/frontend/src/optimizer/plan_visitor/jsonb_stream_key_checker.rs
@@ -17,7 +17,7 @@ use risingwave_common::types::DataType;
 
 use super::{DefaultBehavior, Merge};
 use crate::optimizer::plan_node::generic::GenericPlanRef;
-use crate::optimizer::plan_node::{PlanNode, *};
+use crate::optimizer::plan_node::*;
 use crate::optimizer::plan_visitor::PlanVisitor;
 
 #[derive(Debug, Clone, Default)]
diff --git a/src/frontend/src/optimizer/property/distribution.rs b/src/frontend/src/optimizer/property/distribution.rs
index 4999e1d8630bf..cf02daac47d83 100644
--- a/src/frontend/src/optimizer/property/distribution.rs
+++ b/src/frontend/src/optimizer/property/distribution.rs
@@ -51,7 +51,7 @@ use generic::PhysicalPlanRef;
 use itertools::Itertools;
 use risingwave_batch::worker_manager::worker_node_manager::WorkerNodeSelector;
 use risingwave_common::catalog::{FieldDisplay, Schema, TableId};
-use risingwave_common::hash::ParallelUnitId;
+use risingwave_common::hash::WorkerSlotId;
 use risingwave_pb::batch_plan::exchange_info::{
     ConsistentHashInfo, Distribution as DistributionPb, DistributionMode, HashInfo,
 };
@@ -62,7 +62,6 @@ use crate::catalog::catalog_service::CatalogReader;
 use crate::catalog::FragmentId;
 use crate::error::Result;
 use crate::optimizer::property::Order;
-use crate::optimizer::PlanRef;
 
 /// the distribution property provided by a operator.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -149,14 +148,17 @@ impl Distribution {
                     let vnode_mapping = worker_node_manager
                         .fragment_mapping(Self::get_fragment_id(catalog_reader, table_id)?)?;
 
-                    let pu2id_map: HashMap<ParallelUnitId, u32> = vnode_mapping
+                    let worker_slot_to_id_map: HashMap<WorkerSlotId, u32> = vnode_mapping
                         .iter_unique()
                         .enumerate()
-                        .map(|(i, pu)| (pu, i as u32))
+                        .map(|(i, worker_slot_id)| (worker_slot_id, i as u32))
                         .collect();
 
                     Some(DistributionPb::ConsistentHashInfo(ConsistentHashInfo {
-                        vmap: vnode_mapping.iter().map(|x| pu2id_map[&x]).collect_vec(),
+                        vmap: vnode_mapping
+                            .iter()
+                            .map(|id| worker_slot_to_id_map[&id])
+                            .collect_vec(),
                         key: key.iter().map(|num| *num as u32).collect(),
                     }))
                 }
diff --git a/src/frontend/src/optimizer/property/order.rs b/src/frontend/src/optimizer/property/order.rs
index 1a657e190fac4..68f2120df567b 100644
--- a/src/frontend/src/optimizer/property/order.rs
+++ b/src/frontend/src/optimizer/property/order.rs
@@ -22,7 +22,6 @@ use risingwave_pb::common::PbColumnOrder;
 
 use super::super::plan_node::*;
 use crate::error::Result;
-use crate::optimizer::PlanRef;
 
 // TODO(rc): use this type to replace all `Vec<ColumnOrder>`
 #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)]
diff --git a/src/frontend/src/optimizer/rule/agg_group_by_simplify_rule.rs b/src/frontend/src/optimizer/rule/agg_group_by_simplify_rule.rs
index b700e36987826..d55c00d10bb2d 100644
--- a/src/frontend/src/optimizer/rule/agg_group_by_simplify_rule.rs
+++ b/src/frontend/src/optimizer/rule/agg_group_by_simplify_rule.rs
@@ -54,6 +54,7 @@ impl Rule for AggGroupBySimplifyRule {
                         order_by: vec![],
                         filter: Condition::true_cond(),
                         direct_args: vec![],
+                        user_defined: None,
                     });
                 }
             }
diff --git a/src/frontend/src/optimizer/rule/apply_agg_transpose_rule.rs b/src/frontend/src/optimizer/rule/apply_agg_transpose_rule.rs
index c53357234e350..48c3366d1d825 100644
--- a/src/frontend/src/optimizer/rule/apply_agg_transpose_rule.rs
+++ b/src/frontend/src/optimizer/rule/apply_agg_transpose_rule.rs
@@ -106,10 +106,11 @@ impl Rule for ApplyAggTransposeRule {
                 correlated_id,
                 correlated_indices.clone(),
                 false,
+                false,
             )
             .translate_apply(left, eq_predicates)
         } else {
-            LogicalApply::new(
+            LogicalApply::create(
                 left,
                 input,
                 JoinType::Inner,
@@ -118,7 +119,6 @@ impl Rule for ApplyAggTransposeRule {
                 correlated_indices.clone(),
                 false,
             )
-            .into()
         };
 
         let group_agg = {
@@ -147,7 +147,8 @@ impl Rule for ApplyAggTransposeRule {
                         }
                         AggKind::ArrayAgg
                         | AggKind::JsonbAgg
-                        | AggKind::JsonbObjectAgg => {
+                        | AggKind::JsonbObjectAgg
+                        | AggKind::UserDefined => {
                             let input_ref = InputRef::new(pos_of_constant_column, DataType::Int32);
                             let cond = FunctionCall::new(ExprType::IsNotNull, vec![input_ref.into()]).unwrap();
                             agg_call.filter.conjunctions.push(cond.into());
diff --git a/src/frontend/src/optimizer/rule/apply_dedup_transpose_rule.rs b/src/frontend/src/optimizer/rule/apply_dedup_transpose_rule.rs
index 798744168c424..1b793cddd2816 100644
--- a/src/frontend/src/optimizer/rule/apply_dedup_transpose_rule.rs
+++ b/src/frontend/src/optimizer/rule/apply_dedup_transpose_rule.rs
@@ -57,7 +57,7 @@ impl Rule for ApplyDedupTransposeRule {
             return None;
         }
 
-        let new_apply = LogicalApply::new(
+        let new_apply = LogicalApply::create(
             left,
             dedup_input,
             JoinType::Inner,
@@ -65,8 +65,7 @@ impl Rule for ApplyDedupTransposeRule {
             correlated_id,
             correlated_indices,
             false,
-        )
-        .into();
+        );
 
         let new_dedup = {
             let mut new_dedup_cols: Vec<usize> = (0..apply_left_len).collect();
diff --git a/src/frontend/src/optimizer/rule/apply_expand_transpose_rule.rs b/src/frontend/src/optimizer/rule/apply_expand_transpose_rule.rs
index 694fc803f87ff..9f03d9e420981 100644
--- a/src/frontend/src/optimizer/rule/apply_expand_transpose_rule.rs
+++ b/src/frontend/src/optimizer/rule/apply_expand_transpose_rule.rs
@@ -61,7 +61,7 @@ impl Rule for ApplyExpandTransposeRule {
             return None;
         }
 
-        let new_apply: PlanRef = LogicalApply::new(
+        let new_apply: PlanRef = LogicalApply::create(
             left,
             expand_input,
             JoinType::Inner,
@@ -69,8 +69,7 @@ impl Rule for ApplyExpandTransposeRule {
             correlated_id,
             correlated_indices,
             false,
-        )
-        .into();
+        );
 
         let new_apply_schema_len = new_apply.schema().len();
 
diff --git a/src/frontend/src/optimizer/rule/apply_hop_window_transpose_rule.rs b/src/frontend/src/optimizer/rule/apply_hop_window_transpose_rule.rs
index b3d5d0841bdc5..634a8c6feb45c 100644
--- a/src/frontend/src/optimizer/rule/apply_hop_window_transpose_rule.rs
+++ b/src/frontend/src/optimizer/rule/apply_hop_window_transpose_rule.rs
@@ -62,7 +62,7 @@ impl Rule for ApplyHopWindowTransposeRule {
             return None;
         }
 
-        let new_apply = LogicalApply::new(
+        let new_apply = LogicalApply::create(
             left,
             hop_window_input,
             JoinType::Inner,
@@ -70,8 +70,7 @@ impl Rule for ApplyHopWindowTransposeRule {
             correlated_id,
             correlated_indices,
             false,
-        )
-        .into();
+        );
 
         let new_hop_window = LogicalHopWindow::create(
             new_apply,
diff --git a/src/frontend/src/optimizer/rule/apply_limit_transpose_rule.rs b/src/frontend/src/optimizer/rule/apply_limit_transpose_rule.rs
index 8dc01c195cee5..56395ed61a6b6 100644
--- a/src/frontend/src/optimizer/rule/apply_limit_transpose_rule.rs
+++ b/src/frontend/src/optimizer/rule/apply_limit_transpose_rule.rs
@@ -63,7 +63,7 @@ impl Rule for ApplyLimitTransposeRule {
             return None;
         }
 
-        let new_apply = LogicalApply::new(
+        let new_apply = LogicalApply::create(
             left,
             limit_input,
             JoinType::Inner,
@@ -71,8 +71,7 @@ impl Rule for ApplyLimitTransposeRule {
             correlated_id,
             correlated_indices,
             false,
-        )
-        .into();
+        );
 
         let new_topn = {
             // use the first column as an order to provide determinism for streaming queries.
diff --git a/src/frontend/src/optimizer/rule/apply_over_window_transpose_rule.rs b/src/frontend/src/optimizer/rule/apply_over_window_transpose_rule.rs
index 34ae1902b4d27..bb848561ce0b6 100644
--- a/src/frontend/src/optimizer/rule/apply_over_window_transpose_rule.rs
+++ b/src/frontend/src/optimizer/rule/apply_over_window_transpose_rule.rs
@@ -58,7 +58,7 @@ impl Rule for ApplyOverWindowTransposeRule {
         let apply_left_len = left.schema().len();
         let apply_left_schema = left.schema().clone();
 
-        let new_apply = LogicalApply::new(
+        let new_apply = LogicalApply::create(
             left,
             window_input,
             JoinType::Inner,
@@ -66,8 +66,7 @@ impl Rule for ApplyOverWindowTransposeRule {
             correlated_id,
             correlated_indices,
             false,
-        )
-        .into();
+        );
 
         let new_over_window = {
             // Shift index of window functions' `InputRef` with `apply_left_len`.
diff --git a/src/frontend/src/optimizer/rule/apply_topn_transpose_rule.rs b/src/frontend/src/optimizer/rule/apply_topn_transpose_rule.rs
index 61b887af7ea54..9a6884b33e9a2 100644
--- a/src/frontend/src/optimizer/rule/apply_topn_transpose_rule.rs
+++ b/src/frontend/src/optimizer/rule/apply_topn_transpose_rule.rs
@@ -57,7 +57,7 @@ impl Rule for ApplyTopNTransposeRule {
             return None;
         }
 
-        let new_apply = LogicalApply::new(
+        let new_apply = LogicalApply::create(
             left,
             topn_input,
             JoinType::Inner,
@@ -65,8 +65,7 @@ impl Rule for ApplyTopNTransposeRule {
             correlated_id,
             correlated_indices,
             false,
-        )
-        .into();
+        );
 
         let new_topn = {
             // shift index of topn's `InputRef` with `apply_left_len`.
diff --git a/src/frontend/src/optimizer/rule/logical_filter_expression_simplify_rule.rs b/src/frontend/src/optimizer/rule/logical_filter_expression_simplify_rule.rs
index ebc724c200481..67f7e40ed3455 100644
--- a/src/frontend/src/optimizer/rule/logical_filter_expression_simplify_rule.rs
+++ b/src/frontend/src/optimizer/rule/logical_filter_expression_simplify_rule.rs
@@ -33,6 +33,7 @@ impl Rule for LogicalFilterExpressionSimplifyRule {
     /// The pattern we aim to optimize, e.g.,
     /// 1. (NOT (e)) OR (e) => True
     /// 2. (NOT (e)) AND (e) => False
+    ///
     /// NOTE: `e` should only contain at most a single column
     /// otherwise we will not conduct the optimization
     fn apply(&self, plan: PlanRef) -> Option<PlanRef> {
@@ -153,6 +154,7 @@ fn check_optimizable_pattern(e1: ExprImpl, e2: ExprImpl) -> (bool, Option<ExprIm
 
 /// 1. True or (...) | (...) or True => True
 /// 2. False and (...) | (...) and False => False
+///
 /// NOTE: the `True` and `False` here not only represent a single `ExprImpl::Literal`
 /// but represent every `ExprImpl` that can be *evaluated* to `ScalarImpl::Bool`
 /// during optimization phase as well
diff --git a/src/frontend/src/optimizer/rule/min_max_on_index_rule.rs b/src/frontend/src/optimizer/rule/min_max_on_index_rule.rs
index f7f9ffb02dddd..7c663ea714b6d 100644
--- a/src/frontend/src/optimizer/rule/min_max_on_index_rule.rs
+++ b/src/frontend/src/optimizer/rule/min_max_on_index_rule.rs
@@ -124,6 +124,7 @@ impl MinMaxOnIndexRule {
                             conjunctions: vec![],
                         },
                         direct_args: vec![],
+                        user_defined: None,
                     }],
                     IndexSet::empty(),
                     topn.into(),
@@ -194,6 +195,7 @@ impl MinMaxOnIndexRule {
                         conjunctions: vec![],
                     },
                     direct_args: vec![],
+                    user_defined: None,
                 }],
                 IndexSet::empty(),
                 topn.into(),
diff --git a/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_agg_rule.rs b/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_agg_rule.rs
index 7823c7f41f7b4..c9cb4edc860cd 100644
--- a/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_agg_rule.rs
+++ b/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_agg_rule.rs
@@ -24,7 +24,6 @@ use crate::expr::{Expr, ExprImpl, ExprRewriter, ExprType, FunctionCall, InputRef
 use crate::optimizer::plan_expr_visitor::Strong;
 use crate::optimizer::plan_node::generic::{Agg, GenericPlanNode, GenericPlanRef};
 use crate::optimizer::plan_visitor::{PlanCorrelatedIdFinder, PlanVisitor};
-use crate::optimizer::PlanRef;
 use crate::utils::{Condition, IndexSet};
 
 /// Pull up correlated predicates from the right agg side of Apply to the `on` clause of Join.
diff --git a/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_rule.rs b/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_rule.rs
index b5df4f1738072..d6b59b9553911 100644
--- a/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_rule.rs
+++ b/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_rule.rs
@@ -20,7 +20,6 @@ use super::{BoxedRule, Rule};
 use crate::expr::{CorrelatedId, CorrelatedInputRef, Expr, ExprImpl, ExprRewriter, InputRef};
 use crate::optimizer::plan_node::generic::GenericPlanRef;
 use crate::optimizer::plan_visitor::{PlanCorrelatedIdFinder, PlanVisitor};
-use crate::optimizer::PlanRef;
 use crate::utils::Condition;
 
 /// This rule is for pattern: Apply->Project->Filter.
diff --git a/src/frontend/src/optimizer/rule/stream/stream_project_merge_rule.rs b/src/frontend/src/optimizer/rule/stream/stream_project_merge_rule.rs
index 0eba52e193d22..91ab942e3a7fc 100644
--- a/src/frontend/src/optimizer/rule/stream/stream_project_merge_rule.rs
+++ b/src/frontend/src/optimizer/rule/stream/stream_project_merge_rule.rs
@@ -47,7 +47,15 @@ impl Rule for StreamProjectMergeRule {
             .map(|expr| subst.rewrite_expr(expr))
             .collect();
         let logical_project = generic::Project::new(exprs, inner_project.input());
-        Some(StreamProject::new(logical_project).into())
+
+        // If either of the projects has the hint, we should keep it.
+        let noop_update_hint = outer_project.noop_update_hint() || inner_project.noop_update_hint();
+
+        Some(
+            StreamProject::new(logical_project)
+                .with_noop_update_hint(noop_update_hint)
+                .into(),
+        )
     }
 }
 
diff --git a/src/frontend/src/optimizer/rule/translate_apply_rule.rs b/src/frontend/src/optimizer/rule/translate_apply_rule.rs
index 53a656fa296a9..876ca7d6285b2 100644
--- a/src/frontend/src/optimizer/rule/translate_apply_rule.rs
+++ b/src/frontend/src/optimizer/rule/translate_apply_rule.rs
@@ -54,6 +54,9 @@ pub struct TranslateApplyRule {
 impl Rule for TranslateApplyRule {
     fn apply(&self, plan: PlanRef) -> Option<PlanRef> {
         let apply: &LogicalApply = plan.as_logical_apply()?;
+        if apply.translated() {
+            return None;
+        }
         let mut left: PlanRef = apply.left();
         let right: PlanRef = apply.right();
         let apply_left_len = left.schema().len();
@@ -167,6 +170,15 @@ impl TranslateApplyRule {
                 data_types,
                 index,
             )
+        } else if let Some(apply) = plan.as_logical_apply() {
+            Self::rewrite_apply(
+                apply,
+                correlated_indices,
+                offset,
+                index_mapping,
+                data_types,
+                index,
+            )
         } else if let Some(scan) = plan.as_logical_scan() {
             Self::rewrite_scan(
                 scan,
@@ -199,18 +211,6 @@ impl TranslateApplyRule {
         data_types: &mut HashMap<usize, DataType>,
         index: &mut usize,
     ) -> Option<PlanRef> {
-        // Only accept join which doesn't generate null columns.
-        if !matches!(
-            join.join_type(),
-            JoinType::Inner
-                | JoinType::LeftSemi
-                | JoinType::RightSemi
-                | JoinType::LeftAnti
-                | JoinType::RightAnti
-        ) {
-            return None;
-        }
-
         // TODO: Do we need to take the `on` into account?
         let left_len = join.left().schema().len();
         let (left_idxs, right_idxs): (Vec<_>, Vec<_>) = required_col_idx
@@ -222,28 +222,99 @@ impl TranslateApplyRule {
                     indices.iter_mut().for_each(|index| *index -= left_len);
                     offset += left_len;
                 }
-                if let Some(join) = plan.as_logical_join() {
-                    Self::rewrite_join(join, indices, offset, index_mapping, data_types, index)
-                } else if let Some(scan) = plan.as_logical_scan() {
-                    Self::rewrite_scan(scan, indices, offset, index_mapping, data_types, index)
-                } else {
-                    None
-                }
+                Self::rewrite(&plan, indices, offset, index_mapping, data_types, index)
             };
         match (left_idxs.is_empty(), right_idxs.is_empty()) {
-            (true, false) => rewrite(join.right(), right_idxs, true),
-            (false, true) => rewrite(join.left(), left_idxs, false),
+            (true, false) => {
+                // Only accept join which doesn't generate null columns.
+                match join.join_type() {
+                    JoinType::Inner
+                    | JoinType::LeftSemi
+                    | JoinType::RightSemi
+                    | JoinType::LeftAnti
+                    | JoinType::RightAnti
+                    | JoinType::RightOuter => rewrite(join.right(), right_idxs, true),
+                    JoinType::LeftOuter | JoinType::FullOuter => None,
+                    JoinType::Unspecified => unreachable!(),
+                }
+            }
+            (false, true) => {
+                // Only accept join which doesn't generate null columns.
+                match join.join_type() {
+                    JoinType::Inner
+                    | JoinType::LeftSemi
+                    | JoinType::RightSemi
+                    | JoinType::LeftAnti
+                    | JoinType::RightAnti
+                    | JoinType::LeftOuter => rewrite(join.left(), left_idxs, false),
+                    JoinType::RightOuter | JoinType::FullOuter => None,
+                    JoinType::Unspecified => unreachable!(),
+                }
+            }
             (false, false) => {
-                let left = rewrite(join.left(), left_idxs, false)?;
-                let right = rewrite(join.right(), right_idxs, true)?;
-                let new_join =
-                    LogicalJoin::new(left, right, join.join_type(), Condition::true_cond());
-                Some(new_join.into())
+                // Only accept join which doesn't generate null columns.
+                match join.join_type() {
+                    JoinType::Inner
+                    | JoinType::LeftSemi
+                    | JoinType::RightSemi
+                    | JoinType::LeftAnti
+                    | JoinType::RightAnti => {
+                        let left = rewrite(join.left(), left_idxs, false)?;
+                        let right = rewrite(join.right(), right_idxs, true)?;
+                        let new_join =
+                            LogicalJoin::new(left, right, join.join_type(), Condition::true_cond());
+                        Some(new_join.into())
+                    }
+                    JoinType::LeftOuter | JoinType::RightOuter | JoinType::FullOuter => None,
+                    JoinType::Unspecified => unreachable!(),
+                }
             }
             _ => None,
         }
     }
 
+    /// ```text
+    ///             LogicalApply
+    ///            /            \
+    ///     LogicalApply       RHS1
+    ///    /            \
+    ///  LHS           RHS2
+    /// ```
+    ///
+    /// A common structure of multi scalar subqueries is a chain of `LogicalApply`. To avoid exponential growth of the domain operator, we need to rewrite the apply and try to simplify the domain as much as possible.
+    /// We use a top-down apply order to rewrite the apply, so that we don't need to handle operator like project and aggregation generated by the domain calculation.
+    /// As a cost, we need to add a flag `translated` to the apply operator to remind `translate_apply_rule` that the apply has been translated.
+    fn rewrite_apply(
+        apply: &LogicalApply,
+        required_col_idx: Vec<usize>,
+        offset: usize,
+        index_mapping: &mut ColIndexMapping,
+        data_types: &mut HashMap<usize, DataType>,
+        index: &mut usize,
+    ) -> Option<PlanRef> {
+        // TODO: Do we need to take the `on` into account?
+        let left_len = apply.left().schema().len();
+        let (left_idxs, right_idxs): (Vec<_>, Vec<_>) = required_col_idx
+            .into_iter()
+            .partition(|idx| *idx < left_len);
+        if !left_idxs.is_empty() && right_idxs.is_empty() {
+            // Deal with multi scalar subqueries
+            match apply.join_type() {
+                JoinType::Inner | JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftOuter => {
+                    let plan = apply.left();
+                    Self::rewrite(&plan, left_idxs, offset, index_mapping, data_types, index)
+                }
+                JoinType::RightOuter
+                | JoinType::RightAnti
+                | JoinType::RightSemi
+                | JoinType::FullOuter => None,
+                JoinType::Unspecified => unreachable!(),
+            }
+        } else {
+            None
+        }
+    }
+
     fn rewrite_scan(
         scan: &LogicalScan,
         required_col_idx: Vec<usize>,
diff --git a/src/frontend/src/planner/delete.rs b/src/frontend/src/planner/delete.rs
index 9e777774eee41..c4c34346ec1d2 100644
--- a/src/frontend/src/planner/delete.rs
+++ b/src/frontend/src/planner/delete.rs
@@ -68,7 +68,7 @@ impl Planner {
             plan.schema().names()
         };
 
-        let root = PlanRoot::new(plan, dist, Order::any(), out_fields, out_names);
+        let root = PlanRoot::new_with_logical_plan(plan, dist, Order::any(), out_fields, out_names);
         Ok(root)
     }
 }
diff --git a/src/frontend/src/planner/insert.rs b/src/frontend/src/planner/insert.rs
index 98a89202ff292..078cbd6c6ce0a 100644
--- a/src/frontend/src/planner/insert.rs
+++ b/src/frontend/src/planner/insert.rs
@@ -53,7 +53,7 @@ impl Planner {
         } else {
             plan.schema().names()
         };
-        let root = PlanRoot::new(plan, dist, Order::any(), out_fields, out_names);
+        let root = PlanRoot::new_with_logical_plan(plan, dist, Order::any(), out_fields, out_names);
         Ok(root)
     }
 }
diff --git a/src/frontend/src/planner/mod.rs b/src/frontend/src/planner/mod.rs
index d2f695faa5ab2..12bf25c0372df 100644
--- a/src/frontend/src/planner/mod.rs
+++ b/src/frontend/src/planner/mod.rs
@@ -21,6 +21,7 @@ use crate::optimizer::{OptimizerContextRef, PlanRoot};
 mod delete;
 mod insert;
 mod query;
+mod recursive_union;
 mod relation;
 mod select;
 mod set_expr;
diff --git a/src/frontend/src/planner/query.rs b/src/frontend/src/planner/query.rs
index cf6f46bc671ac..2edde6105d190 100644
--- a/src/frontend/src/planner/query.rs
+++ b/src/frontend/src/planner/query.rs
@@ -68,7 +68,8 @@ impl Planner {
             // Do not output projected_row_id hidden column.
             out_fields.set(0, false);
         }
-        let root = PlanRoot::new(plan, RequiredDist::Any, order, out_fields, out_names);
+        let root =
+            PlanRoot::new_with_logical_plan(plan, RequiredDist::Any, order, out_fields, out_names);
         Ok(root)
     }
 }
diff --git a/src/frontend/src/planner/recursive_union.rs b/src/frontend/src/planner/recursive_union.rs
new file mode 100644
index 0000000000000..655e50e951a4b
--- /dev/null
+++ b/src/frontend/src/planner/recursive_union.rs
@@ -0,0 +1,33 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::binder::{BoundSetExpr, ShareId};
+use crate::error::Result;
+use crate::optimizer::plan_node::LogicalRecursiveUnion;
+use crate::{PlanRef, Planner};
+
+impl Planner {
+    pub(super) fn plan_recursive_union(
+        &mut self,
+        base: BoundSetExpr,
+        recursive: BoundSetExpr,
+        id: ShareId,
+    ) -> Result<PlanRef> {
+        let base = self.plan_set_expr(base, vec![], &[])?;
+        let recursive = self.plan_set_expr(recursive, vec![], &[])?;
+        let plan = LogicalRecursiveUnion::create(base, recursive, id);
+        self.ctx.insert_rcte_cache_plan(id, plan.clone());
+        Ok(plan)
+    }
+}
diff --git a/src/frontend/src/planner/relation.rs b/src/frontend/src/planner/relation.rs
index d593a7308f39f..51d475616e210 100644
--- a/src/frontend/src/planner/relation.rs
+++ b/src/frontend/src/planner/relation.rs
@@ -22,15 +22,15 @@ use risingwave_common::types::{DataType, Interval, ScalarImpl};
 use risingwave_sqlparser::ast::AsOf;
 
 use crate::binder::{
-    BoundBaseTable, BoundJoin, BoundShare, BoundSource, BoundSystemTable, BoundWatermark,
-    BoundWindowTableFunction, Relation, WindowTableFunctionKind,
+    BoundBackCteRef, BoundBaseTable, BoundJoin, BoundShare, BoundSource, BoundSystemTable,
+    BoundWatermark, BoundWindowTableFunction, Relation, WindowTableFunctionKind,
 };
 use crate::error::{ErrorCode, Result};
 use crate::expr::{Expr, ExprImpl, ExprType, FunctionCall, InputRef};
 use crate::optimizer::plan_node::generic::SourceNodeKind;
 use crate::optimizer::plan_node::{
-    LogicalApply, LogicalHopWindow, LogicalJoin, LogicalProject, LogicalScan, LogicalShare,
-    LogicalSource, LogicalSysScan, LogicalTableFunction, LogicalValues, PlanRef,
+    LogicalApply, LogicalCteRef, LogicalHopWindow, LogicalJoin, LogicalProject, LogicalScan,
+    LogicalShare, LogicalSource, LogicalSysScan, LogicalTableFunction, LogicalValues, PlanRef,
 };
 use crate::optimizer::property::Cardinality;
 use crate::planner::Planner;
@@ -57,9 +57,7 @@ impl Planner {
             Relation::Watermark(tf) => self.plan_watermark(*tf),
             // note that rcte (i.e., RecursiveUnion) is included *implicitly* in share.
             Relation::Share(share) => self.plan_share(*share),
-            Relation::BackCteRef(..) => {
-                bail_not_implemented!(issue = 15135, "recursive CTE is not supported")
-            }
+            Relation::BackCteRef(cte_ref) => self.plan_cte_ref(*cte_ref),
         }
     }
 
@@ -197,41 +195,55 @@ impl Planner {
                 Ok(LogicalTableFunction::new(*tf, with_ordinality, self.ctx()).into())
             }
             expr => {
-                let mut schema = Schema {
+                let schema = Schema {
                     // TODO: should be named
                     fields: vec![Field::unnamed(expr.return_type())],
                 };
-                if with_ordinality {
-                    schema
-                        .fields
-                        .push(Field::with_name(DataType::Int64, "ordinality"));
-                    Ok(LogicalValues::create(
-                        vec![vec![expr, ExprImpl::literal_bigint(1)]],
-                        schema,
-                        self.ctx(),
-                    ))
+                let expr_return_type = expr.return_type();
+                let root = LogicalValues::create(vec![vec![expr]], schema, self.ctx());
+                let input_ref = ExprImpl::from(InputRef::new(0, expr_return_type.clone()));
+                let mut exprs = if let DataType::Struct(st) = expr_return_type {
+                    st.iter()
+                        .enumerate()
+                        .map(|(i, (_, ty))| {
+                            let idx = ExprImpl::literal_int(i.try_into().unwrap());
+                            let args = vec![input_ref.clone(), idx];
+                            FunctionCall::new_unchecked(ExprType::Field, args, ty.clone()).into()
+                        })
+                        .collect()
                 } else {
-                    Ok(LogicalValues::create(vec![vec![expr]], schema, self.ctx()))
+                    vec![input_ref]
+                };
+                if with_ordinality {
+                    exprs.push(ExprImpl::literal_bigint(1));
                 }
+                Ok(LogicalProject::create(root, exprs))
             }
         }
     }
 
     pub(super) fn plan_share(&mut self, share: BoundShare) -> Result<PlanRef> {
-        let Either::Left(nonrecursive_query) = share.input else {
-            bail_not_implemented!(issue = 15135, "recursive CTE is not supported");
-        };
-        let id = share.share_id;
-        match self.share_cache.get(&id) {
-            None => {
-                let result = self
-                    .plan_query(nonrecursive_query)?
-                    .into_unordered_subplan();
-                let logical_share = LogicalShare::create(result);
-                self.share_cache.insert(id, logical_share.clone());
-                Ok(logical_share)
+        match share.input {
+            Either::Left(nonrecursive_query) => {
+                let id = share.share_id;
+                match self.share_cache.get(&id) {
+                    None => {
+                        let result = self
+                            .plan_query(nonrecursive_query)?
+                            .into_unordered_subplan();
+                        let logical_share = LogicalShare::create(result);
+                        self.share_cache.insert(id, logical_share.clone());
+                        Ok(logical_share)
+                    }
+                    Some(result) => Ok(result.clone()),
+                }
             }
-            Some(result) => Ok(result.clone()),
+            // for the recursive union in rcte
+            Either::Right(recursive_union) => self.plan_recursive_union(
+                *recursive_union.base,
+                *recursive_union.recursive,
+                share.share_id,
+            ),
         }
     }
 
@@ -239,6 +251,12 @@ impl Planner {
         todo!("plan watermark");
     }
 
+    pub(super) fn plan_cte_ref(&mut self, cte_ref: BoundBackCteRef) -> Result<PlanRef> {
+        // TODO: this is actually duplicated from `plan_recursive_union`, refactor?
+        let base = self.plan_set_expr(cte_ref.base, vec![], &[])?;
+        Ok(LogicalCteRef::create(cte_ref.share_id, base))
+    }
+
     fn collect_col_data_types_for_tumble_window(relation: &Relation) -> Result<Vec<DataType>> {
         let col_data_types = match relation {
             Relation::Source(s) => s
diff --git a/src/frontend/src/planner/update.rs b/src/frontend/src/planner/update.rs
index 83e36f17084db..ef735db1b5a95 100644
--- a/src/frontend/src/planner/update.rs
+++ b/src/frontend/src/planner/update.rs
@@ -65,7 +65,7 @@ impl Planner {
             plan.schema().names()
         };
 
-        let root = PlanRoot::new(plan, dist, Order::any(), out_fields, out_names);
+        let root = PlanRoot::new_with_logical_plan(plan, dist, Order::any(), out_fields, out_names);
         Ok(root)
     }
 }
diff --git a/src/frontend/src/scheduler/distributed/query.rs b/src/frontend/src/scheduler/distributed/query.rs
index c6e866630067b..165bdcee6476b 100644
--- a/src/frontend/src/scheduler/distributed/query.rs
+++ b/src/frontend/src/scheduler/distributed/query.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::collections::HashMap;
-use std::default::Default;
 use std::fmt::{Debug, Formatter};
 use std::mem;
 use std::sync::Arc;
@@ -477,7 +476,7 @@ pub(crate) mod tests {
         ColumnCatalog, ColumnDesc, ConflictBehavior, CreateType, StreamJobStatus,
         DEFAULT_SUPER_USER_ID,
     };
-    use risingwave_common::hash::ParallelUnitMapping;
+    use risingwave_common::hash::{WorkerSlotId, WorkerSlotMapping};
     use risingwave_common::types::DataType;
     use risingwave_pb::common::worker_node::Property;
     use risingwave_pb::common::{HostAddress, ParallelUnit, WorkerNode, WorkerType};
@@ -722,10 +721,12 @@ pub(crate) mod tests {
         let workers = vec![worker1, worker2, worker3];
         let worker_node_manager = Arc::new(WorkerNodeManager::mock(workers));
         let worker_node_selector = WorkerNodeSelector::new(worker_node_manager.clone(), false);
-        worker_node_manager
-            .insert_streaming_fragment_mapping(0, ParallelUnitMapping::new_single(0));
+        worker_node_manager.insert_streaming_fragment_mapping(
+            0,
+            WorkerSlotMapping::new_single(WorkerSlotId::new(0, 0)),
+        );
         worker_node_manager.set_serving_fragment_mapping(
-            vec![(0, ParallelUnitMapping::new_single(0))]
+            vec![(0, WorkerSlotMapping::new_single(WorkerSlotId::new(0, 0)))]
                 .into_iter()
                 .collect(),
         );
diff --git a/src/frontend/src/scheduler/distributed/query_manager.rs b/src/frontend/src/scheduler/distributed/query_manager.rs
index 69d48eb70d976..86a54cf9c0f98 100644
--- a/src/frontend/src/scheduler/distributed/query_manager.rs
+++ b/src/frontend/src/scheduler/distributed/query_manager.rs
@@ -50,6 +50,7 @@ impl DistributedQueryStream {
 }
 
 impl Stream for DistributedQueryStream {
+    // TODO(error-handling): use a concrete error type.
     type Item = Result<DataChunk, BoxedError>;
 
     fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
diff --git a/src/frontend/src/scheduler/distributed/stage.rs b/src/frontend/src/scheduler/distributed/stage.rs
index 193ee6417abdc..8b7e07a0aefcd 100644
--- a/src/frontend/src/scheduler/distributed/stage.rs
+++ b/src/frontend/src/scheduler/distributed/stage.rs
@@ -32,7 +32,7 @@ use risingwave_batch::executor::ExecutorBuilder;
 use risingwave_batch::task::{ShutdownMsg, ShutdownSender, ShutdownToken, TaskId as TaskIdBatch};
 use risingwave_batch::worker_manager::worker_node_manager::WorkerNodeSelector;
 use risingwave_common::array::DataChunk;
-use risingwave_common::hash::ParallelUnitMapping;
+use risingwave_common::hash::WorkerSlotMapping;
 use risingwave_common::util::addr::HostAddr;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_connector::source::SplitMetaData;
@@ -94,7 +94,7 @@ pub enum StageEvent {
         reason: SchedulerError,
     },
     /// All tasks in stage finished.
-    Completed(StageId),
+    Completed(#[allow(dead_code)] StageId),
 }
 
 #[derive(Clone)]
@@ -172,7 +172,7 @@ impl StageExecution {
         ctx: ExecutionContextRef,
     ) -> Self {
         let tasks = (0..stage.parallelism.unwrap())
-            .map(|task_id| (task_id, TaskStatusHolder::new(task_id)))
+            .map(|task_id| (task_id as u64, TaskStatusHolder::new(task_id as u64)))
             .collect();
 
         Self {
@@ -289,7 +289,7 @@ impl StageExecution {
     ///
     /// When this method is called, all tasks should have been scheduled, and their `worker_node`
     /// should have been set.
-    pub fn all_exchange_sources_for(&self, output_id: u32) -> Vec<ExchangeSource> {
+    pub fn all_exchange_sources_for(&self, output_id: u64) -> Vec<ExchangeSource> {
         self.tasks
             .iter()
             .map(|(task_id, status_holder)| {
@@ -353,12 +353,13 @@ impl StageRunner {
             // We let each task read one partition by setting the `vnode_ranges` of the scan node in
             // the task.
             // We schedule the task to the worker node that owns the data partition.
-            let parallel_unit_ids = vnode_bitmaps.keys().cloned().collect_vec();
+            let worker_slot_ids = vnode_bitmaps.keys().cloned().collect_vec();
             let workers = self
                 .worker_node_manager
                 .manager
-                .get_workers_by_parallel_unit_ids(&parallel_unit_ids)?;
-            for (i, (parallel_unit_id, worker)) in parallel_unit_ids
+                .get_workers_by_worker_slot_ids(&worker_slot_ids)?;
+
+            for (i, (worker_slot_id, worker)) in worker_slot_ids
                 .into_iter()
                 .zip_eq_fast(workers.into_iter())
                 .enumerate()
@@ -366,11 +367,11 @@ impl StageRunner {
                 let task_id = TaskIdPb {
                     query_id: self.stage.query_id.id.clone(),
                     stage_id: self.stage.id,
-                    task_id: i as u32,
+                    task_id: i as u64,
                 };
-                let vnode_ranges = vnode_bitmaps[&parallel_unit_id].clone();
+                let vnode_ranges = vnode_bitmaps[&worker_slot_id].clone();
                 let plan_fragment =
-                    self.create_plan_fragment(i as u32, Some(PartitionInfo::Table(vnode_ranges)));
+                    self.create_plan_fragment(i as u64, Some(PartitionInfo::Table(vnode_ranges)));
                 futures.push(self.schedule_task(
                     task_id,
                     plan_fragment,
@@ -391,10 +392,10 @@ impl StageRunner {
                 let task_id = TaskIdPb {
                     query_id: self.stage.query_id.id.clone(),
                     stage_id: self.stage.id,
-                    task_id: id as u32,
+                    task_id: id as u64,
                 };
                 let plan_fragment = self
-                    .create_plan_fragment(id as u32, Some(PartitionInfo::Source(split.to_vec())));
+                    .create_plan_fragment(id as u64, Some(PartitionInfo::Source(split.to_vec())));
                 let worker =
                     self.choose_worker(&plan_fragment, id as u32, self.stage.dml_table_id)?;
                 futures.push(self.schedule_task(
@@ -409,9 +410,9 @@ impl StageRunner {
                 let task_id = TaskIdPb {
                     query_id: self.stage.query_id.id.clone(),
                     stage_id: self.stage.id,
-                    task_id: id,
+                    task_id: id as u64,
                 };
-                let plan_fragment = self.create_plan_fragment(id, None);
+                let plan_fragment = self.create_plan_fragment(id as u64, None);
                 let worker = self.choose_worker(&plan_fragment, id, self.stage.dml_table_id)?;
                 futures.push(self.schedule_task(
                     task_id,
@@ -682,7 +683,7 @@ impl StageRunner {
     fn get_table_dml_vnode_mapping(
         &self,
         table_id: &TableId,
-    ) -> SchedulerResult<ParallelUnitMapping> {
+    ) -> SchedulerResult<WorkerSlotMapping> {
         let guard = self.catalog_reader.read_guard();
 
         let table = guard
@@ -711,11 +712,11 @@ impl StageRunner {
 
         if let Some(table_id) = dml_table_id {
             let vnode_mapping = self.get_table_dml_vnode_mapping(&table_id)?;
-            let parallel_unit_ids = vnode_mapping.iter_unique().collect_vec();
+            let worker_ids = vnode_mapping.iter_unique().collect_vec();
             let candidates = self
                 .worker_node_manager
                 .manager
-                .get_workers_by_parallel_unit_ids(&parallel_unit_ids)?;
+                .get_workers_by_worker_slot_ids(&worker_ids)?;
             if candidates.is_empty() {
                 return Err(BatchError::EmptyWorkerNodes.into());
             }
@@ -741,17 +742,17 @@ impl StageRunner {
                     .table_id
                     .into(),
             )?;
-            let id2pu_vec = self
+            let id_to_worker_slots = self
                 .worker_node_manager
                 .fragment_mapping(fragment_id)?
                 .iter_unique()
                 .collect_vec();
 
-            let pu = id2pu_vec[task_id as usize];
+            let worker_slot_id = id_to_worker_slots[task_id as usize];
             let candidates = self
                 .worker_node_manager
                 .manager
-                .get_workers_by_parallel_unit_ids(&[pu])?;
+                .get_workers_by_worker_slot_ids(&[worker_slot_id])?;
             if candidates.is_empty() {
                 return Err(BatchError::EmptyWorkerNodes.into());
             }
@@ -987,6 +988,22 @@ impl StageRunner {
                     node_body: Some(NodeBody::RowSeqScan(scan_node)),
                 }
             }
+            PlanNodeType::BatchLogSeqScan => {
+                let node_body = execution_plan_node.node.clone();
+                let NodeBody::LogRowSeqScan(mut scan_node) = node_body else {
+                    unreachable!();
+                };
+                let partition = partition
+                    .expect("no partition info for seq scan")
+                    .into_table()
+                    .expect("PartitionInfo should be TablePartitionInfo");
+                scan_node.vnode_bitmap = Some(partition.vnode_bitmap);
+                PlanNodePb {
+                    children: vec![],
+                    identity,
+                    node_body: Some(NodeBody::LogRowSeqScan(scan_node)),
+                }
+            }
             PlanNodeType::BatchSource
             | PlanNodeType::BatchKafkaScan
             | PlanNodeType::BatchIcebergScan => {
diff --git a/src/frontend/src/scheduler/local.rs b/src/frontend/src/scheduler/local.rs
index 7877ab658b792..89104cc895f77 100644
--- a/src/frontend/src/scheduler/local.rs
+++ b/src/frontend/src/scheduler/local.rs
@@ -30,7 +30,7 @@ use risingwave_batch::task::{ShutdownToken, TaskId};
 use risingwave_batch::worker_manager::worker_node_manager::WorkerNodeSelector;
 use risingwave_common::array::DataChunk;
 use risingwave_common::bail;
-use risingwave_common::hash::ParallelUnitMapping;
+use risingwave_common::hash::WorkerSlotMapping;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::tracing::{InstrumentStream, TracingContext};
 use risingwave_connector::source::SplitMetaData;
@@ -55,6 +55,7 @@ use crate::scheduler::task_context::FrontendBatchTaskContext;
 use crate::scheduler::{ReadSnapshot, SchedulerError, SchedulerResult};
 use crate::session::{FrontendEnv, SessionImpl};
 
+// TODO(error-handling): use a concrete error type.
 pub type LocalQueryStream = ReceiverStream<Result<DataChunk, BoxedError>>;
 pub struct LocalQueryExecution {
     sql: String,
@@ -312,12 +313,12 @@ impl LocalQueryExecution {
                     // Similar to the distributed case (StageRunner::schedule_tasks).
                     // Set `vnode_ranges` of the scan node in `local_execute_plan` of each
                     // `exchange_source`.
-                    let (parallel_unit_ids, vnode_bitmaps): (Vec<_>, Vec<_>) =
+                    let (worker_ids, vnode_bitmaps): (Vec<_>, Vec<_>) =
                         vnode_bitmaps.clone().into_iter().unzip();
                     let workers = self
                         .worker_node_manager
                         .manager
-                        .get_workers_by_parallel_unit_ids(&parallel_unit_ids)?;
+                        .get_workers_by_worker_slot_ids(&worker_ids)?;
                     for (idx, (worker_node, partition)) in
                         (workers.into_iter().zip_eq_fast(vnode_bitmaps.into_iter())).enumerate()
                     {
@@ -342,7 +343,7 @@ impl LocalQueryExecution {
                         let exchange_source = ExchangeSource {
                             task_output_id: Some(TaskOutputId {
                                 task_id: Some(PbTaskId {
-                                    task_id: idx as u32,
+                                    task_id: idx as u64,
                                     stage_id: exchange_source_stage_id,
                                     query_id: self.query.query_id.id.clone(),
                                 }),
@@ -388,7 +389,7 @@ impl LocalQueryExecution {
                         let exchange_source = ExchangeSource {
                             task_output_id: Some(TaskOutputId {
                                 task_id: Some(PbTaskId {
-                                    task_id: id as u32,
+                                    task_id: id as u64,
                                     stage_id: exchange_source_stage_id,
                                     query_id: self.query.query_id.id.clone(),
                                 }),
@@ -428,7 +429,7 @@ impl LocalQueryExecution {
                             let exchange_source = ExchangeSource {
                                 task_output_id: Some(TaskOutputId {
                                     task_id: Some(PbTaskId {
-                                        task_id: idx as u32,
+                                        task_id: idx as u64,
                                         stage_id: exchange_source_stage_id,
                                         query_id: self.query.query_id.id.clone(),
                                     }),
@@ -472,6 +473,26 @@ impl LocalQueryExecution {
                     node_body: Some(node_body),
                 })
             }
+            PlanNodeType::BatchLogSeqScan => {
+                let mut node_body = execution_plan_node.node.clone();
+                match &mut node_body {
+                    NodeBody::LogRowSeqScan(ref mut scan_node) => {
+                        if let Some(partition) = partition {
+                            let partition = partition
+                                .into_table()
+                                .expect("PartitionInfo should be TablePartitionInfo here");
+                            scan_node.vnode_bitmap = Some(partition.vnode_bitmap);
+                        }
+                    }
+                    _ => unreachable!(),
+                }
+
+                Ok(PlanNodePb {
+                    children: vec![],
+                    identity,
+                    node_body: Some(node_body),
+                })
+            }
             PlanNodeType::BatchSource
             | PlanNodeType::BatchKafkaScan
             | PlanNodeType::BatchIcebergScan => {
@@ -510,7 +531,8 @@ impl LocalQueryExecution {
                         )?;
 
                         // TODO: should we use `pb::ParallelUnitMapping` here?
-                        node.inner_side_vnode_mapping = mapping.to_expanded();
+                        node.inner_side_vnode_mapping =
+                            mapping.to_expanded().into_iter().map(u64::from).collect();
                         node.worker_nodes = self.worker_node_manager.manager.list_worker_nodes();
                     }
                     _ => unreachable!(),
@@ -565,7 +587,7 @@ impl LocalQueryExecution {
     fn get_table_dml_vnode_mapping(
         &self,
         table_id: &TableId,
-    ) -> SchedulerResult<ParallelUnitMapping> {
+    ) -> SchedulerResult<WorkerSlotMapping> {
         let guard = self.front_env.catalog_reader().read_guard();
 
         let table = guard
@@ -589,11 +611,11 @@ impl LocalQueryExecution {
             // dml should use streaming vnode mapping
             let vnode_mapping = self.get_table_dml_vnode_mapping(table_id)?;
             let worker_node = {
-                let parallel_unit_ids = vnode_mapping.iter_unique().collect_vec();
+                let worker_ids = vnode_mapping.iter_unique().collect_vec();
                 let candidates = self
                     .worker_node_manager
                     .manager
-                    .get_workers_by_parallel_unit_ids(&parallel_unit_ids)?;
+                    .get_workers_by_worker_slot_ids(&worker_ids)?;
                 if candidates.is_empty() {
                     return Err(BatchError::EmptyWorkerNodes.into());
                 }
diff --git a/src/frontend/src/scheduler/plan_fragmenter.rs b/src/frontend/src/scheduler/plan_fragmenter.rs
index f5ed1b1205b69..74ade3fdab836 100644
--- a/src/frontend/src/scheduler/plan_fragmenter.rs
+++ b/src/frontend/src/scheduler/plan_fragmenter.rs
@@ -30,7 +30,7 @@ use risingwave_common::bail;
 use risingwave_common::buffer::{Bitmap, BitmapBuilder};
 use risingwave_common::catalog::TableDesc;
 use risingwave_common::hash::table_distribution::TableDistribution;
-use risingwave_common::hash::{ParallelUnitId, ParallelUnitMapping, VirtualNode};
+use risingwave_common::hash::{VirtualNode, WorkerSlotId, WorkerSlotMapping};
 use risingwave_common::util::scan_range::ScanRange;
 use risingwave_connector::source::filesystem::opendal_source::opendal_enumerator::OpendalEnumerator;
 use risingwave_connector::source::filesystem::opendal_source::{OpendalGcs, OpendalS3};
@@ -75,10 +75,10 @@ impl std::fmt::Display for QueryId {
 pub type StageId = u32;
 
 // Root stage always has only one task.
-pub const ROOT_TASK_ID: u32 = 0;
+pub const ROOT_TASK_ID: u64 = 0;
 // Root task has only one output.
-pub const ROOT_TASK_OUTPUT_ID: u32 = 0;
-pub type TaskId = u32;
+pub const ROOT_TASK_OUTPUT_ID: u64 = 0;
+pub type TaskId = u64;
 
 /// Generated by [`BatchPlanFragmenter`] and used in query execution graph.
 #[derive(Clone, Debug)]
@@ -392,12 +392,12 @@ pub struct TableScanInfo {
     /// full vnode bitmap, since we need to know where to schedule the singleton scan task.
     ///
     /// `None` iff the table is a system table.
-    partitions: Option<HashMap<ParallelUnitId, TablePartitionInfo>>,
+    partitions: Option<HashMap<WorkerSlotId, TablePartitionInfo>>,
 }
 
 impl TableScanInfo {
     /// For normal tables, `partitions` should always be `Some`.
-    pub fn new(name: String, partitions: HashMap<ParallelUnitId, TablePartitionInfo>) -> Self {
+    pub fn new(name: String, partitions: HashMap<WorkerSlotId, TablePartitionInfo>) -> Self {
         Self {
             name,
             partitions: Some(partitions),
@@ -416,7 +416,7 @@ impl TableScanInfo {
         self.name.as_ref()
     }
 
-    pub fn partitions(&self) -> Option<&HashMap<u32, TablePartitionInfo>> {
+    pub fn partitions(&self) -> Option<&HashMap<WorkerSlotId, TablePartitionInfo>> {
         self.partitions.as_ref()
     }
 }
@@ -1064,18 +1064,7 @@ impl BatchPlanFragmenter {
     /// If there are multiple scan nodes in this stage, they must have the same distribution, but
     /// maybe different vnodes partition. We just use the same partition for all the scan nodes.
     fn collect_stage_table_scan(&self, node: PlanRef) -> SchedulerResult<Option<TableScanInfo>> {
-        if node.node_type() == PlanNodeType::BatchExchange {
-            // Do not visit next stage.
-            return Ok(None);
-        }
-        if let Some(scan_node) = node.as_batch_sys_seq_scan() {
-            let name = scan_node.core().table_name.to_owned();
-            return Ok(Some(TableScanInfo::system_table(name)));
-        }
-
-        if let Some(scan_node) = node.as_batch_seq_scan() {
-            let name = scan_node.core().table_name.to_owned();
-            let table_desc = &*scan_node.core().table_desc;
+        let build_table_scan_info = |name, table_desc: &TableDesc, scan_range| {
             let table_catalog = self
                 .catalog_reader
                 .read_guard()
@@ -1085,10 +1074,29 @@ impl BatchPlanFragmenter {
             let vnode_mapping = self
                 .worker_node_manager
                 .fragment_mapping(table_catalog.fragment_id)?;
-            let partitions =
-                derive_partitions(scan_node.scan_ranges(), table_desc, &vnode_mapping)?;
+            let partitions = derive_partitions(scan_range, table_desc, &vnode_mapping)?;
             let info = TableScanInfo::new(name, partitions);
             Ok(Some(info))
+        };
+        if node.node_type() == PlanNodeType::BatchExchange {
+            // Do not visit next stage.
+            return Ok(None);
+        }
+        if let Some(scan_node) = node.as_batch_sys_seq_scan() {
+            let name = scan_node.core().table_name.to_owned();
+            Ok(Some(TableScanInfo::system_table(name)))
+        } else if let Some(scan_node) = node.as_batch_log_seq_scan() {
+            build_table_scan_info(
+                scan_node.core().table_name.to_owned(),
+                &scan_node.core().table_desc,
+                &[],
+            )
+        } else if let Some(scan_node) = node.as_batch_seq_scan() {
+            build_table_scan_info(
+                scan_node.core().table_name.to_owned(),
+                &scan_node.core().table_desc,
+                scan_node.scan_ranges(),
+            )
         } else {
             node.inputs()
                 .into_iter()
@@ -1150,10 +1158,10 @@ impl BatchPlanFragmenter {
 fn derive_partitions(
     scan_ranges: &[ScanRange],
     table_desc: &TableDesc,
-    vnode_mapping: &ParallelUnitMapping,
-) -> SchedulerResult<HashMap<ParallelUnitId, TablePartitionInfo>> {
+    vnode_mapping: &WorkerSlotMapping,
+) -> SchedulerResult<HashMap<WorkerSlotId, TablePartitionInfo>> {
     let num_vnodes = vnode_mapping.len();
-    let mut partitions: HashMap<ParallelUnitId, (BitmapBuilder, Vec<_>)> = HashMap::new();
+    let mut partitions: HashMap<WorkerSlotId, (BitmapBuilder, Vec<_>)> = HashMap::new();
 
     if scan_ranges.is_empty() {
         return Ok(vnode_mapping
@@ -1182,9 +1190,9 @@ fn derive_partitions(
             None => {
                 // put this scan_range to all partitions
                 vnode_mapping.to_bitmaps().into_iter().for_each(
-                    |(parallel_unit_id, vnode_bitmap)| {
+                    |(worker_slot_id, vnode_bitmap)| {
                         let (bitmap, scan_ranges) = partitions
-                            .entry(parallel_unit_id)
+                            .entry(worker_slot_id)
                             .or_insert_with(|| (BitmapBuilder::zeroed(num_vnodes), vec![]));
                         vnode_bitmap
                             .iter()
@@ -1196,9 +1204,9 @@ fn derive_partitions(
             }
             // scan a single partition
             Some(vnode) => {
-                let parallel_unit_id = vnode_mapping[vnode];
+                let worker_slot_id = vnode_mapping[vnode];
                 let (bitmap, scan_ranges) = partitions
-                    .entry(parallel_unit_id)
+                    .entry(worker_slot_id)
                     .or_insert_with(|| (BitmapBuilder::zeroed(num_vnodes), vec![]));
                 bitmap.set(vnode.to_index(), true);
                 scan_ranges.push(scan_range.to_protobuf());
diff --git a/src/frontend/src/scheduler/task_context.rs b/src/frontend/src/scheduler/task_context.rs
index 35f20a6daeffd..e8aafc440f289 100644
--- a/src/frontend/src/scheduler/task_context.rs
+++ b/src/frontend/src/scheduler/task_context.rs
@@ -94,14 +94,6 @@ impl BatchTaskContext for FrontendBatchTaskContext {
         self.session.env().source_metrics()
     }
 
-    fn store_mem_usage(&self, _val: usize) {
-        todo!()
-    }
-
-    fn mem_usage(&self) -> usize {
-        todo!()
-    }
-
     fn create_executor_mem_context(&self, _executor_id: &str) -> MemoryContext {
         MemoryContext::new(Some(self.mem_context.clone()), TrAdderAtomic::new(0))
     }
diff --git a/src/frontend/src/session.rs b/src/frontend/src/session.rs
index 7ff790748a761..7853f97bc9d4a 100644
--- a/src/frontend/src/session.rs
+++ b/src/frontend/src/session.rs
@@ -20,6 +20,7 @@ use std::sync::atomic::{AtomicI32, Ordering};
 use std::sync::{Arc, Weak};
 use std::time::{Duration, Instant};
 
+use anyhow::anyhow;
 use bytes::Bytes;
 use either::Either;
 use parking_lot::{Mutex, RwLock, RwLockReadGuard};
@@ -34,6 +35,7 @@ use pgwire::pg_server::{
 };
 use pgwire::types::{Format, FormatIterator};
 use rand::RngCore;
+use risingwave_batch::spill::spill_op::SpillOp;
 use risingwave_batch::task::{ShutdownSender, ShutdownToken};
 use risingwave_batch::worker_manager::worker_node_manager::{
     WorkerNodeManager, WorkerNodeManagerRef,
@@ -85,7 +87,7 @@ use crate::catalog::connection_catalog::ConnectionCatalog;
 use crate::catalog::root_catalog::Catalog;
 use crate::catalog::subscription_catalog::SubscriptionCatalog;
 use crate::catalog::{
-    check_schema_writable, CatalogError, DatabaseId, OwnedByUserCatalog, SchemaId,
+    check_schema_writable, CatalogError, DatabaseId, OwnedByUserCatalog, SchemaId, TableId,
 };
 use crate::error::{ErrorCode, Result, RwError};
 use crate::handler::describe::infer_describe;
@@ -111,7 +113,7 @@ use crate::user::user_authentication::md5_hash_with_salt;
 use crate::user::user_manager::UserInfoManager;
 use crate::user::user_service::{UserInfoReader, UserInfoWriter, UserInfoWriterImpl};
 use crate::user::UserId;
-use crate::{FrontendOpts, PgResponseStream};
+use crate::{FrontendOpts, PgResponseStream, TableCatalog};
 
 pub(crate) mod current;
 pub(crate) mod cursor_manager;
@@ -164,6 +166,9 @@ pub struct FrontendEnv {
 /// Session map identified by `(process_id, secret_key)`
 type SessionMapRef = Arc<RwLock<HashMap<(i32, i32), Arc<SessionImpl>>>>;
 
+/// The proportion of frontend memory used for batch processing.
+const FRONTEND_BATCH_MEMORY_PROPORTION: f64 = 0.5;
+
 impl FrontendEnv {
     pub fn mock() -> Self {
         use crate::test_utils::{MockCatalogWriter, MockFrontendMetaClient, MockUserInfoWriter};
@@ -338,10 +343,6 @@ impl FrontendEnv {
             MetricsManager::boot_metrics_service(opts.prometheus_listener_addr.clone());
         }
 
-        let mem_context = risingwave_common::memory::MemoryContext::root(
-            frontend_metrics.batch_total_mem.clone(),
-        );
-
         let health_srv = HealthServiceImpl::new();
         let host = opts.health_check_listener_addr.clone();
 
@@ -402,12 +403,23 @@ impl FrontendEnv {
         });
         join_handles.push(join_handle);
 
+        // Clean up the spill directory.
+        #[cfg(not(madsim))]
+        SpillOp::clean_spill_directory()
+            .await
+            .map_err(|err| anyhow!(err))?;
+
         let total_memory_bytes = resource_util::memory::system_memory_available_bytes();
         let heap_profiler =
             HeapProfiler::new(total_memory_bytes, config.server.heap_profiling.clone());
         // Run a background heap profiler
         heap_profiler.start();
 
+        let mem_context = risingwave_common::memory::MemoryContext::root(
+            frontend_metrics.batch_total_mem.clone(),
+            (total_memory_bytes as f64 * FRONTEND_BATCH_MEMORY_PROPORTION) as u64,
+        );
+
         Ok((
             Self {
                 catalog_reader,
@@ -803,6 +815,26 @@ impl SessionImpl {
         }
     }
 
+    pub fn check_secret_name_duplicated(&self, name: ObjectName) -> Result<()> {
+        let db_name = self.database();
+        let catalog_reader = self.env().catalog_reader().read_guard();
+        let (schema_name, secret_name) = {
+            let (schema_name, secret_name) = Binder::resolve_schema_qualified_name(db_name, name)?;
+            let search_path = self.config().search_path();
+            let user_name = &self.auth_context().user_name;
+            let schema_name = match schema_name {
+                Some(schema_name) => schema_name,
+                None => catalog_reader
+                    .first_valid_schema(db_name, &search_path, user_name)?
+                    .name(),
+            };
+            (schema_name, secret_name)
+        };
+        catalog_reader
+            .check_secret_name_duplicated(db_name, &schema_name, &secret_name)
+            .map_err(RwError::from)
+    }
+
     pub fn check_connection_name_duplicated(&self, name: ObjectName) -> Result<()> {
         let db_name = self.database();
         let catalog_reader = self.env().catalog_reader().read_guard();
@@ -905,6 +937,42 @@ impl SessionImpl {
         Ok(subscription.clone())
     }
 
+    pub fn get_table_by_id(&self, table_id: &TableId) -> Result<Arc<TableCatalog>> {
+        let catalog_reader = self.env().catalog_reader().read_guard();
+        Ok(catalog_reader.get_table_by_id(table_id)?.clone())
+    }
+
+    pub fn get_table_by_name(
+        &self,
+        table_name: &str,
+        db_id: u32,
+        schema_id: u32,
+    ) -> Result<Arc<TableCatalog>> {
+        let catalog_reader = self.env().catalog_reader().read_guard();
+        let table = catalog_reader
+            .get_schema_by_id(&DatabaseId::from(db_id), &SchemaId::from(schema_id))?
+            .get_table_by_name(table_name)
+            .ok_or_else(|| {
+                Error::new(
+                    ErrorKind::InvalidInput,
+                    format!("table \"{}\" does not exist", table_name),
+                )
+            })?;
+        Ok(table.clone())
+    }
+
+    pub async fn list_change_log_epochs(
+        &self,
+        table_id: u32,
+        min_epoch: u64,
+        max_count: u32,
+    ) -> Result<Vec<u64>> {
+        self.env
+            .catalog_writer
+            .list_change_log_epochs(table_id, min_epoch, max_count)
+            .await
+    }
+
     pub fn clear_cancel_query_flag(&self) {
         let mut flag = self.current_query_cancel_flag.lock();
         *flag = None;
@@ -1116,6 +1184,10 @@ impl SessionManagerImpl {
         })
     }
 
+    pub fn env(&self) -> &FrontendEnv {
+        &self.env
+    }
+
     fn insert_session(&self, session: Arc<SessionImpl>) {
         let active_sessions = {
             let mut write_guard = self.env.sessions_map.write();
diff --git a/src/frontend/src/session/cursor_manager.rs b/src/frontend/src/session/cursor_manager.rs
index 214dac1a2b5a2..46eca3beb9966 100644
--- a/src/frontend/src/session/cursor_manager.rs
+++ b/src/frontend/src/session/cursor_manager.rs
@@ -12,34 +12,34 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use core::ops::Index;
 use core::time::Duration;
 use std::collections::{HashMap, VecDeque};
+use std::rc::Rc;
 use std::sync::Arc;
 use std::time::Instant;
 
 use bytes::Bytes;
+use fixedbitset::FixedBitSet;
 use futures::StreamExt;
-use itertools::Itertools;
 use pgwire::pg_field_descriptor::PgFieldDescriptor;
+use pgwire::pg_response::StatementType;
 use pgwire::types::Row;
+use risingwave_common::session_config::QueryMode;
 use risingwave_common::types::DataType;
 use risingwave_sqlparser::ast::{Ident, ObjectName, Statement};
 
+use super::SessionImpl;
 use crate::catalog::subscription_catalog::SubscriptionCatalog;
-use crate::error::{ErrorCode, Result, RwError};
-use crate::handler::declare_cursor::create_stream_for_cursor;
-use crate::handler::util::{
-    convert_epoch_to_logstore_i64, convert_logstore_i64_to_unix_millis,
-    gen_query_from_logstore_ge_rw_timestamp, gen_query_from_table_name,
-};
+use crate::catalog::TableId;
+use crate::error::{ErrorCode, Result};
+use crate::handler::declare_cursor::create_stream_for_cursor_stmt;
+use crate::handler::query::{create_stream, gen_batch_plan_fragmenter, BatchQueryPlanResult};
+use crate::handler::util::{convert_logstore_u64_to_unix_millis, gen_query_from_table_name};
 use crate::handler::HandlerArgs;
-use crate::PgResponseStream;
-
-pub const KV_LOG_STORE_EPOCH: &str = "kv_log_store_epoch";
-const KV_LOG_STORE_ROW_OP: &str = "kv_log_store_row_op";
-pub const KV_LOG_STORE_SEQ_ID: &str = "kv_log_store_seq_id";
-pub const KV_LOG_STORE_VNODE: &str = "kv_log_store_vnode";
+use crate::optimizer::plan_node::{generic, BatchLogSeqScan};
+use crate::optimizer::property::{Order, RequiredDist};
+use crate::optimizer::PlanRoot;
+use crate::{OptimizerContext, OptimizerContextRef, PgResponseStream, PlanRef, TableCatalog};
 
 pub enum Cursor {
     Subscription(SubscriptionCursor),
@@ -78,9 +78,7 @@ impl QueryCursor {
             let rows = self.row_stream.next().await;
             let rows = match rows {
                 None => return Ok(None),
-                Some(row) => {
-                    row.map_err(|err| RwError::from(ErrorCode::InternalError(format!("{}", err))))?
-                }
+                Some(row) => row?,
             };
             self.remaining_rows = rows.into_iter().collect();
         }
@@ -106,10 +104,10 @@ impl QueryCursor {
 enum State {
     InitLogStoreQuery {
         // The rw_timestamp used to initiate the query to read from subscription logstore.
-        seek_timestamp: i64,
+        seek_timestamp: u64,
 
         // If specified, the expected_timestamp must be an exact match for the next rw_timestamp.
-        expected_timestamp: Option<i64>,
+        expected_timestamp: Option<u64>,
     },
     Fetch {
         // Whether the query is reading from snapshot
@@ -118,7 +116,7 @@ enum State {
         from_snapshot: bool,
 
         // The rw_timestamp used to initiate the query to read from subscription logstore.
-        rw_timestamp: i64,
+        rw_timestamp: u64,
 
         // The row stream to from the batch query read.
         // It is returned from the batch execution.
@@ -130,6 +128,8 @@ enum State {
 
         // A cache to store the remaining rows from the row stream.
         remaining_rows: VecDeque<Row>,
+
+        expected_timestamp: Option<u64>,
     },
     Invalid,
 }
@@ -137,6 +137,7 @@ enum State {
 pub struct SubscriptionCursor {
     cursor_name: String,
     subscription: Arc<SubscriptionCatalog>,
+    dependent_table_id: TableId,
     cursor_need_drop_time: Instant,
     state: State,
 }
@@ -144,8 +145,9 @@ pub struct SubscriptionCursor {
 impl SubscriptionCursor {
     pub async fn new(
         cursor_name: String,
-        start_timestamp: Option<i64>,
+        start_timestamp: Option<u64>,
         subscription: Arc<SubscriptionCatalog>,
+        dependent_table_id: TableId,
         handle_args: &HandlerArgs,
     ) -> Result<Self> {
         let state = if let Some(start_timestamp) = start_timestamp {
@@ -159,7 +161,7 @@ impl SubscriptionCursor {
             //
             // TODO: is this the right behavior? Should we delay the query stream initiation till the first fetch?
             let (row_stream, pg_descs) =
-                Self::initiate_query(None, &subscription, handle_args.clone()).await?;
+                Self::initiate_query(None, &dependent_table_id, handle_args.clone()).await?;
             let pinned_epoch = handle_args
                 .session
                 .get_pinned_snapshot()
@@ -173,7 +175,7 @@ impl SubscriptionCursor {
                     )
                 })?
                 .0;
-            let start_timestamp = convert_epoch_to_logstore_i64(pinned_epoch);
+            let start_timestamp = pinned_epoch;
 
             State::Fetch {
                 from_snapshot: true,
@@ -181,22 +183,25 @@ impl SubscriptionCursor {
                 row_stream,
                 pg_descs,
                 remaining_rows: VecDeque::new(),
+                expected_timestamp: None,
             }
         };
 
         let cursor_need_drop_time =
-            Instant::now() + Duration::from_secs(subscription.get_retention_seconds()?);
+            Instant::now() + Duration::from_secs(subscription.retention_seconds);
         Ok(Self {
             cursor_name,
             subscription,
+            dependent_table_id,
             cursor_need_drop_time,
             state,
         })
     }
 
-    pub async fn next_row(
+    async fn next_row(
         &mut self,
-        handle_args: HandlerArgs,
+        handle_args: &HandlerArgs,
+        expected_pg_descs: &Vec<PgFieldDescriptor>,
     ) -> Result<(Option<Row>, Vec<PgFieldDescriptor>)> {
         loop {
             match &mut self.state {
@@ -207,64 +212,47 @@ impl SubscriptionCursor {
                     let from_snapshot = false;
 
                     // Initiate a new batch query to continue fetching
-                    let (mut row_stream, pg_descs) = Self::initiate_query(
-                        Some(*seek_timestamp),
-                        &self.subscription,
+                    match Self::get_next_rw_timestamp(
+                        *seek_timestamp,
+                        self.dependent_table_id.table_id,
+                        *expected_timestamp,
                         handle_args.clone(),
                     )
-                    .await?;
-                    self.cursor_need_drop_time = Instant::now()
-                        + Duration::from_secs(self.subscription.get_retention_seconds()?);
-
-                    // Try refill remaining rows
-                    let mut remaining_rows = VecDeque::new();
-                    Self::try_refill_remaining_rows(&mut row_stream, &mut remaining_rows).await?;
-
-                    // Get the rw_timestamp in the first row returned by the query if any.
-                    // new_row_rw_timestamp == None means the query returns empty result.
-                    let new_row_rw_timestamp: Option<i64> = remaining_rows.front().map(|row| {
-                        std::str::from_utf8(row.index(0).as_ref().unwrap())
-                            .unwrap()
-                            .parse()
-                            .unwrap()
-                    });
-
-                    // Check expected_timestamp against the rw_timestamp of the first row.
-                    // Return an error if there is no new row or there is a mismatch.
-                    if let Some(expected_timestamp) = expected_timestamp {
-                        let expected_timestamp = *expected_timestamp;
-                        if new_row_rw_timestamp.is_none()
-                            || new_row_rw_timestamp.unwrap() != expected_timestamp
-                        {
-                            // Transition to Invalid state and return and error
-                            self.state = State::Invalid;
-                            return Err(ErrorCode::CatalogError(
-                                format!(
-                                    " No data found for rw_timestamp {:?}, data may have been recycled, please recreate cursor",
-                                    convert_logstore_i64_to_unix_millis(expected_timestamp)
-                                )
-                                .into(),
+                    .await
+                    {
+                        Ok((Some(rw_timestamp), expected_timestamp)) => {
+                            let (mut row_stream, pg_descs) = Self::initiate_query(
+                                Some(rw_timestamp),
+                                &self.dependent_table_id,
+                                handle_args.clone(),
                             )
-                            .into());
+                            .await?;
+                            self.cursor_need_drop_time = Instant::now()
+                                + Duration::from_secs(self.subscription.retention_seconds);
+                            let mut remaining_rows = VecDeque::new();
+                            Self::try_refill_remaining_rows(&mut row_stream, &mut remaining_rows)
+                                .await?;
+                            // Transition to the Fetch state
+                            self.state = State::Fetch {
+                                from_snapshot,
+                                rw_timestamp,
+                                row_stream,
+                                pg_descs: pg_descs.clone(),
+                                remaining_rows,
+                                expected_timestamp,
+                            };
+                            if (!expected_pg_descs.is_empty()) && expected_pg_descs.ne(&pg_descs) {
+                                // If the user alters the table upstream of the sub, there will be different descs here.
+                                // So we should output data for different descs in two separate batches
+                                return Ok((None, vec![]));
+                            }
+                        }
+                        Ok((None, _)) => return Ok((None, vec![])),
+                        Err(e) => {
+                            self.state = State::Invalid;
+                            return Err(e);
                         }
                     }
-
-                    // Return None if no data is found for the rw_timestamp in logstore.
-                    // This happens when reaching EOF of logstore. This check cannot be moved before the
-                    // expected_timestamp check to ensure that an error is returned on empty result when
-                    // expected_timstamp is set.
-                    if new_row_rw_timestamp.is_none() {
-                        return Ok((None, pg_descs));
-                    }
-
-                    // Transition to the Fetch state
-                    self.state = State::Fetch {
-                        from_snapshot,
-                        rw_timestamp: new_row_rw_timestamp.unwrap(),
-                        row_stream,
-                        pg_descs,
-                        remaining_rows,
-                    };
                 }
                 State::Fetch {
                     from_snapshot,
@@ -272,6 +260,7 @@ impl SubscriptionCursor {
                     row_stream,
                     pg_descs,
                     remaining_rows,
+                    expected_timestamp,
                 } => {
                     let from_snapshot = *from_snapshot;
                     let rw_timestamp = *rw_timestamp;
@@ -283,46 +272,29 @@ impl SubscriptionCursor {
                         // 1. Fetch the next row
                         let new_row = row.take();
                         if from_snapshot {
-                            // 1a. The rw_timestamp in the table is all the same, so don't need to check.
                             return Ok((
-                                Some(Row::new(Self::build_row_with_snapshot(new_row))),
+                                Some(Row::new(Self::build_row(new_row, None)?)),
                                 pg_descs.clone(),
                             ));
-                        }
-
-                        let new_row_rw_timestamp: i64 = new_row
-                            .get(0)
-                            .unwrap()
-                            .as_ref()
-                            .map(|bytes| std::str::from_utf8(bytes).unwrap().parse().unwrap())
-                            .unwrap();
-
-                        if new_row_rw_timestamp != rw_timestamp {
-                            // 1b. Find the next rw_timestamp.
-                            // Initiate a new batch query to avoid query timeout and pinning version for too long.
-                            // expected_timestamp shouold be set to ensure there is no data missing in the next query.
-                            self.state = State::InitLogStoreQuery {
-                                seek_timestamp: new_row_rw_timestamp,
-                                expected_timestamp: Some(new_row_rw_timestamp),
-                            };
                         } else {
-                            // 1c. The rw_timestamp of this row is equal to self.rw_timestamp, return row
                             return Ok((
-                                Some(Row::new(Self::build_row_with_logstore(
-                                    new_row,
-                                    rw_timestamp,
-                                )?)),
+                                Some(Row::new(Self::build_row(new_row, Some(rw_timestamp))?)),
                                 pg_descs.clone(),
                             ));
                         }
                     } else {
                         // 2. Reach EOF for the current query.
-                        // Initiate a new batch query using the rw_timestamp + 1.
-                        // expected_timestamp don't need to be set as the next rw_timestamp is unknown.
-                        self.state = State::InitLogStoreQuery {
-                            seek_timestamp: rw_timestamp + 1,
-                            expected_timestamp: None,
-                        };
+                        if let Some(expected_timestamp) = expected_timestamp {
+                            self.state = State::InitLogStoreQuery {
+                                seek_timestamp: *expected_timestamp,
+                                expected_timestamp: Some(*expected_timestamp),
+                            };
+                        } else {
+                            self.state = State::InitLogStoreQuery {
+                                seek_timestamp: rw_timestamp + 1,
+                                expected_timestamp: None,
+                            };
+                        }
                     }
                 }
                 State::Invalid => {
@@ -348,41 +320,82 @@ impl SubscriptionCursor {
             )
             .into());
         }
-        // `FETCH NEXT` is equivalent to `FETCH 1`.
-        if count != 1 {
-            Err(crate::error::ErrorCode::InternalError(
-                "FETCH count with subscription is not supported".to_string(),
-            )
-            .into())
-        } else {
-            let (row, pg_descs) = self.next_row(handle_args).await?;
-            if let Some(row) = row {
-                Ok((vec![row], pg_descs))
-            } else {
-                Ok((vec![], pg_descs))
+
+        let mut ans = Vec::with_capacity(std::cmp::min(100, count) as usize);
+        let mut cur = 0;
+        let mut pg_descs_ans = vec![];
+        while cur < count {
+            let (row, descs_ans) = self.next_row(&handle_args, &pg_descs_ans).await?;
+            match row {
+                Some(row) => {
+                    pg_descs_ans = descs_ans;
+                    cur += 1;
+                    ans.push(row);
+                }
+                None => {
+                    break;
+                }
             }
         }
+
+        Ok((ans, pg_descs_ans))
+    }
+
+    async fn get_next_rw_timestamp(
+        seek_timestamp: u64,
+        table_id: u32,
+        expected_timestamp: Option<u64>,
+        handle_args: HandlerArgs,
+    ) -> Result<(Option<u64>, Option<u64>)> {
+        // The epoch here must be pulled every time, otherwise there will be cache consistency issues
+        let new_epochs = handle_args
+            .session
+            .catalog_writer()?
+            .list_change_log_epochs(table_id, seek_timestamp, 2)
+            .await?;
+        if let Some(expected_timestamp) = expected_timestamp
+            && (new_epochs.is_empty() || &expected_timestamp != new_epochs.first().unwrap())
+        {
+            return Err(ErrorCode::CatalogError(
+                format!(
+                    " No data found for rw_timestamp {:?}, data may have been recycled, please recreate cursor",
+                    convert_logstore_u64_to_unix_millis(expected_timestamp)
+                )
+                .into(),
+            )
+            .into());
+        }
+        Ok((new_epochs.get(0).cloned(), new_epochs.get(1).cloned()))
     }
 
     async fn initiate_query(
-        rw_timestamp: Option<i64>,
-        subscription: &SubscriptionCatalog,
+        rw_timestamp: Option<u64>,
+        dependent_table_id: &TableId,
         handle_args: HandlerArgs,
     ) -> Result<(PgResponseStream, Vec<PgFieldDescriptor>)> {
-        let query_stmt = if let Some(rw_timestamp) = rw_timestamp {
-            Statement::Query(Box::new(gen_query_from_logstore_ge_rw_timestamp(
-                &subscription.get_log_store_name(),
-                rw_timestamp,
-            )))
+        let session = handle_args.clone().session;
+        let table_catalog = session.get_table_by_id(dependent_table_id)?;
+        let (row_stream, pg_descs) = if let Some(rw_timestamp) = rw_timestamp {
+            let context = OptimizerContext::from_handler_args(handle_args);
+            let plan_fragmenter_result = gen_batch_plan_fragmenter(
+                &session,
+                Self::create_batch_plan_for_cursor(
+                    &table_catalog,
+                    &session,
+                    context.into(),
+                    rw_timestamp,
+                    rw_timestamp,
+                )?,
+            )?;
+            create_stream(session, plan_fragmenter_result, vec![]).await?
         } else {
-            let subscription_from_table_name = ObjectName(vec![Ident::from(
-                subscription.subscription_from_name.as_ref(),
-            )]);
-            Statement::Query(Box::new(gen_query_from_table_name(
+            let subscription_from_table_name =
+                ObjectName(vec![Ident::from(table_catalog.name.as_ref())]);
+            let query_stmt = Statement::Query(Box::new(gen_query_from_table_name(
                 subscription_from_table_name,
-            )))
+            )));
+            create_stream_for_cursor_stmt(handle_args, query_stmt).await?
         };
-        let (row_stream, pg_descs) = create_stream_for_cursor(handle_args, query_stmt).await?;
         Ok((
             row_stream,
             Self::build_desc(pg_descs, rw_timestamp.is_none()),
@@ -396,66 +409,97 @@ impl SubscriptionCursor {
         if remaining_rows.is_empty()
             && let Some(row_set) = row_stream.next().await
         {
-            remaining_rows.extend(row_set.map_err(|e| {
-                ErrorCode::InternalError(format!("Cursor get next chunk error {:?}", e.to_string()))
-            })?);
+            remaining_rows.extend(row_set?);
         }
         Ok(())
     }
 
-    pub fn build_row_with_snapshot(row: Vec<Option<Bytes>>) -> Vec<Option<Bytes>> {
-        let mut new_row = vec![None, Some(Bytes::from(1i16.to_string()))];
-        new_row.extend(row);
-        new_row
-    }
-
-    pub fn build_row_with_logstore(
+    pub fn build_row(
         mut row: Vec<Option<Bytes>>,
-        rw_timestamp: i64,
+        rw_timestamp: Option<u64>,
     ) -> Result<Vec<Option<Bytes>>> {
-        let mut new_row = vec![Some(Bytes::from(
-            convert_logstore_i64_to_unix_millis(rw_timestamp).to_string(),
-        ))];
-        // need remove kv_log_store_epoch
-        new_row.extend(row.drain(1..row.len()).collect_vec());
-        Ok(new_row)
+        let new_row = if let Some(rw_timestamp) = rw_timestamp {
+            vec![Some(Bytes::from(
+                convert_logstore_u64_to_unix_millis(rw_timestamp).to_string(),
+            ))]
+        } else {
+            vec![Some(Bytes::from(1i16.to_string())), None]
+        };
+        row.extend(new_row);
+        Ok(row)
     }
 
     pub fn build_desc(
         mut descs: Vec<PgFieldDescriptor>,
         from_snapshot: bool,
     ) -> Vec<PgFieldDescriptor> {
-        let mut new_descs = vec![
-            PgFieldDescriptor::new(
-                "rw_timestamp".to_owned(),
-                DataType::Int64.to_oid(),
-                DataType::Int64.type_len(),
-            ),
-            PgFieldDescriptor::new(
+        if from_snapshot {
+            descs.push(PgFieldDescriptor::new(
                 "op".to_owned(),
                 DataType::Int16.to_oid(),
                 DataType::Int16.type_len(),
-            ),
-        ];
-        // need remove kv_log_store_epoch and kv_log_store_row_op
-        if from_snapshot {
-            new_descs.extend(descs)
-        } else {
-            assert_eq!(
-                descs.get(0).unwrap().get_name(),
-                KV_LOG_STORE_EPOCH,
-                "Cursor query logstore: first column must be {}",
-                KV_LOG_STORE_EPOCH
-            );
-            assert_eq!(
-                descs.get(1).unwrap().get_name(),
-                KV_LOG_STORE_ROW_OP,
-                "Cursor query logstore: first column must be {}",
-                KV_LOG_STORE_ROW_OP
-            );
-            new_descs.extend(descs.drain(2..descs.len()));
+            ));
         }
-        new_descs
+        descs.push(PgFieldDescriptor::new(
+            "rw_timestamp".to_owned(),
+            DataType::Int64.to_oid(),
+            DataType::Int64.type_len(),
+        ));
+        descs
+    }
+
+    pub fn create_batch_plan_for_cursor(
+        table_catalog: &TableCatalog,
+        session: &SessionImpl,
+        context: OptimizerContextRef,
+        old_epoch: u64,
+        new_epoch: u64,
+    ) -> Result<BatchQueryPlanResult> {
+        let out_col_idx = table_catalog
+            .columns
+            .iter()
+            .enumerate()
+            .filter(|(_, v)| !v.is_hidden)
+            .map(|(i, _)| i)
+            .collect::<Vec<_>>();
+        let core = generic::LogScan::new(
+            table_catalog.name.clone(),
+            out_col_idx,
+            Rc::new(table_catalog.table_desc()),
+            context,
+            old_epoch,
+            new_epoch,
+        );
+        let batch_log_seq_scan = BatchLogSeqScan::new(core);
+        let schema = batch_log_seq_scan
+            .core()
+            .schema_without_table_name()
+            .clone();
+        let out_fields = FixedBitSet::from_iter(0..schema.len());
+        let out_names = batch_log_seq_scan.core().column_names();
+        // Here we just need a plan_root to call the method, only out_fields and out_names will be used
+        let plan_root = PlanRoot::new_with_batch_plan(
+            PlanRef::from(batch_log_seq_scan.clone()),
+            RequiredDist::single(),
+            Order::default(),
+            out_fields,
+            out_names,
+        );
+        let (batch_log_seq_scan, query_mode) = match session.config().query_mode() {
+            QueryMode::Auto => (plan_root.gen_batch_local_plan()?, QueryMode::Local),
+            QueryMode::Local => (plan_root.gen_batch_local_plan()?, QueryMode::Local),
+            QueryMode::Distributed => (
+                plan_root.gen_batch_distributed_plan()?,
+                QueryMode::Distributed,
+            ),
+        };
+        Ok(BatchQueryPlanResult {
+            plan: batch_log_seq_scan,
+            query_mode,
+            schema,
+            stmt_type: StatementType::SELECT,
+            dependent_relations: table_catalog.dependent_relations.clone(),
+        })
     }
 }
 
@@ -468,7 +512,8 @@ impl CursorManager {
     pub async fn add_subscription_cursor(
         &self,
         cursor_name: String,
-        start_timestamp: Option<i64>,
+        start_timestamp: Option<u64>,
+        dependent_table_id: TableId,
         subscription: Arc<SubscriptionCatalog>,
         handle_args: &HandlerArgs,
     ) -> Result<()> {
@@ -476,6 +521,7 @@ impl CursorManager {
             cursor_name.clone(),
             start_timestamp,
             subscription,
+            dependent_table_id,
             handle_args,
         )
         .await?;
diff --git a/src/frontend/src/stream_fragmenter/mod.rs b/src/frontend/src/stream_fragmenter/mod.rs
index e7548ce5fa176..f2d768cc076f4 100644
--- a/src/frontend/src/stream_fragmenter/mod.rs
+++ b/src/frontend/src/stream_fragmenter/mod.rs
@@ -285,10 +285,6 @@ fn build_fragment(
                 current_fragment.fragment_type_mask |= FragmentTypeFlag::Sink as u32
             }
 
-            NodeBody::Subscription(_) => {
-                current_fragment.fragment_type_mask |= FragmentTypeFlag::Subscription as u32
-            }
-
             NodeBody::TopN(_) => current_fragment.requires_singleton = true,
 
             NodeBody::StreamScan(node) => {
diff --git a/src/frontend/src/telemetry.rs b/src/frontend/src/telemetry.rs
index 631914cee7a4f..d50b9999b946f 100644
--- a/src/frontend/src/telemetry.rs
+++ b/src/frontend/src/telemetry.rs
@@ -33,6 +33,7 @@ impl FrontendTelemetryCreator {
 
 #[async_trait::async_trait]
 impl TelemetryReportCreator for FrontendTelemetryCreator {
+    #[allow(refining_impl_trait)]
     async fn create_report(
         &self,
         tracking_id: String,
@@ -97,7 +98,6 @@ mod tests {
         assert_eq!(report.base.node_type, TelemetryNodeType::Frontend);
     }
 
-    use risingwave_common::telemetry::pb_compatible::TelemetryToProtobuf;
     use risingwave_common::telemetry::{post_telemetry_report_pb, TELEMETRY_REPORT_URL};
 
     // It is ok to
diff --git a/src/frontend/src/test_utils.rs b/src/frontend/src/test_utils.rs
index bda6683797251..97ab9f4eecdc9 100644
--- a/src/frontend/src/test_utils.rs
+++ b/src/frontend/src/test_utils.rs
@@ -55,7 +55,7 @@ use risingwave_pb::meta::list_fragment_distribution_response::FragmentDistributi
 use risingwave_pb::meta::list_object_dependencies_response::PbObjectDependencies;
 use risingwave_pb::meta::list_table_fragment_states_response::TableFragmentState;
 use risingwave_pb::meta::list_table_fragments_response::TableFragmentInfo;
-use risingwave_pb::meta::{EventLog, PbTableParallelism, SystemParams};
+use risingwave_pb::meta::{EventLog, PbTableParallelism, PbThrottleTarget, SystemParams};
 use risingwave_pb::stream_plan::StreamFragmentGraph;
 use risingwave_pb::user::update_user_request::UpdateField;
 use risingwave_pb::user::{GrantPrivilege, UserInfo};
@@ -64,7 +64,7 @@ use tempfile::{Builder, NamedTempFile};
 
 use crate::catalog::catalog_service::CatalogWriter;
 use crate::catalog::root_catalog::Catalog;
-use crate::catalog::{ConnectionId, DatabaseId, SchemaId};
+use crate::catalog::{ConnectionId, DatabaseId, SchemaId, SecretId};
 use crate::error::{ErrorCode, Result};
 use crate::handler::RwPgResponse;
 use crate::meta_client::FrontendMetaClient;
@@ -242,6 +242,15 @@ impl CatalogWriter for MockCatalogWriter {
         Ok(())
     }
 
+    async fn list_change_log_epochs(
+        &self,
+        _table_id: u32,
+        _min_epoch: u64,
+        _max_count: u32,
+    ) -> Result<Vec<u64>> {
+        unreachable!()
+    }
+
     async fn create_schema(
         &self,
         db_id: DatabaseId,
@@ -325,12 +334,8 @@ impl CatalogWriter for MockCatalogWriter {
         self.create_sink_inner(sink, graph)
     }
 
-    async fn create_subscription(
-        &self,
-        subscription: PbSubscription,
-        graph: StreamFragmentGraph,
-    ) -> Result<()> {
-        self.create_subscription_inner(subscription, graph)
+    async fn create_subscription(&self, subscription: PbSubscription) -> Result<()> {
+        self.create_subscription_inner(subscription)
     }
 
     async fn create_index(
@@ -368,6 +373,17 @@ impl CatalogWriter for MockCatalogWriter {
         unreachable!()
     }
 
+    async fn create_secret(
+        &self,
+        _secret_name: String,
+        _database_id: u32,
+        _schema_id: u32,
+        _owner_id: u32,
+        _payload: Vec<u8>,
+    ) -> Result<()> {
+        unreachable!()
+    }
+
     async fn comment_on(&self, _comment: PbComment) -> Result<()> {
         unreachable!()
     }
@@ -525,6 +541,10 @@ impl CatalogWriter for MockCatalogWriter {
         unreachable!()
     }
 
+    async fn drop_secret(&self, _secret_id: SecretId) -> Result<()> {
+        unreachable!()
+    }
+
     async fn drop_database(&self, database_id: u32) -> Result<()> {
         self.catalog.write().drop_database(database_id);
         Ok(())
@@ -773,11 +793,7 @@ impl MockCatalogWriter {
         Ok(())
     }
 
-    fn create_subscription_inner(
-        &self,
-        mut subscription: PbSubscription,
-        _graph: StreamFragmentGraph,
-    ) -> Result<()> {
+    fn create_subscription_inner(&self, mut subscription: PbSubscription) -> Result<()> {
         subscription.id = self.gen_id();
         self.catalog.write().create_subscription(&subscription);
         self.add_table_or_subscription_id(
@@ -1059,6 +1075,15 @@ impl FrontendMetaClient for MockFrontendMetaClient {
     async fn recover(&self) -> RpcResult<()> {
         unimplemented!()
     }
+
+    async fn apply_throttle(
+        &self,
+        _kind: PbThrottleTarget,
+        _id: u32,
+        _rate_limit: Option<u32>,
+    ) -> RpcResult<()> {
+        unimplemented!()
+    }
 }
 
 #[cfg(test)]
diff --git a/src/frontend/src/user/user_catalog.rs b/src/frontend/src/user/user_catalog.rs
index a33e9423fc6ea..905348d37dfa6 100644
--- a/src/frontend/src/user/user_catalog.rs
+++ b/src/frontend/src/user/user_catalog.rs
@@ -16,7 +16,7 @@ use std::collections::hash_map::Entry;
 use std::collections::HashMap;
 
 use risingwave_common::acl::{AclMode, AclModeSet};
-use risingwave_pb::user::grant_privilege::{Object as GrantObject, Object};
+use risingwave_pb::user::grant_privilege::{Action, Object as GrantObject, Object};
 use risingwave_pb::user::{PbAuthInfo, PbGrantPrivilege, PbUserInfo};
 
 use crate::catalog::{DatabaseId, SchemaId};
@@ -168,4 +168,33 @@ impl UserCatalog {
         self.get_acl(object)
             .map_or(false, |acl_set| acl_set.has_mode(mode))
     }
+
+    pub fn check_privilege_with_grant_option(
+        &self,
+        object: &GrantObject,
+        actions: &Vec<(Action, bool)>,
+    ) -> bool {
+        if self.is_super {
+            return true;
+        }
+        let mut action_map: HashMap<_, _> = actions.iter().map(|action| (action, false)).collect();
+
+        for privilege in &self.grant_privileges {
+            if privilege.get_object().unwrap() != object {
+                continue;
+            }
+            for awo in &privilege.action_with_opts {
+                let action = awo.get_action().unwrap();
+                let with_grant_option = awo.with_grant_option;
+
+                for (&key, found) in &mut action_map {
+                    let (required_action, required_grant_option) = *key;
+                    if action == required_action && (!required_grant_option | with_grant_option) {
+                        *found = true;
+                    }
+                }
+            }
+        }
+        action_map.values().all(|&found| found)
+    }
 }
diff --git a/src/frontend/src/user/user_privilege.rs b/src/frontend/src/user/user_privilege.rs
index 2b474bda0554e..d4ad71e64120d 100644
--- a/src/frontend/src/user/user_privilege.rs
+++ b/src/frontend/src/user/user_privilege.rs
@@ -94,7 +94,7 @@ pub fn available_prost_privilege(object: PbObject, for_dml_table: bool) -> PbGra
                 &acl::ALL_AVAILABLE_MVIEW_MODES
             }
         }
-        PbObject::ViewId(_) => &acl::ALL_AVAILABLE_VIEW_MODES,
+        PbObject::ViewId(_) => &acl::ALL_AVAILABLE_TABLE_MODES,
         PbObject::SinkId(_) => &acl::ALL_AVAILABLE_SINK_MODES,
         PbObject::SubscriptionId(_) => &acl::ALL_AVAILABLE_SUBSCRIPTION_MODES,
         PbObject::FunctionId(_) => &acl::ALL_AVAILABLE_FUNCTION_MODES,
diff --git a/src/frontend/src/utils/condition.rs b/src/frontend/src/utils/condition.rs
index 5161877d1d163..d78adcbd07a97 100644
--- a/src/frontend/src/utils/condition.rs
+++ b/src/frontend/src/utils/condition.rs
@@ -1018,10 +1018,8 @@ mod cast_compare {
 #[cfg(test)]
 mod tests {
     use rand::Rng;
-    use risingwave_common::types::DataType;
 
     use super::*;
-    use crate::expr::{FunctionCall, InputRef};
 
     #[test]
     fn test_split() {
diff --git a/src/frontend/src/utils/with_options.rs b/src/frontend/src/utils/with_options.rs
index dc3e66bdfbac2..01993fd24108a 100644
--- a/src/frontend/src/utils/with_options.rs
+++ b/src/frontend/src/utils/with_options.rs
@@ -12,11 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashMap};
-use std::convert::TryFrom;
+use std::collections::BTreeMap;
 use std::num::NonZeroU32;
 
-use risingwave_connector::source::kafka::{
+use risingwave_connector::source::kafka::private_link::{
     insert_privatelink_broker_rewrite_map, CONNECTION_NAME_KEY, PRIVATELINK_ENDPOINT_KEY,
 };
 use risingwave_connector::WithPropertiesExt;
@@ -36,7 +35,7 @@ mod options {
     pub const RETENTION_SECONDS: &str = "retention_seconds";
 }
 
-/// Options or properties extracted from the `WITH` clause of DDLs.
+/// Options or properties extracted fro m the `WITH` clause of DDLs.
 #[derive(Default, Clone, Debug, PartialEq, Eq, Hash)]
 pub struct WithOptions {
     inner: BTreeMap<String, String>,
@@ -57,17 +56,13 @@ impl std::ops::DerefMut for WithOptions {
 }
 
 impl WithOptions {
-    /// Create a new [`WithOptions`] from a [`HashMap`].
-    pub fn new(inner: HashMap<String, String>) -> Self {
+    /// Create a new [`WithOptions`] from a [`BTreeMap`].
+    pub fn new(inner: BTreeMap<String, String>) -> Self {
         Self {
             inner: inner.into_iter().collect(),
         }
     }
 
-    pub fn from_inner(inner: BTreeMap<String, String>) -> Self {
-        Self { inner }
-    }
-
     /// Get the reference of the inner map.
     pub fn inner(&self) -> &BTreeMap<String, String> {
         &self.inner
@@ -83,7 +78,7 @@ impl WithOptions {
     }
 
     /// Convert to connector props, remove the key-value pairs used in the top-level.
-    pub fn into_connector_props(self) -> HashMap<String, String> {
+    pub fn into_connector_props(self) -> BTreeMap<String, String> {
         self.inner
             .into_iter()
             .filter(|(key, _)| key != OverwriteOptions::STREAMING_RATE_LIMIT_KEY)
@@ -121,6 +116,15 @@ impl WithOptions {
     }
 }
 
+pub(crate) fn resolve_secret_in_with_options(
+    _with_options: &mut WithOptions,
+    _session: &SessionImpl,
+) -> RwResult<BTreeMap<String, u32>> {
+    // todo: implement the function and take `resolve_privatelink_in_with_option` as reference
+
+    Ok(BTreeMap::new())
+}
+
 pub(crate) fn resolve_privatelink_in_with_option(
     with_options: &mut WithOptions,
     schema_name: &Option<String>,
diff --git a/src/java_binding/Cargo.toml b/src/java_binding/Cargo.toml
index 477f19878cbd9..0966b700a713f 100644
--- a/src/java_binding/Cargo.toml
+++ b/src/java_binding/Cargo.toml
@@ -10,13 +10,32 @@ ignored = ["workspace-hack"]
 normal = ["workspace-hack"]
 
 [dependencies]
+anyhow = "1"
+bytes = "1"
+cfg-or-panic = "0.2"
+foyer ={ workspace = true }
+futures = { version = "0.3", default-features = false, features = ["alloc"] }
 jni = "0.21.1"
 prost = { workspace = true }
 risingwave_common = { workspace = true }
+risingwave_hummock_sdk = { workspace = true }
 risingwave_jni_core = { workspace = true }
+risingwave_object_store = { workspace = true }
 risingwave_pb = { workspace = true }
+risingwave_storage = { workspace = true }
+rw_futures_util = { workspace = true }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+tokio = { version = "0.2", package = "madsim-tokio", features = [
+    "fs",
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "macros",
+    "time",
+    "signal",
+] }
+tracing = "0.1"
 
 [dev-dependencies]
 risingwave_expr = { workspace = true }
diff --git a/src/jni_core/src/hummock_iterator.rs b/src/java_binding/src/hummock_iterator.rs
similarity index 77%
rename from src/jni_core/src/hummock_iterator.rs
rename to src/java_binding/src/hummock_iterator.rs
index 42e584fb3820c..4b6fc5b01742d 100644
--- a/src/jni_core/src/hummock_iterator.rs
+++ b/src/java_binding/src/hummock_iterator.rs
@@ -14,86 +14,100 @@
 
 use std::sync::Arc;
 
+use anyhow::anyhow;
 use bytes::Bytes;
-use futures::{Stream, TryStreamExt};
+use foyer::HybridCacheBuilder;
+use futures::{TryFutureExt, TryStreamExt};
 use risingwave_common::catalog::ColumnDesc;
-use risingwave_common::config::{EvictionConfig, MetricLevel, ObjectStoreConfig};
+use risingwave_common::config::{MetricLevel, ObjectStoreConfig};
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::row::OwnedRow;
 use risingwave_common::util::value_encoding::column_aware_row_encoding::ColumnAwareSerde;
 use risingwave_common::util::value_encoding::{BasicSerde, EitherSerde, ValueRowDeserializer};
 use risingwave_hummock_sdk::key::{prefixed_range_with_vnode, TableKeyRange};
 use risingwave_hummock_sdk::version::HummockVersion;
+use risingwave_jni_core::HummockJavaBindingIterator;
 use risingwave_object_store::object::build_remote_object_store;
 use risingwave_object_store::object::object_metrics::ObjectStoreMetrics;
 use risingwave_pb::java_binding::key_range::Bound;
 use risingwave_pb::java_binding::{KeyRange, ReadPlan};
-use risingwave_storage::error::StorageResult;
+use risingwave_storage::error::{StorageError, StorageResult};
 use risingwave_storage::hummock::local_version::pinned_version::PinnedVersion;
 use risingwave_storage::hummock::store::version::HummockVersionReader;
 use risingwave_storage::hummock::store::HummockStorageIterator;
 use risingwave_storage::hummock::{
-    get_committed_read_version_tuple, CachePolicy, FileCache, SstableStore, SstableStoreConfig,
+    get_committed_read_version_tuple, CachePolicy, HummockError, SstableStore, SstableStoreConfig,
 };
 use risingwave_storage::monitor::{global_hummock_state_store_metrics, HummockStateStoreMetrics};
 use risingwave_storage::row_serde::value_serde::ValueRowSerdeNew;
 use risingwave_storage::store::{ReadOptions, StateStoreIterExt};
-use risingwave_storage::table::KeyedRow;
 use rw_futures_util::select_all;
 use tokio::sync::mpsc::unbounded_channel;
 
-type SelectAllIterStream = impl Stream<Item = StorageResult<KeyedRow<Bytes>>> + Unpin;
-type SingleIterStream = impl Stream<Item = StorageResult<KeyedRow<Bytes>>>;
+type SingleIterStream = HummockJavaBindingIterator;
 
-fn select_all_vnode_stream(streams: Vec<SingleIterStream>) -> SelectAllIterStream {
-    select_all(streams.into_iter().map(Box::pin))
+fn select_all_vnode_stream(streams: Vec<SingleIterStream>) -> HummockJavaBindingIterator {
+    Box::pin(select_all(streams))
 }
 
 fn to_deserialized_stream(
     iter: HummockStorageIterator,
     row_serde: EitherSerde,
 ) -> SingleIterStream {
-    iter.into_stream(move |(key, value)| {
-        Ok(KeyedRow::new(
-            key.user_key.table_key.copy_into(),
-            row_serde.deserialize(value).map(OwnedRow::new)?,
-        ))
-    })
+    Box::pin(
+        iter.into_stream(move |(key, value)| {
+            Ok((
+                Bytes::copy_from_slice(key.user_key.table_key.0),
+                row_serde.deserialize(value).map(OwnedRow::new)?,
+            ))
+        })
+        .map_err(|e| anyhow!(e)),
+    )
 }
 
-pub struct HummockJavaBindingIterator {
-    stream: SelectAllIterStream,
-}
-
-impl HummockJavaBindingIterator {
-    pub async fn new(read_plan: ReadPlan) -> StorageResult<Self> {
+pub(crate) async fn new_hummock_java_binding_iter(
+    read_plan: ReadPlan,
+) -> StorageResult<HummockJavaBindingIterator> {
+    {
         // Note(bugen): should we forward the implementation to the `StorageTable`?
         let object_store = Arc::new(
             build_remote_object_store(
                 &read_plan.object_store_url,
                 Arc::new(ObjectStoreMetrics::unused()),
                 "Hummock",
-                ObjectStoreConfig::default(),
+                Arc::new(ObjectStoreConfig::default()),
             )
             .await,
         );
+
+        let meta_cache = HybridCacheBuilder::new()
+            .memory(1 << 10)
+            .with_shards(2)
+            .storage()
+            .build()
+            .map_err(HummockError::foyer_error)
+            .map_err(StorageError::from)
+            .await?;
+        let block_cache = HybridCacheBuilder::new()
+            .memory(1 << 10)
+            .with_shards(2)
+            .storage()
+            .build()
+            .map_err(HummockError::foyer_error)
+            .map_err(StorageError::from)
+            .await?;
+
         let sstable_store = Arc::new(SstableStore::new(SstableStoreConfig {
             store: object_store,
             path: read_plan.data_dir,
-            block_cache_capacity: 1 << 10,
-            block_cache_shard_num: 2,
-            block_cache_eviction: EvictionConfig::for_test(),
-            meta_cache_capacity: 1 << 10,
-            meta_cache_shard_num: 2,
-            meta_cache_eviction: EvictionConfig::for_test(),
             prefetch_buffer_capacity: 1 << 10,
             max_prefetch_block_number: 16,
-            data_file_cache: FileCache::none(),
-            meta_file_cache: FileCache::none(),
             recent_filter: None,
             state_store_metrics: Arc::new(global_hummock_state_store_metrics(
                 MetricLevel::Disabled,
             )),
+            meta_cache,
+            block_cache,
         }));
         let reader = HummockVersionReader::new(
             sstable_store,
@@ -157,11 +171,7 @@ impl HummockJavaBindingIterator {
 
         let stream = select_all_vnode_stream(streams);
 
-        Ok(Self { stream })
-    }
-
-    pub async fn next(&mut self) -> StorageResult<Option<KeyedRow<Bytes>>> {
-        self.stream.try_next().await
+        Ok(stream)
     }
 }
 
diff --git a/src/java_binding/src/lib.rs b/src/java_binding/src/lib.rs
index ef5bb228b0cab..4fd089918bd5b 100644
--- a/src/java_binding/src/lib.rs
+++ b/src/java_binding/src/lib.rs
@@ -12,16 +12,83 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![feature(type_alias_impl_trait)]
+#![feature(try_blocks)]
+
+mod hummock_iterator;
 use std::ffi::c_void;
+use std::ops::Deref;
 
+use anyhow::anyhow;
+use cfg_or_panic::cfg_or_panic;
+use jni::objects::JByteArray;
 use jni::sys::{jint, JNI_VERSION_1_2};
-use jni::JavaVM;
-use risingwave_jni_core::register_native_method_for_jvm;
+use jni::{JNIEnv, JavaVM};
+use prost::Message;
+use risingwave_common::error::AsReport;
+use risingwave_jni_core::jvm_runtime::{jvm_env, register_java_binding_native_methods};
+use risingwave_jni_core::{
+    execute_and_catch, gen_class_name, to_guarded_slice, EnvParam, JavaBindingIterator, Pointer,
+    JAVA_BINDING_ASYNC_RUNTIME,
+};
+
+use crate::hummock_iterator::new_hummock_java_binding_iter;
+
+fn register_hummock_java_binding_native_methods(
+    env: &mut JNIEnv<'_>,
+) -> Result<(), jni::errors::Error> {
+    let binding_class = env
+        .find_class(gen_class_name!(com.risingwave.java.binding.HummockIterator))
+        .inspect_err(|e| tracing::error!(error = ?e.as_report(), "jvm find class error"))?;
+    macro_rules! gen_native_method_array {
+        () => {{
+            risingwave_jni_core::split_extract_plain_native_methods! {{long iteratorNewHummock(byte[] readPlan);}, gen_native_method_array}
+        }};
+        ({$({ $func_name:ident, {$($ret:tt)+}, {$($args:tt)*} })*}) => {
+            [
+                $(
+                    risingwave_jni_core::gen_native_method_entry! {
+                        Java_com_risingwave_java_binding_HummockIterator_, $func_name, {$($ret)+}, {$($args)*}
+                    },
+                )*
+            ]
+        }
+    }
+    env.register_native_methods(binding_class, &gen_native_method_array!())
+        .inspect_err(
+            |e| tracing::error!(error = ?e.as_report(), "jvm register native methods error"),
+        )?;
+
+    tracing::info!("register native methods for jvm successfully");
+    Ok(())
+}
 
 #[no_mangle]
 #[allow(non_snake_case)]
 pub extern "system" fn JNI_OnLoad(jvm: JavaVM, _reserved: *mut c_void) -> jint {
-    let _ = register_native_method_for_jvm(&jvm)
-        .inspect_err(|_e| eprintln!("unable to register native method"));
+    let result: Result<(), jni::errors::Error> = try {
+        let mut env = jvm_env(&jvm)?;
+        register_java_binding_native_methods(&mut env)?;
+        register_hummock_java_binding_native_methods(&mut env)?;
+    };
+    let _ =
+        result.inspect_err(|e| eprintln!("unable to register native method: {:?}", e.as_report()));
+
     JNI_VERSION_1_2
 }
+
+#[cfg_or_panic(not(madsim))]
+#[no_mangle]
+extern "system" fn Java_com_risingwave_java_binding_HummockIterator_iteratorNewHummock<'a>(
+    env: EnvParam<'a>,
+    read_plan: JByteArray<'a>,
+) -> Pointer<'static, JavaBindingIterator<'static>> {
+    execute_and_catch(env, move |env| {
+        let read_plan = Message::decode(to_guarded_slice(&read_plan, env)?.deref())?;
+        let iter = JAVA_BINDING_ASYNC_RUNTIME
+            .block_on(new_hummock_java_binding_iter(read_plan))
+            .map_err(|e| anyhow!(e))?;
+        let iter = JavaBindingIterator::new_hummock_iter(iter);
+        Ok(iter.into())
+    })
+}
diff --git a/src/jni_core/Cargo.toml b/src/jni_core/Cargo.toml
index 1e2e2fb3c8218..a16776add6c6f 100644
--- a/src/jni_core/Cargo.toml
+++ b/src/jni_core/Cargo.toml
@@ -14,6 +14,7 @@ anyhow = "1"
 bytes = "1"
 cfg-or-panic = "0.2"
 chrono = { version = "0.4", default-features = false }
+foyer ={ workspace = true }
 fs-err = "2"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 itertools = { workspace = true }
@@ -21,10 +22,7 @@ jni = { version = "0.21.1", features = ["invocation"] }
 paste = "1"
 prost = { workspace = true }
 risingwave_common = { workspace = true }
-risingwave_hummock_sdk = { workspace = true }
-risingwave_object_store = { workspace = true }
 risingwave_pb = { workspace = true }
-risingwave_storage = { workspace = true }
 rw_futures_util = { workspace = true }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
diff --git a/src/jni_core/src/jvm_runtime.rs b/src/jni_core/src/jvm_runtime.rs
index c369ad7b38939..53818c17e40d8 100644
--- a/src/jni_core/src/jvm_runtime.rs
+++ b/src/jni_core/src/jvm_runtime.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use core::option::Option::Some;
 use std::ffi::c_void;
 use std::path::PathBuf;
 use std::sync::OnceLock;
@@ -21,13 +20,12 @@ use anyhow::{bail, Context};
 use fs_err as fs;
 use fs_err::PathExt;
 use jni::objects::{JObject, JString};
-use jni::strings::JNIString;
-use jni::{InitArgsBuilder, JNIEnv, JNIVersion, JavaVM, NativeMethod};
+use jni::{AttachGuard, InitArgsBuilder, JNIEnv, JNIVersion, JavaVM};
 use risingwave_common::util::resource_util::memory::system_memory_available_bytes;
 use thiserror_ext::AsReport;
 use tracing::error;
 
-use crate::call_method;
+use crate::{call_method, call_static_method};
 
 /// Use 10% of compute total memory by default. Compute node uses 0.7 * system memory by default.
 const DEFAULT_MEMORY_PROPORTION: f64 = 0.07;
@@ -107,8 +105,7 @@ impl JavaVmWrapper {
             .option("-Dis_embedded_connector=true")
             .option(format!("-Djava.class.path={}", class_vec.join(":")))
             .option("-Xms16m")
-            .option(format!("-Xmx{}", jvm_heap_size))
-            .option("-Dcdc.source.wait.streaming.before.exit.seconds=30");
+            .option(format!("-Xmx{}", jvm_heap_size));
 
         tracing::info!("JVM args: {:?}", args_builder);
         let jvm_args = args_builder.build().context("invalid jvm args")?;
@@ -124,19 +121,27 @@ impl JavaVmWrapper {
 
         tracing::info!("initialize JVM successfully");
 
-        register_native_method_for_jvm(&jvm).context("failed to register native method")?;
+        let result: std::result::Result<(), jni::errors::Error> = try {
+            let mut env = jvm_env(&jvm)?;
+            register_java_binding_native_methods(&mut env)?;
+        };
+
+        result.context("failed to register native method")?;
 
         Ok(jvm)
     }
 }
 
-pub fn register_native_method_for_jvm(jvm: &JavaVM) -> Result<(), jni::errors::Error> {
-    let mut env = jvm
-        .attach_current_thread()
-        .inspect_err(|e| tracing::error!(error = ?e.as_report(), "jvm attach thread error"))?;
+pub fn jvm_env(jvm: &JavaVM) -> Result<AttachGuard<'_>, jni::errors::Error> {
+    jvm.attach_current_thread()
+        .inspect_err(|e| tracing::error!(error = ?e.as_report(), "jvm attach thread error"))
+}
 
+pub fn register_java_binding_native_methods(
+    env: &mut JNIEnv<'_>,
+) -> Result<(), jni::errors::Error> {
     let binding_class = env
-        .find_class("com/risingwave/java/binding/Binding")
+        .find_class(gen_class_name!(com.risingwave.java.binding.Binding))
         .inspect_err(|e| tracing::error!(error = ?e.as_report(), "jvm find class error"))?;
     use crate::*;
     macro_rules! gen_native_method_array {
@@ -146,14 +151,8 @@ pub fn register_native_method_for_jvm(jvm: &JavaVM) -> Result<(), jni::errors::E
         ({$({ $func_name:ident, {$($ret:tt)+}, {$($args:tt)*} })*}) => {
             [
                 $(
-                    {
-                        let fn_ptr = paste::paste! {[<Java_com_risingwave_java_binding_Binding_ $func_name> ]} as *mut c_void;
-                        let sig = $crate::gen_jni_sig! { {$($ret)+}, {$($args)*}};
-                        NativeMethod {
-                            name: JNIString::from(stringify! {$func_name}),
-                            sig: JNIString::from(sig),
-                            fn_ptr,
-                        }
+                    $crate::gen_native_method_entry! {
+                        Java_com_risingwave_java_binding_Binding_, $func_name, {$($ret)+}, {$($args)*}
                     },
                 )*
             ]
@@ -242,3 +241,36 @@ pub fn jobj_to_str(env: &mut JNIEnv<'_>, obj: JObject<'_>) -> anyhow::Result<Str
     let java_str = env.get_string(&jstr)?;
     Ok(java_str.to_str()?.to_string())
 }
+
+/// Dumps the JVM stack traces.
+///
+/// # Returns
+///
+/// - `Ok(None)` if JVM is not initialized.
+/// - `Ok(Some(String))` if JVM is initialized and stack traces are dumped.
+/// - `Err` if failed to dump stack traces.
+pub fn dump_jvm_stack_traces() -> anyhow::Result<Option<String>> {
+    match JVM.get() {
+        None => Ok(None),
+        Some(jvm) => {
+            let mut env = jvm
+                .attach_current_thread()
+                .with_context(|| "Failed to attach thread to JVM")?;
+
+            let result = call_static_method!(
+                env,
+                {com.risingwave.connector.api.Monitor},
+                {String dumpStackTrace()}
+            )
+            .with_context(|| "Failed to call Java function")?;
+            let result = JString::from(result);
+            let result = env
+                .get_string(&result)
+                .with_context(|| "Failed to convert JString")?;
+            let result = result
+                .to_str()
+                .with_context(|| "Failed to convert JavaStr")?;
+            Ok(Some(result.to_string()))
+        }
+    }
+}
diff --git a/src/jni_core/src/lib.rs b/src/jni_core/src/lib.rs
index 7ff8e5aa930e8..18d1807948d21 100644
--- a/src/jni_core/src/lib.rs
+++ b/src/jni_core/src/lib.rs
@@ -18,7 +18,6 @@
 #![feature(type_alias_impl_trait)]
 #![feature(try_blocks)]
 
-pub mod hummock_iterator;
 pub mod jvm_runtime;
 mod macros;
 mod tracing_slf4j;
@@ -33,6 +32,8 @@ use anyhow::anyhow;
 use bytes::Bytes;
 use cfg_or_panic::cfg_or_panic;
 use chrono::{Datelike, NaiveDateTime, Timelike};
+use futures::stream::BoxStream;
+use futures::TryStreamExt;
 use jni::objects::{
     AutoElements, GlobalRef, JByteArray, JClass, JMethodID, JObject, JStaticMethodID, JString,
     JValueOwned, ReleaseMode,
@@ -42,6 +43,7 @@ use jni::sys::{
     jboolean, jbyte, jdouble, jfloat, jint, jlong, jshort, jsize, jvalue, JNI_FALSE, JNI_TRUE,
 };
 use jni::JNIEnv;
+pub use paste::paste;
 use prost::{DecodeError, Message};
 use risingwave_common::array::{ArrayError, StreamChunk};
 use risingwave_common::hash::VirtualNode;
@@ -54,17 +56,14 @@ use risingwave_pb::connector_service::{
     SinkWriterStreamRequest, SinkWriterStreamResponse,
 };
 use risingwave_pb::data::Op;
-use risingwave_storage::error::StorageError;
 use thiserror::Error;
 use thiserror_ext::AsReport;
 use tokio::runtime::Runtime;
 use tokio::sync::mpsc::{Receiver, Sender};
 use tracing_slf4j::*;
 
-use crate::hummock_iterator::HummockJavaBindingIterator;
-pub use crate::jvm_runtime::register_native_method_for_jvm;
-
-static RUNTIME: LazyLock<Runtime> = LazyLock::new(|| tokio::runtime::Runtime::new().unwrap());
+pub static JAVA_BINDING_ASYNC_RUNTIME: LazyLock<Runtime> =
+    LazyLock::new(|| tokio::runtime::Runtime::new().unwrap());
 
 #[derive(Error, Debug)]
 pub enum BindingError {
@@ -78,7 +77,7 @@ pub enum BindingError {
     #[error("StorageError {error}")]
     Storage {
         #[from]
-        error: StorageError,
+        error: anyhow::Error,
         backtrace: Backtrace,
     },
 
@@ -201,7 +200,7 @@ impl<'a> EnvParam<'a> {
     }
 }
 
-fn execute_and_catch<'env, F, Ret>(mut env: EnvParam<'env>, inner: F) -> Ret
+pub fn execute_and_catch<'env, F, Ret>(mut env: EnvParam<'env>, inner: F) -> Ret
 where
     F: FnOnce(&mut EnvParam<'env>) -> Result<Ret>,
     Ret: Default + 'env,
@@ -245,9 +244,10 @@ struct JavaClassMethodCache {
 }
 
 // TODO: may only return a RowRef
-type StreamChunkRowIterator<'a> = impl Iterator<Item = (Op, OwnedRow)> + 'a;
+pub type StreamChunkRowIterator<'a> = impl Iterator<Item = (Op, OwnedRow)> + 'a;
+pub type HummockJavaBindingIterator = BoxStream<'static, anyhow::Result<(Bytes, OwnedRow)>>;
 
-enum JavaBindingIteratorInner<'a> {
+pub enum JavaBindingIteratorInner<'a> {
     Hummock(HummockJavaBindingIterator),
     StreamChunk(StreamChunkRowIterator<'a>),
 }
@@ -288,12 +288,22 @@ struct RowCursor {
     extra: RowExtra,
 }
 
-struct JavaBindingIterator<'a> {
+pub struct JavaBindingIterator<'a> {
     inner: JavaBindingIteratorInner<'a>,
     cursor: Option<RowCursor>,
     class_cache: JavaClassMethodCache,
 }
 
+impl JavaBindingIterator<'static> {
+    pub fn new_hummock_iter(iter: HummockJavaBindingIterator) -> Self {
+        Self {
+            inner: JavaBindingIteratorInner::Hummock(iter),
+            cursor: None,
+            class_cache: Default::default(),
+        }
+    }
+}
+
 impl<'a> Deref for JavaBindingIterator<'a> {
     type Target = OwnedRow;
 
@@ -311,24 +321,6 @@ extern "system" fn Java_com_risingwave_java_binding_Binding_vnodeCount(_env: Env
     VirtualNode::COUNT as jint
 }
 
-#[cfg_or_panic(not(madsim))]
-#[no_mangle]
-extern "system" fn Java_com_risingwave_java_binding_Binding_iteratorNewHummock<'a>(
-    env: EnvParam<'a>,
-    read_plan: JByteArray<'a>,
-) -> Pointer<'static, JavaBindingIterator<'static>> {
-    execute_and_catch(env, move |env| {
-        let read_plan = Message::decode(to_guarded_slice(&read_plan, env)?.deref())?;
-        let iter = RUNTIME.block_on(HummockJavaBindingIterator::new(read_plan))?;
-        let iter = JavaBindingIterator {
-            inner: JavaBindingIteratorInner::Hummock(iter),
-            cursor: None,
-            class_cache: Default::default(),
-        };
-        Ok(iter.into())
-    })
-}
-
 #[cfg_or_panic(not(madsim))]
 #[no_mangle]
 extern "system" fn Java_com_risingwave_java_binding_Binding_iteratorNewStreamChunk<'a>(
@@ -355,16 +347,15 @@ extern "system" fn Java_com_risingwave_java_binding_Binding_iteratorNext<'a>(
         let iter = pointer.as_mut();
         match &mut iter.inner {
             JavaBindingIteratorInner::Hummock(ref mut hummock_iter) => {
-                match RUNTIME.block_on(hummock_iter.next())? {
+                match JAVA_BINDING_ASYNC_RUNTIME.block_on(hummock_iter.try_next())? {
                     None => {
                         iter.cursor = None;
                         Ok(JNI_FALSE)
                     }
-                    Some(keyed_row) => {
-                        let (key, row) = keyed_row.into_parts();
+                    Some((key, row)) => {
                         iter.cursor = Some(RowCursor {
                             row,
-                            extra: RowExtra::Key(key.0),
+                            extra: RowExtra::Key(key),
                         });
                         Ok(JNI_TRUE)
                     }
diff --git a/src/jni_core/src/macros.rs b/src/jni_core/src/macros.rs
index 1b2f79f829564..982ccda06ecf0 100644
--- a/src/jni_core/src/macros.rs
+++ b/src/jni_core/src/macros.rs
@@ -448,10 +448,6 @@ macro_rules! for_all_plain_native_methods {
 
                 public static native int vnodeCount();
 
-                // hummock iterator method
-                // Return a pointer to the iterator
-                static native long iteratorNewHummock(byte[] readPlan);
-
                 static native long iteratorNewStreamChunk(long pointer);
 
                 static native boolean iteratorNext(long pointer);
@@ -839,6 +835,23 @@ macro_rules! call_method {
     }};
 }
 
+#[macro_export]
+macro_rules! gen_native_method_entry {
+    (
+        $class_prefix:ident, $func_name:ident, {$($ret:tt)+}, {$($args:tt)*}
+    ) => {{
+        {
+            let fn_ptr = $crate::paste! {[<$class_prefix $func_name> ]} as *mut c_void;
+            let sig = $crate::gen_jni_sig! { {$($ret)+}, {$($args)*}};
+            jni::NativeMethod {
+                name: jni::strings::JNIString::from(stringify! {$func_name}),
+                sig: jni::strings::JNIString::from(sig),
+                fn_ptr,
+            }
+        }
+    }};
+}
+
 #[cfg(test)]
 mod tests {
     use std::fmt::Formatter;
@@ -891,7 +904,6 @@ mod tests {
                 tracingSlf4jEvent                        (Ljava/lang/String;Ljava/lang/String;ILjava/lang/String;)V,
                 tracingSlf4jEventEnabled                 (I)Z,
                 vnodeCount                               ()I,
-                iteratorNewHummock                       ([B)J,
                 iteratorNewStreamChunk                   (J)J,
                 iteratorNext                             (J)Z,
                 iteratorClose                            (J)V,
diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml
index c43c43c0dc01f..6252d845788af 100644
--- a/src/meta/Cargo.toml
+++ b/src/meta/Cargo.toml
@@ -14,13 +14,15 @@ ignored = ["workspace-hack"]
 normal = ["workspace-hack"]
 
 [dependencies]
+aes-siv = "0.7"
 anyhow = "1"
 arc-swap = "1"
 assert_matches = "1"
 async-trait = "0.1"
 aws-config = { workspace = true }
 aws-sdk-ec2 = { workspace = true }
-base64-url = { version = "2.0.0" }
+base64-url = { version = "3.0.0" }
+bincode = "1.3"
 bytes = { version = "1", features = ["serde"] }
 chrono = "0.4"
 clap = { workspace = true }
diff --git a/src/meta/model_v2/migration/src/lib.rs b/src/meta/model_v2/migration/src/lib.rs
index 7feaf88788693..66f136b6159d1 100644
--- a/src/meta/model_v2/migration/src/lib.rs
+++ b/src/meta/model_v2/migration/src/lib.rs
@@ -9,6 +9,8 @@ mod m20240410_082733_with_version_column_migration;
 mod m20240410_154406_session_params;
 mod m20240417_062305_subscription_internal_table_name;
 mod m20240418_142249_function_runtime;
+mod m20240506_112555_subscription_partial_ckpt;
+mod m20240525_090457_secret;
 
 pub struct Migrator;
 
@@ -23,6 +25,8 @@ impl MigratorTrait for Migrator {
             Box::new(m20240410_154406_session_params::Migration),
             Box::new(m20240417_062305_subscription_internal_table_name::Migration),
             Box::new(m20240418_142249_function_runtime::Migration),
+            Box::new(m20240506_112555_subscription_partial_ckpt::Migration),
+            Box::new(m20240525_090457_secret::Migration),
         ]
     }
 }
diff --git a/src/meta/model_v2/migration/src/m20230908_072257_init.rs b/src/meta/model_v2/migration/src/m20230908_072257_init.rs
index 260344a8b4fd9..7840c40cd8f30 100644
--- a/src/meta/model_v2/migration/src/m20230908_072257_init.rs
+++ b/src/meta/model_v2/migration/src/m20230908_072257_init.rs
@@ -1222,7 +1222,7 @@ enum Function {
 }
 
 #[derive(DeriveIden)]
-enum Object {
+pub(crate) enum Object {
     Table,
     Oid,
     ObjType,
diff --git a/src/meta/model_v2/migration/src/m20231008_020431_hummock.rs b/src/meta/model_v2/migration/src/m20231008_020431_hummock.rs
index 99f4d701c6445..42fe0c424f99c 100644
--- a/src/meta/model_v2/migration/src/m20231008_020431_hummock.rs
+++ b/src/meta/model_v2/migration/src/m20231008_020431_hummock.rs
@@ -30,7 +30,11 @@ impl MigrationTrait for Migration {
                             .not_null()
                             .primary_key(),
                     )
-                    .col(ColumnDef::new(CompactionTask::Task).binary().not_null())
+                    .col(
+                        ColumnDef::new(CompactionTask::Task)
+                            .blob(BlobSize::Long)
+                            .not_null(),
+                    )
                     .col(
                         ColumnDef::new(CompactionTask::ContextId)
                             .integer()
@@ -50,7 +54,7 @@ impl MigrationTrait for Migration {
                             .not_null()
                             .primary_key(),
                     )
-                    .col(ColumnDef::new(CompactionConfig::Config).binary())
+                    .col(ColumnDef::new(CompactionConfig::Config).blob(BlobSize::Long))
                     .to_owned(),
             )
             .await?;
@@ -65,7 +69,7 @@ impl MigrationTrait for Migration {
                             .not_null()
                             .primary_key(),
                     )
-                    .col(ColumnDef::new(CompactionStatus::Status).binary())
+                    .col(ColumnDef::new(CompactionStatus::Status).blob(BlobSize::Long))
                     .to_owned(),
             )
             .await?;
@@ -138,7 +142,7 @@ impl MigrationTrait for Migration {
                             .boolean()
                             .not_null(),
                     )
-                    .col(ColumnDef::new(HummockVersionDelta::FullVersionDelta).binary())
+                    .col(ColumnDef::new(HummockVersionDelta::FullVersionDelta).blob(BlobSize::Long))
                     .to_owned(),
             )
             .await?;
diff --git a/src/meta/model_v2/migration/src/m20240304_074901_subscription.rs b/src/meta/model_v2/migration/src/m20240304_074901_subscription.rs
index 50564c9b211a6..244c6abe11604 100644
--- a/src/meta/model_v2/migration/src/m20240304_074901_subscription.rs
+++ b/src/meta/model_v2/migration/src/m20240304_074901_subscription.rs
@@ -37,6 +37,17 @@ impl MigrationTrait for Migration {
                             .string()
                             .not_null(),
                     )
+                    .foreign_key(
+                        &mut ForeignKey::create()
+                            .name("FK_subscription_object_id")
+                            .from(Subscription::Table, Subscription::SubscriptionId)
+                            .to(
+                                crate::m20230908_072257_init::Object::Table,
+                                crate::m20230908_072257_init::Object::Oid,
+                            )
+                            .on_delete(ForeignKeyAction::Cascade)
+                            .to_owned(),
+                    )
                     .to_owned(),
             )
             .await?;
diff --git a/src/meta/model_v2/migration/src/m20240417_062305_subscription_internal_table_name.rs b/src/meta/model_v2/migration/src/m20240417_062305_subscription_internal_table_name.rs
index a4c6f60928c91..2fadc64b447d8 100644
--- a/src/meta/model_v2/migration/src/m20240417_062305_subscription_internal_table_name.rs
+++ b/src/meta/model_v2/migration/src/m20240417_062305_subscription_internal_table_name.rs
@@ -10,7 +10,7 @@ impl MigrationTrait for Migration {
                 MigrationTable::alter()
                     .table(Subscription::Table)
                     .add_column(
-                        ColumnDef::new(Subscription::SubscriptionInternalTableName).integer(),
+                        ColumnDef::new(Subscription::SubscriptionInternalTableName).string(),
                     )
                     .to_owned(),
             )
diff --git a/src/meta/model_v2/migration/src/m20240506_112555_subscription_partial_ckpt.rs b/src/meta/model_v2/migration/src/m20240506_112555_subscription_partial_ckpt.rs
new file mode 100644
index 0000000000000..715be0e7cd23f
--- /dev/null
+++ b/src/meta/model_v2/migration/src/m20240506_112555_subscription_partial_ckpt.rs
@@ -0,0 +1,124 @@
+use sea_orm_migration::prelude::{Table as MigrationTable, *};
+
+use crate::drop_tables;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        // drop tables cascade.
+        drop_tables!(manager, Subscription);
+        manager
+            .create_table(
+                MigrationTable::create()
+                    .table(Subscription::Table)
+                    .col(
+                        ColumnDef::new(Subscription::SubscriptionId)
+                            .integer()
+                            .primary_key(),
+                    )
+                    .col(ColumnDef::new(Subscription::Name).string().not_null())
+                    .col(ColumnDef::new(Subscription::Definition).string().not_null())
+                    .col(
+                        ColumnDef::new(Subscription::RetentionSeconds)
+                            .string()
+                            .big_integer(),
+                    )
+                    .col(
+                        ColumnDef::new(Subscription::SubscriptionState)
+                            .string()
+                            .integer(),
+                    )
+                    .col(
+                        ColumnDef::new(Subscription::DependentTableId)
+                            .integer()
+                            .not_null(),
+                    )
+                    .foreign_key(
+                        &mut ForeignKey::create()
+                            .name("FK_subscription_object_id")
+                            .from(Subscription::Table, Subscription::SubscriptionId)
+                            .to(
+                                crate::m20230908_072257_init::Object::Table,
+                                crate::m20230908_072257_init::Object::Oid,
+                            )
+                            .on_delete(ForeignKeyAction::Cascade)
+                            .to_owned(),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        // drop tables cascade.
+        drop_tables!(manager, Subscription);
+        manager
+            .create_table(
+                MigrationTable::create()
+                    .table(Subscription::Table)
+                    .col(
+                        ColumnDef::new(Subscription::SubscriptionId)
+                            .integer()
+                            .primary_key(),
+                    )
+                    .col(ColumnDef::new(Subscription::Name).string().not_null())
+                    .col(ColumnDef::new(Subscription::Definition).string().not_null())
+                    .col(ColumnDef::new(Subscription::Columns).binary().not_null())
+                    .col(ColumnDef::new(Subscription::PlanPk).binary().not_null())
+                    .col(
+                        ColumnDef::new(Subscription::DistributionKey)
+                            .json_binary()
+                            .not_null(),
+                    )
+                    .col(
+                        ColumnDef::new(Subscription::Properties)
+                            .json_binary()
+                            .not_null(),
+                    )
+                    .col(
+                        ColumnDef::new(Subscription::SubscriptionFromName)
+                            .string()
+                            .not_null(),
+                    )
+                    .col(ColumnDef::new(Subscription::SubscriptionInternalTableName).string())
+                    .foreign_key(
+                        &mut ForeignKey::create()
+                            .name("FK_subscription_object_id")
+                            .from(Subscription::Table, Subscription::SubscriptionId)
+                            .to(
+                                crate::m20230908_072257_init::Object::Table,
+                                crate::m20230908_072257_init::Object::Oid,
+                            )
+                            .on_delete(ForeignKeyAction::Cascade)
+                            .to_owned(),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+        Ok(())
+    }
+}
+
+#[derive(DeriveIden)]
+enum Subscription {
+    Table,
+    // common
+    SubscriptionId,
+    Name,
+    Definition,
+    // before
+    Columns,
+    PlanPk,
+    DistributionKey,
+    Properties,
+    SubscriptionFromName,
+    SubscriptionInternalTableName,
+    // after
+    RetentionSeconds,
+    SubscriptionState,
+    DependentTableId,
+}
diff --git a/src/meta/model_v2/migration/src/m20240525_090457_secret.rs b/src/meta/model_v2/migration/src/m20240525_090457_secret.rs
new file mode 100644
index 0000000000000..f16bfca5ec035
--- /dev/null
+++ b/src/meta/model_v2/migration/src/m20240525_090457_secret.rs
@@ -0,0 +1,79 @@
+use sea_orm_migration::prelude::{Table as MigrationTable, *};
+
+use crate::{assert_not_has_tables, drop_tables};
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        assert_not_has_tables!(manager, Secret);
+        manager
+            .create_table(
+                MigrationTable::create()
+                    .table(Secret::Table)
+                    .if_not_exists()
+                    .col(
+                        ColumnDef::new(Secret::SecretId)
+                            .integer()
+                            .not_null()
+                            .primary_key(),
+                    )
+                    .col(ColumnDef::new(Secret::Name).string().not_null())
+                    .col(ColumnDef::new(Secret::Value).binary().not_null())
+                    .foreign_key(
+                        &mut ForeignKey::create()
+                            .name("FK_secret_object_id")
+                            .from(Secret::Table, Secret::SecretId)
+                            .to(
+                                crate::m20230908_072257_init::Object::Table,
+                                crate::m20230908_072257_init::Object::Oid,
+                            )
+                            .on_delete(ForeignKeyAction::Cascade)
+                            .to_owned(),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+
+        // Add a new column to the table
+        manager
+            .alter_table(
+                MigrationTable::alter()
+                    .table(Sink::Table)
+                    .add_column(ColumnDef::new(Sink::SecretRef).json_binary())
+                    .to_owned(),
+            )
+            .await?;
+
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        drop_tables!(manager, Secret);
+        manager
+            .alter_table(
+                MigrationTable::alter()
+                    .table(Sink::Table)
+                    .drop_column(Sink::SecretRef)
+                    .to_owned(),
+            )
+            .await?;
+        Ok(())
+    }
+}
+
+#[derive(DeriveIden)]
+enum Secret {
+    Table,
+    SecretId,
+    Name,
+    Value,
+}
+
+#[derive(DeriveIden)]
+enum Sink {
+    Table,
+    SecretRef,
+}
diff --git a/src/meta/model_v2/src/actor.rs b/src/meta/model_v2/src/actor.rs
index 2ebafd23a51ad..c75eac7dbc4cf 100644
--- a/src/meta/model_v2/src/actor.rs
+++ b/src/meta/model_v2/src/actor.rs
@@ -14,12 +14,13 @@
 
 use risingwave_pb::meta::table_fragments::actor_status::PbActorState;
 use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
 
 use crate::{
     ActorId, ActorUpstreamActors, ConnectorSplits, ExprContext, FragmentId, VnodeBitmap, WorkerId,
 };
 
-#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum ActorStatus {
     #[sea_orm(string_value = "INACTIVE")]
@@ -47,7 +48,7 @@ impl From<ActorStatus> for PbActorState {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "actor")]
 pub struct Model {
     #[sea_orm(primary_key)]
diff --git a/src/meta/model_v2/src/actor_dispatcher.rs b/src/meta/model_v2/src/actor_dispatcher.rs
index 1d6a50665b12a..81211cc572701 100644
--- a/src/meta/model_v2/src/actor_dispatcher.rs
+++ b/src/meta/model_v2/src/actor_dispatcher.rs
@@ -14,10 +14,11 @@
 
 use risingwave_pb::stream_plan::{PbDispatcher, PbDispatcherType};
 use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
 
 use crate::{ActorId, ActorMapping, FragmentId, I32Array};
 
-#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Deserialize, Serialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum DispatcherType {
     #[sea_orm(string_value = "HASH")]
@@ -81,7 +82,7 @@ impl From<Model> for PbDispatcher {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Deserialize, Serialize)]
 #[sea_orm(table_name = "actor_dispatcher")]
 pub struct Model {
     #[sea_orm(primary_key)]
diff --git a/src/meta/model_v2/src/catalog_version.rs b/src/meta/model_v2/src/catalog_version.rs
index 53c6f9109635c..cfcd90c12f6a0 100644
--- a/src/meta/model_v2/src/catalog_version.rs
+++ b/src/meta/model_v2/src/catalog_version.rs
@@ -13,8 +13,9 @@
 // limitations under the License.
 
 use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum VersionCategory {
     #[sea_orm(string_value = "NOTIFICATION")]
@@ -23,7 +24,7 @@ pub enum VersionCategory {
     TableRevision,
 }
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "catalog_version")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/cluster.rs b/src/meta/model_v2/src/cluster.rs
index 767094f1a1e14..39d0bc3439980 100644
--- a/src/meta/model_v2/src/cluster.rs
+++ b/src/meta/model_v2/src/cluster.rs
@@ -13,8 +13,9 @@
 // limitations under the License.
 
 use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "cluster")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/compaction_task.rs b/src/meta/model_v2/src/compaction_task.rs
index 074fe9af450ea..302bd15744f28 100644
--- a/src/meta/model_v2/src/compaction_task.rs
+++ b/src/meta/model_v2/src/compaction_task.rs
@@ -14,7 +14,6 @@
 
 use risingwave_pb::hummock::{CompactTask as PbCompactTask, CompactTaskAssignment};
 use sea_orm::entity::prelude::*;
-use serde::{Deserialize, Serialize};
 
 use crate::{CompactionTaskId, WorkerId};
 
diff --git a/src/meta/model_v2/src/connection.rs b/src/meta/model_v2/src/connection.rs
index 0e513e7061fd6..a6cfa4aefb58c 100644
--- a/src/meta/model_v2/src/connection.rs
+++ b/src/meta/model_v2/src/connection.rs
@@ -16,10 +16,11 @@ use risingwave_pb::catalog::connection::PbInfo;
 use risingwave_pb::catalog::PbConnection;
 use sea_orm::entity::prelude::*;
 use sea_orm::ActiveValue::Set;
+use serde::{Deserialize, Serialize};
 
 use crate::{ConnectionId, PrivateLinkService};
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "connection")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/database.rs b/src/meta/model_v2/src/database.rs
index f8951f63c9681..089a687129696 100644
--- a/src/meta/model_v2/src/database.rs
+++ b/src/meta/model_v2/src/database.rs
@@ -15,10 +15,11 @@
 use risingwave_pb::catalog::PbDatabase;
 use sea_orm::entity::prelude::*;
 use sea_orm::ActiveValue::Set;
+use serde::{Deserialize, Serialize};
 
 use crate::DatabaseId;
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "database")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/fragment.rs b/src/meta/model_v2/src/fragment.rs
index af1d529a05980..7f69584538593 100644
--- a/src/meta/model_v2/src/fragment.rs
+++ b/src/meta/model_v2/src/fragment.rs
@@ -14,10 +14,11 @@
 
 use risingwave_pb::meta::table_fragments::fragment::PbFragmentDistributionType;
 use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
 
 use crate::{FragmentId, FragmentVnodeMapping, I32Array, ObjectId, StreamNode};
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "fragment")]
 pub struct Model {
     #[sea_orm(primary_key)]
@@ -31,7 +32,7 @@ pub struct Model {
     pub upstream_fragment_id: I32Array,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum DistributionType {
     #[sea_orm(string_value = "SINGLE")]
diff --git a/src/meta/model_v2/src/function.rs b/src/meta/model_v2/src/function.rs
index 520df5cd2bf05..5589c1daa023f 100644
--- a/src/meta/model_v2/src/function.rs
+++ b/src/meta/model_v2/src/function.rs
@@ -16,10 +16,11 @@ use risingwave_pb::catalog::function::Kind;
 use risingwave_pb::catalog::PbFunction;
 use sea_orm::entity::prelude::*;
 use sea_orm::ActiveValue::Set;
+use serde::{Deserialize, Serialize};
 
 use crate::{DataType, DataTypeArray, FunctionId};
 
-#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum FunctionKind {
     #[sea_orm(string_value = "Scalar")]
@@ -30,7 +31,7 @@ pub enum FunctionKind {
     Aggregate,
 }
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "function")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/hummock_sequence.rs b/src/meta/model_v2/src/hummock_sequence.rs
index 63cfe6592458b..58156c33266f7 100644
--- a/src/meta/model_v2/src/hummock_sequence.rs
+++ b/src/meta/model_v2/src/hummock_sequence.rs
@@ -13,13 +13,13 @@
 // limitations under the License.
 
 use sea_orm::entity::prelude::*;
-
+use serde::{Deserialize, Serialize};
 pub const COMPACTION_TASK_ID: &str = "compaction_task";
 pub const COMPACTION_GROUP_ID: &str = "compaction_group";
 pub const SSTABLE_OBJECT_ID: &str = "sstable_object";
 pub const META_BACKUP_ID: &str = "meta_backup";
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Default)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Default, Serialize, Deserialize)]
 #[sea_orm(table_name = "hummock_sequence")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/hummock_version_delta.rs b/src/meta/model_v2/src/hummock_version_delta.rs
index ae69f5eca9a1d..317e3c7199863 100644
--- a/src/meta/model_v2/src/hummock_version_delta.rs
+++ b/src/meta/model_v2/src/hummock_version_delta.rs
@@ -14,7 +14,6 @@
 
 use risingwave_pb::hummock::PbHummockVersionDelta;
 use sea_orm::entity::prelude::*;
-use serde::{Deserialize, Serialize};
 
 use crate::{Epoch, HummockVersionId};
 
diff --git a/src/meta/model_v2/src/index.rs b/src/meta/model_v2/src/index.rs
index 8f6dd60d6d76d..ca2f39c0f179f 100644
--- a/src/meta/model_v2/src/index.rs
+++ b/src/meta/model_v2/src/index.rs
@@ -15,10 +15,11 @@
 use risingwave_pb::catalog::PbIndex;
 use sea_orm::entity::prelude::*;
 use sea_orm::ActiveValue::Set;
+use serde::{Deserialize, Serialize};
 
 use crate::{ExprNodeArray, IndexId, TableId};
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "index")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/lib.rs b/src/meta/model_v2/src/lib.rs
index 05fb29cb70bac..647ce99ec1e78 100644
--- a/src/meta/model_v2/src/lib.rs
+++ b/src/meta/model_v2/src/lib.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::BTreeMap;
 
 use risingwave_pb::catalog::{PbCreateType, PbStreamJobStatus};
 use risingwave_pb::meta::table_fragments::PbState as PbStreamJobState;
@@ -43,6 +43,8 @@ pub mod index;
 pub mod object;
 pub mod object_dependency;
 pub mod schema;
+pub mod secret;
+pub mod serde_seaql_migration;
 pub mod session_parameter;
 pub mod sink;
 pub mod source;
@@ -71,6 +73,7 @@ pub type IndexId = ObjectId;
 pub type ViewId = ObjectId;
 pub type FunctionId = ObjectId;
 pub type ConnectionId = ObjectId;
+pub type SecretId = ObjectId;
 pub type UserId = i32;
 pub type PrivilegeId = i32;
 
@@ -83,7 +86,7 @@ pub type HummockSstableObjectId = i64;
 pub type FragmentId = i32;
 pub type ActorId = i32;
 
-#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum JobStatus {
     #[sea_orm(string_value = "INITIAL")]
@@ -115,7 +118,7 @@ impl From<JobStatus> for PbStreamJobState {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum CreateType {
     #[sea_orm(string_value = "BACKGROUND")]
@@ -170,7 +173,7 @@ macro_rules! derive_from_json_struct {
 /// Defines struct with a byte array that derives `DeriveValueType`, it will helps to map blob stored in database to Pb struct.
 macro_rules! derive_from_blob {
     ($struct_name:ident, $field_type:ty) => {
-        #[derive(Clone, PartialEq, Eq, Serialize, Deserialize, DeriveValueType)]
+        #[derive(Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, DeriveValueType)]
         pub struct $struct_name(#[sea_orm] Vec<u8>);
 
         impl $struct_name {
@@ -212,7 +215,7 @@ macro_rules! derive_from_blob {
 /// Defines struct with a byte array that derives `DeriveValueType`, it will helps to map blob stored in database to Pb struct array.
 macro_rules! derive_array_from_blob {
     ($struct_name:ident, $field_type:ty, $field_array_name:ident) => {
-        #[derive(Clone, PartialEq, Eq, DeriveValueType)]
+        #[derive(Clone, PartialEq, Eq, DeriveValueType, serde::Deserialize, serde::Serialize)]
         pub struct $struct_name(#[sea_orm] Vec<u8>);
 
         #[derive(Clone, PartialEq, ::prost::Message)]
@@ -255,7 +258,7 @@ macro_rules! derive_array_from_blob {
     };
 }
 
-pub(crate) use {derive_array_from_blob, derive_from_blob, derive_from_json_struct};
+pub(crate) use {derive_array_from_blob, derive_from_blob};
 
 derive_from_json_struct!(I32Array, Vec<i32>);
 
@@ -283,6 +286,8 @@ impl From<BTreeMap<u32, Vec<u32>>> for ActorUpstreamActors {
     }
 }
 
+derive_from_json_struct!(SecretRef, BTreeMap<String, u32>);
+
 derive_from_blob!(StreamNode, PbStreamNode);
 derive_from_blob!(DataType, risingwave_pb::data::PbDataType);
 derive_array_from_blob!(
@@ -295,7 +300,7 @@ derive_array_from_blob!(
     risingwave_pb::plan_common::PbField,
     PbFieldArray
 );
-derive_from_json_struct!(Property, HashMap<String, String>);
+derive_from_json_struct!(Property, BTreeMap<String, String>);
 derive_from_blob!(ColumnCatalog, risingwave_pb::plan_common::PbColumnCatalog);
 derive_array_from_blob!(
     ColumnCatalogArray,
diff --git a/src/meta/model_v2/src/object.rs b/src/meta/model_v2/src/object.rs
index 6df5db623ae3c..663f436fcbcb6 100644
--- a/src/meta/model_v2/src/object.rs
+++ b/src/meta/model_v2/src/object.rs
@@ -13,10 +13,11 @@
 // limitations under the License.
 
 use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
 
 use crate::{DatabaseId, ObjectId, SchemaId, UserId};
 
-#[derive(Clone, Debug, PartialEq, Eq, Copy, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Debug, PartialEq, Eq, Copy, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum ObjectType {
     #[sea_orm(string_value = "DATABASE")]
@@ -39,6 +40,8 @@ pub enum ObjectType {
     Connection,
     #[sea_orm(string_value = "SUBSCRIPTION")]
     Subscription,
+    #[sea_orm(string_value = "SECRET")]
+    Secret,
 }
 
 impl ObjectType {
@@ -54,11 +57,12 @@ impl ObjectType {
             ObjectType::Function => "function",
             ObjectType::Connection => "connection",
             ObjectType::Subscription => "subscription",
+            ObjectType::Secret => "secret",
         }
     }
 }
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "object")]
 pub struct Model {
     #[sea_orm(primary_key)]
diff --git a/src/meta/model_v2/src/object_dependency.rs b/src/meta/model_v2/src/object_dependency.rs
index 0adaacfced3a9..d5ca89215a93d 100644
--- a/src/meta/model_v2/src/object_dependency.rs
+++ b/src/meta/model_v2/src/object_dependency.rs
@@ -13,10 +13,11 @@
 // limitations under the License.
 
 use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
 
 use crate::{ObjectId, UserId};
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "object_dependency")]
 pub struct Model {
     #[sea_orm(primary_key)]
diff --git a/src/meta/model_v2/src/prelude.rs b/src/meta/model_v2/src/prelude.rs
index 7d3c8cde7acb1..b17eae112aef0 100644
--- a/src/meta/model_v2/src/prelude.rs
+++ b/src/meta/model_v2/src/prelude.rs
@@ -32,6 +32,7 @@ pub use super::index::Entity as Index;
 pub use super::object::Entity as Object;
 pub use super::object_dependency::Entity as ObjectDependency;
 pub use super::schema::Entity as Schema;
+pub use super::secret::Entity as Secret;
 pub use super::session_parameter::Entity as SessionParameter;
 pub use super::sink::Entity as Sink;
 pub use super::source::Entity as Source;
diff --git a/src/meta/model_v2/src/schema.rs b/src/meta/model_v2/src/schema.rs
index e1183d0bbbb2e..89c1c5c75205b 100644
--- a/src/meta/model_v2/src/schema.rs
+++ b/src/meta/model_v2/src/schema.rs
@@ -15,10 +15,11 @@
 use risingwave_pb::catalog::PbSchema;
 use sea_orm::entity::prelude::*;
 use sea_orm::ActiveValue::Set;
+use serde::{Deserialize, Serialize};
 
 use crate::SchemaId;
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "schema")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/secret.rs b/src/meta/model_v2/src/secret.rs
new file mode 100644
index 0000000000000..0d122267bb4bc
--- /dev/null
+++ b/src/meta/model_v2/src/secret.rs
@@ -0,0 +1,58 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use risingwave_pb::catalog::PbSecret;
+use sea_orm::entity::prelude::*;
+use sea_orm::ActiveValue::Set;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
+#[sea_orm(table_name = "secret")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub secret_id: i32,
+    pub name: String,
+    #[sea_orm(column_type = "Binary(BlobSize::Blob(None))")]
+    pub value: Vec<u8>,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {
+    #[sea_orm(
+        belongs_to = "super::object::Entity",
+        from = "Column::SecretId",
+        to = "super::object::Column::Oid",
+        on_update = "NoAction",
+        on_delete = "Cascade"
+    )]
+    Object,
+}
+
+impl Related<super::object::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Object.def()
+    }
+}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+impl From<PbSecret> for ActiveModel {
+    fn from(secret: PbSecret) -> Self {
+        Self {
+            secret_id: Set(secret.id as _),
+            name: Set(secret.name),
+            value: Set(secret.value),
+        }
+    }
+}
diff --git a/src/meta/model_v2/src/serde_seaql_migration.rs b/src/meta/model_v2/src/serde_seaql_migration.rs
new file mode 100644
index 0000000000000..6cc75f57c0313
--- /dev/null
+++ b/src/meta/model_v2/src/serde_seaql_migration.rs
@@ -0,0 +1,31 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// It duplicates the one found in crate sea-orm-migration, but derives serde.
+/// It's only used by metadata backup/restore.
+use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
+#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Serialize, Deserialize)]
+// One should override the name of migration table via `MigratorTrait::migration_table_name` method
+#[sea_orm(table_name = "seaql_migrations")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub version: String,
+    pub applied_at: i64,
+}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
+
+impl ActiveModelBehavior for ActiveModel {}
diff --git a/src/meta/model_v2/src/session_parameter.rs b/src/meta/model_v2/src/session_parameter.rs
index b0623270982f7..1fcb3f5bc2dae 100644
--- a/src/meta/model_v2/src/session_parameter.rs
+++ b/src/meta/model_v2/src/session_parameter.rs
@@ -14,7 +14,7 @@
 
 use sea_orm::entity::prelude::*;
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, serde::Deserialize, serde::Serialize)]
 #[sea_orm(table_name = "session_parameter")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/sink.rs b/src/meta/model_v2/src/sink.rs
index ab7e869daee68..78d0806f98a5e 100644
--- a/src/meta/model_v2/src/sink.rs
+++ b/src/meta/model_v2/src/sink.rs
@@ -15,13 +15,14 @@
 use risingwave_pb::catalog::{PbSink, PbSinkType};
 use sea_orm::entity::prelude::*;
 use sea_orm::ActiveValue::Set;
+use serde::{Deserialize, Serialize};
 
 use crate::{
-    ColumnCatalogArray, ColumnOrderArray, ConnectionId, I32Array, Property, SinkFormatDesc, SinkId,
-    TableId,
+    ColumnCatalogArray, ColumnOrderArray, ConnectionId, I32Array, Property, SecretRef,
+    SinkFormatDesc, SinkId, TableId,
 };
 
-#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum SinkType {
     #[sea_orm(string_value = "APPEND_ONLY")]
@@ -53,7 +54,7 @@ impl From<PbSinkType> for SinkType {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "sink")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
@@ -71,6 +72,8 @@ pub struct Model {
     pub sink_from_name: String,
     pub sink_format_desc: Option<SinkFormatDesc>,
     pub target_table: Option<TableId>,
+    // `secret_ref` stores a json string, mapping from property name to secret id.
+    pub secret_ref: Option<SecretRef>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -126,6 +129,7 @@ impl From<PbSink> for ActiveModel {
             sink_from_name: Set(pb_sink.sink_from_name),
             sink_format_desc: Set(pb_sink.format_desc.as_ref().map(|x| x.into())),
             target_table: Set(pb_sink.target_table.map(|x| x as _)),
+            secret_ref: Set(Some(SecretRef::from(pb_sink.secret_ref))),
         }
     }
 }
diff --git a/src/meta/model_v2/src/source.rs b/src/meta/model_v2/src/source.rs
index 2b0e511e4afe6..be2d2f7110cab 100644
--- a/src/meta/model_v2/src/source.rs
+++ b/src/meta/model_v2/src/source.rs
@@ -16,13 +16,14 @@ use risingwave_pb::catalog::source::OptionalAssociatedTableId;
 use risingwave_pb::catalog::PbSource;
 use sea_orm::entity::prelude::*;
 use sea_orm::ActiveValue::Set;
+use serde::{Deserialize, Serialize};
 
 use crate::{
     ColumnCatalogArray, ConnectionId, I32Array, Property, SourceId, StreamSourceInfo, TableId,
     WatermarkDescArray,
 };
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "source")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/streaming_job.rs b/src/meta/model_v2/src/streaming_job.rs
index bbb6ac332f2e0..1ab9225f43cda 100644
--- a/src/meta/model_v2/src/streaming_job.rs
+++ b/src/meta/model_v2/src/streaming_job.rs
@@ -13,10 +13,11 @@
 // limitations under the License.
 
 use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
 
 use crate::{CreateType, JobStatus, StreamingParallelism};
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "streaming_job")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/subscription.rs b/src/meta/model_v2/src/subscription.rs
index 8a695c2b4c659..3d53c4b767804 100644
--- a/src/meta/model_v2/src/subscription.rs
+++ b/src/meta/model_v2/src/subscription.rs
@@ -15,22 +15,20 @@
 use risingwave_pb::catalog::PbSubscription;
 use sea_orm::entity::prelude::*;
 use sea_orm::ActiveValue::Set;
+use serde::{Deserialize, Serialize};
 
-use crate::{ColumnCatalogArray, ColumnOrderArray, I32Array, Property, SubscriptionId};
+use crate::SubscriptionId;
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "subscription")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
     pub subscription_id: SubscriptionId,
     pub name: String,
-    pub columns: ColumnCatalogArray,
-    pub plan_pk: ColumnOrderArray,
-    pub distribution_key: I32Array,
-    pub properties: Property,
+    pub retention_seconds: i64,
     pub definition: String,
-    pub subscription_from_name: String,
-    pub subscription_internal_table_name: Option<String>,
+    pub subscription_state: i32,
+    pub dependent_table_id: i32,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
@@ -58,13 +56,10 @@ impl From<PbSubscription> for ActiveModel {
         Self {
             subscription_id: Set(pb_subscription.id as _),
             name: Set(pb_subscription.name),
-            columns: Set(pb_subscription.column_catalogs.into()),
-            plan_pk: Set(pb_subscription.plan_pk.into()),
-            distribution_key: Set(pb_subscription.distribution_key.into()),
-            properties: Set(pb_subscription.properties.into()),
+            retention_seconds: Set(pb_subscription.retention_seconds as _),
             definition: Set(pb_subscription.definition),
-            subscription_from_name: Set(pb_subscription.subscription_from_name),
-            subscription_internal_table_name: Set(pb_subscription.subscription_internal_table_name),
+            subscription_state: Set(pb_subscription.subscription_state),
+            dependent_table_id: Set(pb_subscription.dependent_table_id as _),
         }
     }
 }
diff --git a/src/meta/model_v2/src/system_parameter.rs b/src/meta/model_v2/src/system_parameter.rs
index f8447c57ab7a6..4e76e8aae7f96 100644
--- a/src/meta/model_v2/src/system_parameter.rs
+++ b/src/meta/model_v2/src/system_parameter.rs
@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use sea_orm::entity::prelude::*;
-
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+use serde::{Deserialize, Serialize};
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "system_parameter")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/table.rs b/src/meta/model_v2/src/table.rs
index 974f1c8defacd..75fb66fcb2a5e 100644
--- a/src/meta/model_v2/src/table.rs
+++ b/src/meta/model_v2/src/table.rs
@@ -18,13 +18,16 @@ use risingwave_pb::catalog::{PbHandleConflictBehavior, PbTable};
 use sea_orm::entity::prelude::*;
 use sea_orm::ActiveValue::Set;
 use sea_orm::NotSet;
+use serde::{Deserialize, Serialize};
 
 use crate::{
     Cardinality, ColumnCatalogArray, ColumnOrderArray, FragmentId, I32Array, ObjectId, SourceId,
     TableId, TableVersion,
 };
 
-#[derive(Clone, Debug, PartialEq, Hash, Copy, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(
+    Clone, Debug, PartialEq, Hash, Copy, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize,
+)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum TableType {
     #[sea_orm(string_value = "TABLE")]
@@ -60,7 +63,7 @@ impl From<PbTableType> for TableType {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum HandleConflictBehavior {
     #[sea_orm(string_value = "OVERWRITE")]
@@ -98,7 +101,7 @@ impl From<PbHandleConflictBehavior> for HandleConflictBehavior {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "table")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/user.rs b/src/meta/model_v2/src/user.rs
index f238683a83b02..7991d4f135881 100644
--- a/src/meta/model_v2/src/user.rs
+++ b/src/meta/model_v2/src/user.rs
@@ -16,10 +16,11 @@ use risingwave_pb::user::PbUserInfo;
 use sea_orm::entity::prelude::*;
 use sea_orm::ActiveValue::Set;
 use sea_orm::NotSet;
+use serde::{Deserialize, Serialize};
 
 use crate::{AuthInfo, UserId};
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "user")]
 pub struct Model {
     #[sea_orm(primary_key)]
diff --git a/src/meta/model_v2/src/user_privilege.rs b/src/meta/model_v2/src/user_privilege.rs
index 0ce0b78242cc7..48f9a38a5504c 100644
--- a/src/meta/model_v2/src/user_privilege.rs
+++ b/src/meta/model_v2/src/user_privilege.rs
@@ -14,10 +14,11 @@
 
 use risingwave_pb::user::grant_privilege::PbAction;
 use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
 
 use crate::{ObjectId, PrivilegeId, UserId};
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Debug, Hash, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum Action {
     #[sea_orm(string_value = "INSERT")]
@@ -69,7 +70,7 @@ impl From<Action> for PbAction {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "user_privilege")]
 pub struct Model {
     #[sea_orm(primary_key)]
diff --git a/src/meta/model_v2/src/view.rs b/src/meta/model_v2/src/view.rs
index 0e32cd2275150..54783f8ac387a 100644
--- a/src/meta/model_v2/src/view.rs
+++ b/src/meta/model_v2/src/view.rs
@@ -15,10 +15,11 @@
 use risingwave_pb::catalog::PbView;
 use sea_orm::entity::prelude::*;
 use sea_orm::ActiveValue::Set;
+use serde::{Deserialize, Serialize};
 
 use crate::{FieldArray, Property, ViewId};
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "view")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/model_v2/src/worker.rs b/src/meta/model_v2/src/worker.rs
index 8694bb1c6d20c..ee2ab9b22d3c8 100644
--- a/src/meta/model_v2/src/worker.rs
+++ b/src/meta/model_v2/src/worker.rs
@@ -16,10 +16,11 @@ use risingwave_pb::common::worker_node::PbState;
 use risingwave_pb::common::{PbWorkerNode, PbWorkerType};
 use sea_orm::entity::prelude::*;
 use sea_orm::ActiveValue::Set;
+use serde::{Deserialize, Serialize};
 
 use crate::{TransactionId, WorkerId};
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Debug, Hash, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum WorkerType {
     #[sea_orm(string_value = "FRONTEND")]
@@ -59,7 +60,7 @@ impl From<WorkerType> for PbWorkerType {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)]
+#[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
 #[sea_orm(rs_type = "String", db_type = "String(None)")]
 pub enum WorkerStatus {
     #[sea_orm(string_value = "STARTING")]
@@ -101,7 +102,7 @@ impl From<&PbWorkerNode> for ActiveModel {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "worker")]
 pub struct Model {
     #[sea_orm(primary_key)]
diff --git a/src/meta/model_v2/src/worker_property.rs b/src/meta/model_v2/src/worker_property.rs
index a206043532821..3ab8d411c8b5c 100644
--- a/src/meta/model_v2/src/worker_property.rs
+++ b/src/meta/model_v2/src/worker_property.rs
@@ -13,10 +13,11 @@
 // limitations under the License.
 
 use sea_orm::entity::prelude::*;
+use serde::{Deserialize, Serialize};
 
 use crate::{I32Array, WorkerId};
 
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)]
 #[sea_orm(table_name = "worker_property")]
 pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
diff --git a/src/meta/node/src/lib.rs b/src/meta/node/src/lib.rs
index 0330f2af1de3d..0e49f0805bf1b 100644
--- a/src/meta/node/src/lib.rs
+++ b/src/meta/node/src/lib.rs
@@ -76,7 +76,7 @@ pub struct MetaNodeOpts {
 
     /// Endpoint of the SQL service, make it non-option when SQL service is required.
     #[clap(long, hide = true, env = "RW_SQL_ENDPOINT")]
-    pub sql_endpoint: Option<String>,
+    pub sql_endpoint: Option<Secret<String>>,
 
     /// The HTTP REST-API address of the Prometheus instance associated to this cluster.
     /// This address is used to serve `PromQL` queries to Prometheus.
@@ -221,7 +221,11 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
             },
             MetaBackend::Mem => MetaStoreBackend::Mem,
             MetaBackend::Sql => MetaStoreBackend::Sql {
-                endpoint: opts.sql_endpoint.expect("sql endpoint is required"),
+                endpoint: opts
+                    .sql_endpoint
+                    .expect("sql endpoint is required")
+                    .expose_secret()
+                    .to_string(),
             },
         };
 
@@ -256,25 +260,26 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
 
         const MIN_TIMEOUT_INTERVAL_SEC: u64 = 20;
         let compaction_task_max_progress_interval_secs = {
-            (config
-                .storage
-                .object_store
-                .object_store_read_timeout_ms
-                .max(config.storage.object_store.object_store_upload_timeout_ms)
-                .max(
-                    config
-                        .storage
-                        .object_store
-                        .object_store_streaming_read_timeout_ms,
-                )
-                .max(
-                    config
-                        .storage
-                        .object_store
-                        .object_store_streaming_upload_timeout_ms,
-                )
-                .max(config.meta.compaction_task_max_progress_interval_secs * 1000))
-                / 1000
+            let retry_config = &config.storage.object_store.retry;
+            let max_streming_read_timeout_ms = (retry_config.streaming_read_attempt_timeout_ms
+                + retry_config.req_backoff_max_delay_ms)
+                * retry_config.streaming_read_retry_attempts as u64;
+            let max_streaming_upload_timeout_ms = (retry_config
+                .streaming_upload_attempt_timeout_ms
+                + retry_config.req_backoff_max_delay_ms)
+                * retry_config.streaming_upload_retry_attempts as u64;
+            let max_upload_timeout_ms = (retry_config.upload_attempt_timeout_ms
+                + retry_config.req_backoff_max_delay_ms)
+                * retry_config.upload_retry_attempts as u64;
+            let max_read_timeout_ms = (retry_config.read_attempt_timeout_ms
+                + retry_config.req_backoff_max_delay_ms)
+                * retry_config.read_retry_attempts as u64;
+            let max_timeout_ms = max_streming_read_timeout_ms
+                .max(max_upload_timeout_ms)
+                .max(max_streaming_upload_timeout_ms)
+                .max(max_read_timeout_ms)
+                .max(config.meta.compaction_task_max_progress_interval_secs * 1000);
+            max_timeout_ms / 1000
         } + MIN_TIMEOUT_INTERVAL_SEC;
 
         let (mut join_handle, leader_lost_handle, shutdown_send) = rpc_serve(
@@ -338,6 +343,12 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
                 table_write_throughput_threshold: config.meta.table_write_throughput_threshold,
                 min_table_split_write_throughput: config.meta.min_table_split_write_throughput,
                 partition_vnode_count: config.meta.partition_vnode_count,
+                compact_task_table_size_partition_threshold_low: config
+                    .meta
+                    .compact_task_table_size_partition_threshold_low,
+                compact_task_table_size_partition_threshold_high: config
+                    .meta
+                    .compact_task_table_size_partition_threshold_high,
                 do_not_config_object_storage_lifecycle: config
                     .meta
                     .do_not_config_object_storage_lifecycle,
@@ -347,7 +358,7 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
                 compaction_task_max_progress_interval_secs,
                 compaction_config: Some(config.meta.compaction_config),
                 cut_table_size_limit: config.meta.cut_table_size_limit,
-                hybird_partition_vnode_count: config.meta.hybird_partition_vnode_count,
+                hybrid_partition_node_count: config.meta.hybrid_partition_vnode_count,
                 event_log_enabled: config.meta.event_log_enabled,
                 event_log_channel_max_size: config.meta.event_log_channel_max_size,
                 advertise_addr: opts.advertise_addr,
@@ -363,6 +374,12 @@ pub fn start(opts: MetaNodeOpts) -> Pin<Box<dyn Future<Output = ()> + Send>> {
                     .enable_check_task_level_overlap,
                 enable_dropped_column_reclaim: config.meta.enable_dropped_column_reclaim,
                 object_store_config: config.storage.object_store,
+                max_trivial_move_task_count_per_loop: config
+                    .meta
+                    .developer
+                    .max_trivial_move_task_count_per_loop,
+                max_get_task_probe_times: config.meta.developer.max_get_task_probe_times,
+                secret_store_private_key: config.meta.secret_store_private_key,
             },
             config.system.into_init_system_params(),
             Default::default(),
diff --git a/src/meta/node/src/server.rs b/src/meta/node/src/server.rs
index bf5bb72ed731b..e8b738305dce7 100644
--- a/src/meta/node/src/server.rs
+++ b/src/meta/node/src/server.rs
@@ -25,7 +25,7 @@ use risingwave_common::monitor::connection::{RouterExt, TcpConfig};
 use risingwave_common::session_config::SessionConfig;
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::telemetry::manager::TelemetryManager;
-use risingwave_common::telemetry::telemetry_env_enabled;
+use risingwave_common::telemetry::{report_scarf_enabled, report_to_scarf, telemetry_env_enabled};
 use risingwave_common_service::metrics_manager::MetricsManager;
 use risingwave_common_service::tracing::TracingExtractLayer;
 use risingwave_meta::barrier::StreamRpcManager;
@@ -587,6 +587,11 @@ pub async fn start_service_as_election_leader(
         .unwrap(),
     );
 
+    hummock_manager
+        .may_fill_backward_state_table_info()
+        .await
+        .unwrap();
+
     // Initialize services.
     let backup_manager = BackupManager::new(
         env.clone(),
@@ -654,7 +659,7 @@ pub async fn start_service_as_election_leader(
     );
     let health_srv = HealthServiceImpl::new();
     let backup_srv = BackupServiceImpl::new(backup_manager);
-    let telemetry_srv = TelemetryInfoServiceImpl::new(env.meta_store_ref());
+    let telemetry_srv = TelemetryInfoServiceImpl::new(env.meta_store());
     let system_params_srv = SystemParamsServiceImpl::new(env.system_params_manager_impl_ref());
     let session_params_srv = SessionParamsServiceImpl::new(env.session_params_manager_impl_ref());
     let serving_srv =
@@ -754,6 +759,11 @@ pub async fn start_service_as_election_leader(
     } else {
         tracing::info!("Telemetry didn't start due to meta backend or config");
     }
+    if report_scarf_enabled() {
+        tokio::spawn(report_to_scarf());
+    } else {
+        tracing::info!("Scarf reporting is disabled");
+    };
 
     if let Some(pair) = env.event_log_manager_ref().take_join_handle() {
         sub_tasks.push(pair);
diff --git a/src/meta/service/src/cloud_service.rs b/src/meta/service/src/cloud_service.rs
index b77b751b281e7..5777f1c405009 100644
--- a/src/meta/service/src/cloud_service.rs
+++ b/src/meta/service/src/cloud_service.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::BTreeMap;
 use std::sync::LazyLock;
 
 use async_trait::async_trait;
@@ -147,7 +147,6 @@ impl CloudService for CloudServiceImpl {
             }
         }
         // try fetch kafka metadata, return error message on failure
-        let source_cfg: HashMap<String, String> = source_cfg.into_iter().collect();
         let props = ConnectorProperties::extract(source_cfg, false);
         if let Err(e) = props {
             return Ok(new_rwc_validate_fail_response(
diff --git a/src/meta/service/src/cluster_service.rs b/src/meta/service/src/cluster_service.rs
index d69debbe90816..7a81098cdc67b 100644
--- a/src/meta/service/src/cluster_service.rs
+++ b/src/meta/service/src/cluster_service.rs
@@ -145,7 +145,7 @@ impl ClusterService for ClusterServiceImpl {
                 let _ = mgr.cluster_manager.delete_worker_node(host).await?;
             }
             MetadataManager::V2(mgr) => {
-                let _ = mgr.cluster_controller.delete_worker(host).await?;
+                mgr.cluster_controller.delete_worker(host).await?;
             }
         }
 
diff --git a/src/meta/service/src/ddl_service.rs b/src/meta/service/src/ddl_service.rs
index f5012e5796e69..2e4ba23e02d8f 100644
--- a/src/meta/service/src/ddl_service.rs
+++ b/src/meta/service/src/ddl_service.rs
@@ -25,7 +25,7 @@ use risingwave_pb::catalog::connection::private_link_service::{
 };
 use risingwave_pb::catalog::connection::PbPrivateLinkService;
 use risingwave_pb::catalog::table::OptionalAssociatedSourceId;
-use risingwave_pb::catalog::{connection, Comment, Connection, CreateType};
+use risingwave_pb::catalog::{connection, Comment, Connection, CreateType, Secret};
 use risingwave_pb::ddl_service::ddl_service_server::DdlService;
 use risingwave_pb::ddl_service::drop_table_request::PbSourceId;
 use risingwave_pb::ddl_service::*;
@@ -148,6 +148,41 @@ impl DdlService for DdlServiceImpl {
         }))
     }
 
+    async fn create_secret(
+        &self,
+        request: Request<CreateSecretRequest>,
+    ) -> Result<Response<CreateSecretResponse>, Status> {
+        let req = request.into_inner();
+        let pb_secret = Secret {
+            id: 0,
+            name: req.get_name().clone(),
+            database_id: req.get_database_id(),
+            value: req.get_value().clone(),
+            owner: req.get_owner_id(),
+            schema_id: req.get_schema_id(),
+        };
+        let version = self
+            .ddl_controller
+            .run_command(DdlCommand::CreateSecret(pb_secret))
+            .await?;
+
+        Ok(Response::new(CreateSecretResponse { version }))
+    }
+
+    async fn drop_secret(
+        &self,
+        request: Request<DropSecretRequest>,
+    ) -> Result<Response<DropSecretResponse>, Status> {
+        let req = request.into_inner();
+        let secret_id = req.get_secret_id();
+        let version = self
+            .ddl_controller
+            .run_command(DdlCommand::DropSecret(secret_id))
+            .await?;
+
+        Ok(Response::new(DropSecretResponse { version }))
+    }
+
     async fn create_schema(
         &self,
         request: Request<CreateSchemaRequest>,
@@ -320,16 +355,7 @@ impl DdlService for DdlServiceImpl {
         let req = request.into_inner();
 
         let subscription = req.get_subscription()?.clone();
-        let fragment_graph = req.get_fragment_graph()?.clone();
-
-        let stream_job = StreamingJob::Subscription(subscription);
-
-        let command = DdlCommand::CreateStreamingJob(
-            stream_job,
-            fragment_graph,
-            CreateType::Foreground,
-            None,
-        );
+        let command = DdlCommand::CreateSubscription(subscription);
 
         let version = self.ddl_controller.run_command(command).await?;
 
@@ -347,11 +373,7 @@ impl DdlService for DdlServiceImpl {
         let subscription_id = request.subscription_id;
         let drop_mode = DropMode::from_request_setting(request.cascade);
 
-        let command = DdlCommand::DropStreamingJob(
-            StreamingJobId::Subscription(subscription_id),
-            drop_mode,
-            None,
-        );
+        let command = DdlCommand::DropSubscription(subscription_id, drop_mode);
 
         let version = self.ddl_controller.run_command(command).await?;
 
diff --git a/src/meta/service/src/hummock_service.rs b/src/meta/service/src/hummock_service.rs
index d2991d4a005d6..1e46438cb8ddd 100644
--- a/src/meta/service/src/hummock_service.rs
+++ b/src/meta/service/src/hummock_service.rs
@@ -666,6 +666,22 @@ impl HummockManagerService for HummockServiceImpl {
         let response = Response::new(CancelCompactTaskResponse { ret });
         return Ok(response);
     }
+
+    async fn list_change_log_epochs(
+        &self,
+        request: Request<ListChangeLogEpochsRequest>,
+    ) -> Result<Response<ListChangeLogEpochsResponse>, Status> {
+        let ListChangeLogEpochsRequest {
+            table_id,
+            min_epoch,
+            max_count,
+        } = request.into_inner();
+        let epochs = self
+            .hummock_manager
+            .list_change_log_epochs(table_id, min_epoch, max_count)
+            .await;
+        Ok(Response::new(ListChangeLogEpochsResponse { epochs }))
+    }
 }
 
 #[cfg(test)]
diff --git a/src/meta/service/src/notification_service.rs b/src/meta/service/src/notification_service.rs
index e9a5e4a017ad4..e4a8d298e0788 100644
--- a/src/meta/service/src/notification_service.rs
+++ b/src/meta/service/src/notification_service.rs
@@ -24,7 +24,7 @@ use risingwave_pb::hummock::WriteLimits;
 use risingwave_pb::meta::meta_snapshot::SnapshotVersion;
 use risingwave_pb::meta::notification_service_server::NotificationService;
 use risingwave_pb::meta::{
-    FragmentParallelUnitMapping, GetSessionParamsResponse, MetaSnapshot, SubscribeRequest,
+    FragmentWorkerSlotMapping, GetSessionParamsResponse, MetaSnapshot, SubscribeRequest,
     SubscribeType,
 };
 use risingwave_pb::user::UserInfo;
@@ -80,6 +80,7 @@ impl NotificationServiceImpl {
                     views,
                     functions,
                     connections,
+                    secrets,
                 ) = catalog_guard.database.get_catalog();
                 let users = catalog_guard.user.list_users();
                 let notification_version = self.env.notification_manager().current_version().await;
@@ -95,6 +96,7 @@ impl NotificationServiceImpl {
                         views,
                         functions,
                         connections,
+                        secrets,
                     ),
                     users,
                     notification_version,
@@ -114,6 +116,7 @@ impl NotificationServiceImpl {
                         views,
                         functions,
                         connections,
+                        secrets,
                     ),
                     users,
                 ) = catalog_guard.snapshot().await?;
@@ -130,6 +133,7 @@ impl NotificationServiceImpl {
                         views,
                         functions,
                         connections,
+                        secrets,
                     ),
                     users,
                     notification_version,
@@ -138,9 +142,9 @@ impl NotificationServiceImpl {
         }
     }
 
-    async fn get_parallel_unit_mapping_snapshot(
+    async fn get_worker_slot_mapping_snapshot(
         &self,
-    ) -> MetaResult<(Vec<FragmentParallelUnitMapping>, NotificationVersion)> {
+    ) -> MetaResult<(Vec<FragmentWorkerSlotMapping>, NotificationVersion)> {
         match &self.metadata_manager {
             MetadataManager::V1(mgr) => {
                 let fragment_guard = mgr.fragment_manager.get_fragment_read_guard().await;
@@ -161,11 +165,11 @@ impl NotificationServiceImpl {
         }
     }
 
-    fn get_serving_vnode_mappings(&self) -> Vec<FragmentParallelUnitMapping> {
+    fn get_serving_vnode_mappings(&self) -> Vec<FragmentWorkerSlotMapping> {
         self.serving_vnode_mapping
             .all()
             .iter()
-            .map(|(fragment_id, mapping)| FragmentParallelUnitMapping {
+            .map(|(fragment_id, mapping)| FragmentWorkerSlotMapping {
                 fragment_id: *fragment_id,
                 mapping: Some(mapping.to_protobuf()),
             })
@@ -237,13 +241,16 @@ impl NotificationServiceImpl {
                 views,
                 functions,
                 connections,
+                secrets,
             ),
             users,
             catalog_version,
         ) = self.get_catalog_snapshot().await?;
-        let (parallel_unit_mappings, parallel_unit_mapping_version) =
-            self.get_parallel_unit_mapping_snapshot().await?;
-        let serving_parallel_unit_mappings = self.get_serving_vnode_mappings();
+
+        let (streaming_worker_slot_mappings, streaming_worker_slot_mapping_version) =
+            self.get_worker_slot_mapping_snapshot().await?;
+        let serving_worker_slot_mappings = self.get_serving_vnode_mappings();
+
         let (nodes, worker_node_version) = self.get_worker_node_snapshot().await?;
 
         let hummock_snapshot = Some(self.hummock_manager.latest_snapshot());
@@ -269,16 +276,17 @@ impl NotificationServiceImpl {
             subscriptions,
             functions,
             connections,
+            secrets,
             users,
-            parallel_unit_mappings,
             nodes,
             hummock_snapshot,
-            serving_parallel_unit_mappings,
             version: Some(SnapshotVersion {
                 catalog_version,
-                parallel_unit_mapping_version,
                 worker_node_version,
+                streaming_worker_slot_mapping_version,
             }),
+            serving_worker_slot_mappings,
+            streaming_worker_slot_mappings,
             session_params,
             ..Default::default()
         })
diff --git a/src/meta/service/src/serving_service.rs b/src/meta/service/src/serving_service.rs
index d1b013e078e0f..92c5fd9db4837 100644
--- a/src/meta/service/src/serving_service.rs
+++ b/src/meta/service/src/serving_service.rs
@@ -16,7 +16,7 @@ use itertools::Itertools;
 use risingwave_meta::manager::MetadataManager;
 use risingwave_pb::meta::serving_service_server::ServingService;
 use risingwave_pb::meta::{
-    FragmentParallelUnitMapping, GetServingVnodeMappingsRequest, GetServingVnodeMappingsResponse,
+    FragmentWorkerSlotMapping, GetServingVnodeMappingsRequest, GetServingVnodeMappingsResponse,
 };
 use tonic::{Request, Response, Status};
 
@@ -49,7 +49,7 @@ impl ServingService for ServingServiceImpl {
             .serving_vnode_mapping
             .all()
             .into_iter()
-            .map(|(fragment_id, mapping)| FragmentParallelUnitMapping {
+            .map(|(fragment_id, mapping)| FragmentWorkerSlotMapping {
                 fragment_id,
                 mapping: Some(mapping.to_protobuf()),
             })
@@ -78,8 +78,8 @@ impl ServingService for ServingServiceImpl {
             }
         };
         Ok(Response::new(GetServingVnodeMappingsResponse {
-            mappings,
             fragment_to_table,
+            worker_slot_mappings: mappings,
         }))
     }
 }
diff --git a/src/meta/service/src/session_config.rs b/src/meta/service/src/session_config.rs
index a4a7d1b70591a..a3999a3d27426 100644
--- a/src/meta/service/src/session_config.rs
+++ b/src/meta/service/src/session_config.rs
@@ -19,7 +19,6 @@ use risingwave_pb::meta::{
     GetSessionParamsRequest, GetSessionParamsResponse, SetSessionParamRequest,
     SetSessionParamResponse,
 };
-use serde_json;
 use thiserror_ext::AsReport;
 use tonic::{Request, Response, Status};
 
diff --git a/src/meta/service/src/stream_service.rs b/src/meta/service/src/stream_service.rs
index cf9a8b1a3e48c..be520132b167f 100644
--- a/src/meta/service/src/stream_service.rs
+++ b/src/meta/service/src/stream_service.rs
@@ -108,7 +108,7 @@ impl StreamManagerService for StreamServiceImpl {
         let request = request.into_inner();
 
         let actor_to_apply = match request.kind() {
-            ThrottleTarget::Source => {
+            ThrottleTarget::Source | ThrottleTarget::TableWithSource => {
                 self.metadata_manager
                     .update_source_rate_limit_by_source_id(request.id as SourceId, request.rate)
                     .await?
diff --git a/src/meta/service/src/user_service.rs b/src/meta/service/src/user_service.rs
index d7dd4f5b4de4b..cab1b29228055 100644
--- a/src/meta/service/src/user_service.rs
+++ b/src/meta/service/src/user_service.rs
@@ -76,7 +76,7 @@ impl UserServiceImpl {
                     }
                     expanded_privileges.push(privilege);
                 }
-            } else if let Some(Object::AllDmlTablesSchemaId(schema_id)) = &privilege.object {
+            } else if let Some(Object::AllDmlRelationsSchemaId(schema_id)) = &privilege.object {
                 let tables = match &self.metadata_manager {
                     MetadataManager::V1(mgr) => {
                         mgr.catalog_manager.list_dml_table_ids(*schema_id).await
@@ -89,6 +89,16 @@ impl UserServiceImpl {
                         .map(|id| id as _)
                         .collect(),
                 };
+                let views = match &self.metadata_manager {
+                    MetadataManager::V1(mgr) => mgr.catalog_manager.list_view_ids(*schema_id).await,
+                    MetadataManager::V2(mgr) => mgr
+                        .catalog_controller
+                        .list_view_ids(*schema_id as _)
+                        .await?
+                        .into_iter()
+                        .map(|id| id as _)
+                        .collect(),
+                };
                 for table_id in tables {
                     let mut privilege = privilege.clone();
                     privilege.object = Some(Object::TableId(table_id));
@@ -100,6 +110,17 @@ impl UserServiceImpl {
                     }
                     expanded_privileges.push(privilege);
                 }
+                for view_id in views {
+                    let mut privilege = privilege.clone();
+                    privilege.object = Some(Object::ViewId(view_id));
+                    if let Some(true) = with_grant_option {
+                        privilege
+                            .action_with_opts
+                            .iter_mut()
+                            .for_each(|p| p.with_grant_option = true);
+                    }
+                    expanded_privileges.push(privilege);
+                }
             } else if let Some(Object::AllSourcesSchemaId(schema_id)) = &privilege.object {
                 let sources = match &self.metadata_manager {
                     MetadataManager::V1(mgr) => {
diff --git a/src/meta/src/backup_restore/backup_manager.rs b/src/meta/src/backup_restore/backup_manager.rs
index 6fdcb1d13ba3b..1b3dd61152278 100644
--- a/src/meta/src/backup_restore/backup_manager.rs
+++ b/src/meta/src/backup_restore/backup_manager.rs
@@ -31,11 +31,11 @@ use risingwave_pb::meta::subscribe_response::{Info, Operation};
 use thiserror_ext::AsReport;
 use tokio::task::JoinHandle;
 
-use crate::backup_restore::meta_snapshot_builder;
 use crate::backup_restore::metrics::BackupManagerMetrics;
+use crate::backup_restore::{meta_snapshot_builder, meta_snapshot_builder_v2};
 use crate::hummock::sequence::next_meta_backup_id;
 use crate::hummock::{HummockManagerRef, HummockVersionSafePoint};
-use crate::manager::{LocalNotification, MetaSrvEnv};
+use crate::manager::{LocalNotification, MetaSrvEnv, MetaStoreImpl};
 use crate::rpc::metrics::MetaMetrics;
 use crate::MetaResult;
 
@@ -345,23 +345,41 @@ impl BackupWorker {
     fn start(self, job_id: u64, remarks: Option<String>) -> JoinHandle<()> {
         let backup_manager_clone = self.backup_manager.clone();
         let job = async move {
-            let mut snapshot_builder = meta_snapshot_builder::MetaSnapshotV1Builder::new(
-                backup_manager_clone.env.meta_store().as_kv().clone(),
-            );
-            // Reuse job id as snapshot id.
             let hummock_manager = backup_manager_clone.hummock_manager.clone();
-            snapshot_builder
-                .build(job_id, async move {
-                    hummock_manager.get_current_version().await
-                })
-                .await?;
-            let snapshot = snapshot_builder.finish()?;
-            backup_manager_clone
-                .backup_store
-                .load()
-                .0
-                .create(&snapshot, remarks)
-                .await?;
+            let hummock_version_builder =
+                async move { hummock_manager.get_current_version().await };
+            match backup_manager_clone.env.meta_store() {
+                MetaStoreImpl::Kv(kv) => {
+                    let mut snapshot_builder =
+                        meta_snapshot_builder::MetaSnapshotV1Builder::new(kv.clone());
+                    // Reuse job id as snapshot id.
+                    snapshot_builder
+                        .build(job_id, hummock_version_builder)
+                        .await?;
+                    let snapshot = snapshot_builder.finish()?;
+                    backup_manager_clone
+                        .backup_store
+                        .load()
+                        .0
+                        .create(&snapshot, remarks)
+                        .await?;
+                }
+                MetaStoreImpl::Sql(sql) => {
+                    let mut snapshot_builder =
+                        meta_snapshot_builder_v2::MetaSnapshotV2Builder::new(sql.clone());
+                    // Reuse job id as snapshot id.
+                    snapshot_builder
+                        .build(job_id, hummock_version_builder)
+                        .await?;
+                    let snapshot = snapshot_builder.finish()?;
+                    backup_manager_clone
+                        .backup_store
+                        .load()
+                        .0
+                        .create(&snapshot, remarks)
+                        .await?;
+                }
+            }
             Ok(BackupJobResult::Succeeded)
         };
         tokio::spawn(async move {
@@ -383,7 +401,7 @@ async fn create_snapshot_store(
             &config.0,
             metric,
             "Meta Backup",
-            object_store_config.clone(),
+            Arc::new(object_store_config.clone()),
         )
         .await,
     );
diff --git a/src/meta/src/backup_restore/meta_snapshot_builder.rs b/src/meta/src/backup_restore/meta_snapshot_builder.rs
index 323f49b68663e..bb8a1eb919fdc 100644
--- a/src/meta/src/backup_restore/meta_snapshot_builder.rs
+++ b/src/meta/src/backup_restore/meta_snapshot_builder.rs
@@ -187,10 +187,8 @@ mod tests {
         let meta_store = MemStore::new();
 
         let mut builder = MetaSnapshotBuilder::new(meta_store.clone());
-        let hummock_version = HummockVersion {
-            id: 1,
-            ..Default::default()
-        };
+        let mut hummock_version = HummockVersion::default();
+        hummock_version.id = 1;
         let get_ckpt_builder = |v: &HummockVersion| {
             let v_ = v.clone();
             async move { v_ }
diff --git a/src/meta/src/backup_restore/meta_snapshot_builder_v2.rs b/src/meta/src/backup_restore/meta_snapshot_builder_v2.rs
index 107db4d271194..9e4ad6a0c05a1 100644
--- a/src/meta/src/backup_restore/meta_snapshot_builder_v2.rs
+++ b/src/meta/src/backup_restore/meta_snapshot_builder_v2.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![expect(dead_code, reason = "WIP")]
-
 use std::future::Future;
 
 use itertools::Itertools;
@@ -23,12 +21,35 @@ use risingwave_backup::MetaSnapshotId;
 use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
 use risingwave_meta_model_v2 as model_v2;
 use risingwave_pb::hummock::PbHummockVersionDelta;
-use sea_orm::{EntityTrait, QueryOrder, TransactionTrait};
+use sea_orm::{DbErr, EntityTrait, QueryOrder, TransactionTrait};
 
 use crate::controller::SqlMetaStore;
 
 const VERSION: u32 = 2;
 
+fn map_db_err(e: DbErr) -> BackupError {
+    BackupError::MetaStorage(e.into())
+}
+
+macro_rules! define_set_metadata {
+    ($( {$name:ident, $mod_path:ident::$mod_name:ident} ),*) => {
+        async fn set_metadata(
+            metadata: &mut MetadataV2,
+            txn: &sea_orm::DatabaseTransaction,
+        ) -> BackupResult<()> {
+          $(
+              metadata.$name = $mod_path::$mod_name::Entity::find()
+                                    .all(txn)
+                                    .await
+                                    .map_err(map_db_err)?;
+          )*
+          Ok(())
+        }
+    };
+}
+
+risingwave_backup::for_all_metadata_models_v2!(define_set_metadata);
+
 pub struct MetaSnapshotV2Builder {
     snapshot: MetaSnapshotV2,
     meta_store: SqlMetaStore,
@@ -61,12 +82,12 @@ impl MetaSnapshotV2Builder {
                 Some(sea_orm::AccessMode::ReadOnly),
             )
             .await
-            .map_err(|e| BackupError::MetaStorage(e.into()))?;
+            .map_err(map_db_err)?;
         let version_deltas = model_v2::prelude::HummockVersionDelta::find()
             .order_by_asc(model_v2::hummock_version_delta::Column::Id)
             .all(&txn)
             .await
-            .map_err(|e| BackupError::MetaStorage(e.into()))?
+            .map_err(map_db_err)?
             .into_iter()
             .map_into::<PbHummockVersionDelta>()
             .map(|pb_delta| HummockVersionDelta::from_persisted_protobuf(&pb_delta));
@@ -89,30 +110,14 @@ impl MetaSnapshotV2Builder {
             }
             redo_state
         };
-        let version_stats = model_v2::prelude::HummockVersionStats::find_by_id(
-            hummock_version.id as model_v2::HummockVersionId,
-        )
-        .one(&txn)
-        .await
-        .map_err(|e| BackupError::MetaStorage(e.into()))?
-        .unwrap_or_else(|| panic!("version stats for version {} not found", hummock_version.id));
-        let compaction_configs = model_v2::prelude::CompactionConfig::find()
-            .all(&txn)
-            .await
-            .map_err(|e| BackupError::MetaStorage(e.into()))?;
-
-        // TODO: other metadata
-        let cluster_id = "TODO".to_string();
-
-        txn.commit()
-            .await
-            .map_err(|e| BackupError::MetaStorage(e.into()))?;
-        self.snapshot.metadata = MetadataV2 {
-            cluster_id,
+        let mut metadata = MetadataV2 {
             hummock_version,
-            version_stats,
-            compaction_configs,
+            ..Default::default()
         };
+        set_metadata(&mut metadata, &txn).await?;
+
+        txn.commit().await.map_err(map_db_err)?;
+        self.snapshot.metadata = metadata;
         Ok(())
     }
 
diff --git a/src/meta/src/backup_restore/restore.rs b/src/meta/src/backup_restore/restore.rs
index dfc4f37234b37..1ba13c8cbf8bf 100644
--- a/src/meta/src/backup_restore/restore.rs
+++ b/src/meta/src/backup_restore/restore.rs
@@ -41,6 +41,8 @@ pub struct RestoreOpts {
     /// Type of meta store to restore.
     #[clap(long, value_enum, default_value_t = MetaBackend::Etcd)]
     pub meta_store_type: MetaBackend,
+    #[clap(long, default_value_t = String::from(""))]
+    pub sql_endpoint: String,
     /// Etcd endpoints.
     #[clap(long, default_value_t = String::from(""))]
     pub etcd_endpoints: String,
@@ -80,7 +82,7 @@ async fn restore_hummock_version(
             hummock_storage_url,
             Arc::new(ObjectStoreMetrics::unused()),
             "Version Checkpoint",
-            ObjectStoreConfig::default(),
+            Arc::new(ObjectStoreConfig::default()),
         )
         .await,
     );
@@ -140,7 +142,7 @@ async fn restore_impl(
     match &meta_store {
         MetaStoreBackendImpl::Sql(m) => {
             if format_version < 2 {
-                todo!("write model V1 to meta store V2");
+                unimplemented!("not supported: write model V1 to meta store V2");
             } else {
                 dispatch(
                     target_id,
@@ -227,6 +229,7 @@ mod tests {
         RestoreOpts {
             meta_snapshot_id: 1,
             meta_store_type: MetaBackend::Mem,
+            sql_endpoint: "".to_string(),
             etcd_endpoints: "".to_string(),
             etcd_auth: false,
             etcd_username: "".to_string(),
@@ -263,9 +266,10 @@ mod tests {
         let snapshot = MetaSnapshot {
             id: opts.meta_snapshot_id,
             metadata: ClusterMetadata {
-                hummock_version: HummockVersion {
-                    id: 123,
-                    ..Default::default()
+                hummock_version: {
+                    let mut version = HummockVersion::default();
+                    version.id = 123;
+                    version
                 },
                 system_param: system_param.clone(),
                 ..Default::default()
@@ -445,9 +449,10 @@ mod tests {
                         memcomparable::to_vec(&"some_value_2".to_string()).unwrap(),
                     ),
                 ]),
-                hummock_version: HummockVersion {
-                    id: 123,
-                    ..Default::default()
+                hummock_version: {
+                    let mut version = HummockVersion::default();
+                    version.id = 123;
+                    version
                 },
                 system_param: system_param.clone(),
                 ..Default::default()
diff --git a/src/meta/src/backup_restore/restore_impl/v2.rs b/src/meta/src/backup_restore/restore_impl/v2.rs
index 5195845f9e252..a431b455063bb 100644
--- a/src/meta/src/backup_restore/restore_impl/v2.rs
+++ b/src/meta/src/backup_restore/restore_impl/v2.rs
@@ -12,13 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::iter;
-
 use risingwave_backup::error::{BackupError, BackupResult};
 use risingwave_backup::meta_snapshot::MetaSnapshot;
 use risingwave_backup::meta_snapshot_v2::{MetaSnapshotV2, MetadataV2};
 use risingwave_backup::storage::{MetaSnapshotStorage, MetaSnapshotStorageRef};
 use risingwave_backup::MetaSnapshotId;
+use sea_orm::{DatabaseBackend, DbBackend, DbErr, Statement};
 
 use crate::backup_restore::restore_impl::{Loader, Writer};
 use crate::controller::SqlMetaStore;
@@ -36,13 +35,51 @@ impl LoaderV2 {
 #[async_trait::async_trait]
 impl Loader<MetadataV2> for LoaderV2 {
     async fn load(&self, target_id: MetaSnapshotId) -> BackupResult<MetaSnapshot<MetadataV2>> {
-        let target_snapshot: MetaSnapshotV2 = self.backup_store.get(target_id).await?;
+        let snapshot_list = &self.backup_store.manifest().snapshot_metadata;
+        let mut target_snapshot: MetaSnapshotV2 = self.backup_store.get(target_id).await?;
         tracing::info!(
             "snapshot {} before rewrite:\n{}",
             target_id,
             target_snapshot
         );
-        todo!("validate and rewrite seq")
+        let newest_id = snapshot_list
+            .iter()
+            .map(|m| m.id)
+            .max()
+            .expect("should exist");
+        assert!(
+            newest_id >= target_id,
+            "newest_id={}, target_id={}",
+            newest_id,
+            target_id
+        );
+
+        // validate and rewrite seq
+        if newest_id > target_id {
+            let newest_snapshot: MetaSnapshotV2 = self.backup_store.get(newest_id).await?;
+            for seq in &target_snapshot.metadata.hummock_sequences {
+                let newest = newest_snapshot
+                    .metadata
+                    .hummock_sequences
+                    .iter()
+                    .find(|s| s.name == seq.name)
+                    .unwrap_or_else(|| {
+                        panic!(
+                            "violate superset requirement. Hummock sequence name {}",
+                            seq.name
+                        )
+                    });
+                assert!(newest.seq >= seq.seq, "violate monotonicity requirement");
+            }
+            target_snapshot.metadata.hummock_sequences = newest_snapshot.metadata.hummock_sequences;
+            tracing::info!(
+                "snapshot {} after rewrite by snapshot {}:\n{}",
+                target_id,
+                newest_id,
+                target_snapshot,
+            );
+        }
+        Ok(target_snapshot)
     }
 }
 
@@ -61,10 +98,100 @@ impl Writer<MetadataV2> for WriterModelV2ToMetaStoreV2 {
     async fn write(&self, target_snapshot: MetaSnapshot<MetadataV2>) -> BackupResult<()> {
         let metadata = target_snapshot.metadata;
         let db = &self.meta_store.conn;
-        insert_models(iter::once(metadata.version_stats), db).await?;
-        insert_models(metadata.compaction_configs, db).await?;
-        todo!("write other metadata")
+        insert_models(metadata.seaql_migrations.clone(), db).await?;
+        insert_models(metadata.clusters.clone(), db).await?;
+        insert_models(metadata.version_stats.clone(), db).await?;
+        insert_models(metadata.compaction_configs.clone(), db).await?;
+        insert_models(metadata.hummock_sequences.clone(), db).await?;
+        insert_models(metadata.workers.clone(), db).await?;
+        insert_models(metadata.worker_properties.clone(), db).await?;
+        insert_models(metadata.users.clone(), db).await?;
+        insert_models(metadata.user_privileges.clone(), db).await?;
+        insert_models(metadata.objects.clone(), db).await?;
+        insert_models(metadata.object_dependencies.clone(), db).await?;
+        insert_models(metadata.databases.clone(), db).await?;
+        insert_models(metadata.schemas.clone(), db).await?;
+        insert_models(metadata.streaming_jobs.clone(), db).await?;
+        insert_models(metadata.fragments.clone(), db).await?;
+        insert_models(metadata.actors.clone(), db).await?;
+        insert_models(metadata.actor_dispatchers.clone(), db).await?;
+        insert_models(metadata.connections.clone(), db).await?;
+        insert_models(metadata.sources.clone(), db).await?;
+        insert_models(metadata.tables.clone(), db).await?;
+        insert_models(metadata.sinks.clone(), db).await?;
+        insert_models(metadata.views.clone(), db).await?;
+        insert_models(metadata.indexes.clone(), db).await?;
+        insert_models(metadata.functions.clone(), db).await?;
+        insert_models(metadata.system_parameters.clone(), db).await?;
+        insert_models(metadata.catalog_versions.clone(), db).await?;
+        insert_models(metadata.subscriptions.clone(), db).await?;
+        insert_models(metadata.session_parameters.clone(), db).await?;
+        insert_models(metadata.secrets.clone(), db).await?;
+        // update_auto_inc must be called last.
+        update_auto_inc(&metadata, db).await?;
+        Ok(())
+    }
+}
+
+fn map_db_err(e: DbErr) -> BackupError {
+    BackupError::MetaStorage(e.into())
+}
+
+// TODO: the code snippet is similar to the one found in migration.rs
+async fn update_auto_inc(
+    metadata: &MetadataV2,
+    db: &impl sea_orm::ConnectionTrait,
+) -> BackupResult<()> {
+    match db.get_database_backend() {
+        DbBackend::MySql => {
+            if let Some(next_worker_id) = metadata.workers.iter().map(|w| w.worker_id + 1).max() {
+                db.execute(Statement::from_string(
+                    DatabaseBackend::MySql,
+                    format!("ALTER TABLE worker AUTO_INCREMENT = {next_worker_id};"),
+                ))
+                .await
+                .map_err(map_db_err)?;
+            }
+            if let Some(next_object_id) = metadata.objects.iter().map(|o| o.oid + 1).max() {
+                db.execute(Statement::from_string(
+                    DatabaseBackend::MySql,
+                    format!("ALTER TABLE object AUTO_INCREMENT = {next_object_id};"),
+                ))
+                .await
+                .map_err(map_db_err)?;
+            }
+            if let Some(next_user_id) = metadata.users.iter().map(|u| u.user_id + 1).max() {
+                db.execute(Statement::from_string(
+                    DatabaseBackend::MySql,
+                    format!("ALTER TABLE user AUTO_INCREMENT = {next_user_id};"),
+                ))
+                .await
+                .map_err(map_db_err)?;
+            }
+        }
+        DbBackend::Postgres => {
+            db.execute(Statement::from_string(
+                DatabaseBackend::Postgres,
+                "SELECT setval('worker_worker_id_seq', (SELECT MAX(worker_id) FROM worker));",
+            ))
+            .await
+            .map_err(map_db_err)?;
+            db.execute(Statement::from_string(
+                DatabaseBackend::Postgres,
+                "SELECT setval('object_oid_seq', (SELECT MAX(oid) FROM object) + 1);",
+            ))
+            .await
+            .map_err(map_db_err)?;
+            db.execute(Statement::from_string(
+                DatabaseBackend::Postgres,
+                "SELECT setval('user_user_id_seq', (SELECT MAX(user_id) FROM \"user\") + 1);",
+            ))
+            .await
+            .map_err(map_db_err)?;
+        }
+        DbBackend::Sqlite => {}
     }
+    Ok(())
 }
 
 async fn insert_models<S, A>(
@@ -81,16 +208,13 @@ where
     if <S as sea_orm::ModelTrait>::Entity::find()
         .one(db)
         .await
-        .map_err(|e| BackupError::MetaStorage(e.into()))?
+        .map_err(map_db_err)?
         .is_some()
     {
         return Err(BackupError::NonemptyMetaStorage);
     }
     for m in models {
-        m.into_active_model()
-            .insert(db)
-            .await
-            .map_err(|e| BackupError::MetaStorage(e.into()))?;
+        m.into_active_model().insert(db).await.map_err(map_db_err)?;
     }
     Ok(())
 }
diff --git a/src/meta/src/backup_restore/utils.rs b/src/meta/src/backup_restore/utils.rs
index 48eec28f4ffea..178a8ac3e00c7 100644
--- a/src/meta/src/backup_restore/utils.rs
+++ b/src/meta/src/backup_restore/utils.rs
@@ -17,11 +17,12 @@ use std::time::Duration;
 
 use anyhow::Context;
 use etcd_client::ConnectOptions;
-use risingwave_backup::error::BackupResult;
+use risingwave_backup::error::{BackupError, BackupResult};
 use risingwave_backup::storage::{MetaSnapshotStorageRef, ObjectStoreMetaSnapshotStorage};
 use risingwave_common::config::{MetaBackend, ObjectStoreConfig};
 use risingwave_object_store::object::build_remote_object_store;
 use risingwave_object_store::object::object_metrics::ObjectStoreMetrics;
+use sea_orm::DbBackend;
 
 use crate::backup_restore::RestoreOpts;
 use crate::controller::SqlMetaStore;
@@ -32,7 +33,6 @@ use crate::MetaStoreBackend;
 pub enum MetaStoreBackendImpl {
     Etcd(EtcdMetaStore),
     Mem(MemStore),
-    #[expect(dead_code, reason = "WIP")]
     Sql(SqlMetaStore),
 }
 
@@ -62,7 +62,9 @@ pub async fn get_meta_store(opts: RestoreOpts) -> BackupResult<MetaStoreBackendI
             },
         },
         MetaBackend::Mem => MetaStoreBackend::Mem,
-        MetaBackend::Sql => panic!("not supported"),
+        MetaBackend::Sql => MetaStoreBackend::Sql {
+            endpoint: opts.sql_endpoint,
+        },
     };
     match meta_store_backend {
         MetaStoreBackend::Etcd {
@@ -80,7 +82,25 @@ pub async fn get_meta_store(opts: RestoreOpts) -> BackupResult<MetaStoreBackendI
             Ok(MetaStoreBackendImpl::Etcd(EtcdMetaStore::new(client)))
         }
         MetaStoreBackend::Mem => Ok(MetaStoreBackendImpl::Mem(MemStore::new())),
-        MetaStoreBackend::Sql { .. } => panic!("not supported"),
+        MetaStoreBackend::Sql { endpoint } => {
+            let max_connection = if DbBackend::Sqlite.is_prefix_of(&endpoint) {
+                // Due to the fact that Sqlite is prone to the error "(code: 5) database is locked" under concurrent access,
+                // here we forcibly specify the number of connections as 1.
+                1
+            } else {
+                10
+            };
+            let mut options = sea_orm::ConnectOptions::new(endpoint);
+            options
+                .max_connections(max_connection)
+                .connect_timeout(Duration::from_secs(10))
+                .idle_timeout(Duration::from_secs(30));
+            let conn = sea_orm::Database::connect(options)
+                .await
+                .map_err(|e| BackupError::MetaStorage(e.into()))?;
+            let meta_store_sql = SqlMetaStore::new(conn);
+            Ok(MetaStoreBackendImpl::Sql(meta_store_sql))
+        }
     }
 }
 
@@ -89,7 +109,7 @@ pub async fn get_backup_store(opts: RestoreOpts) -> BackupResult<MetaSnapshotSto
         &opts.backup_storage_url,
         Arc::new(ObjectStoreMetrics::unused()),
         "Meta Backup",
-        ObjectStoreConfig::default(),
+        Arc::new(ObjectStoreConfig::default()),
     )
     .await;
     let backup_store =
diff --git a/src/meta/src/barrier/command.rs b/src/meta/src/barrier/command.rs
index 6cece710bd10b..865835c49edeb 100644
--- a/src/meta/src/barrier/command.rs
+++ b/src/meta/src/barrier/command.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::collections::{HashMap, HashSet};
-use std::default::Default;
 use std::sync::Arc;
 
 use futures::future::try_join_all;
@@ -21,6 +20,8 @@ use itertools::Itertools;
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::ActorMapping;
+use risingwave_common::types::Timestamptz;
+use risingwave_common::util::epoch::Epoch;
 use risingwave_connector::source::SplitImpl;
 use risingwave_hummock_sdk::HummockEpoch;
 use risingwave_pb::catalog::CreateType;
@@ -32,12 +33,13 @@ use risingwave_pb::stream_plan::barrier_mutation::Mutation;
 use risingwave_pb::stream_plan::throttle_mutation::RateLimit;
 use risingwave_pb::stream_plan::update_mutation::*;
 use risingwave_pb::stream_plan::{
-    AddMutation, BarrierMutation, CombinedMutation, Dispatcher, Dispatchers, PauseMutation,
-    ResumeMutation, SourceChangeSplitMutation, StopMutation, StreamActor, ThrottleMutation,
-    UpdateMutation,
+    AddMutation, BarrierMutation, CombinedMutation, CreateSubscriptionMutation, Dispatcher,
+    Dispatchers, DropSubscriptionMutation, PauseMutation, ResumeMutation,
+    SourceChangeSplitMutation, StopMutation, StreamActor, ThrottleMutation, UpdateMutation,
 };
 use risingwave_pb::stream_service::WaitEpochCommitRequest;
 use thiserror_ext::AsReport;
+use tracing::warn;
 
 use super::info::{ActorDesc, CommandActorChanges, InflightActorInfo};
 use super::trace::TracedEpoch;
@@ -81,12 +83,19 @@ pub struct Reschedule {
     pub newly_created_actors: Vec<(StreamActor, PbActorStatus)>,
 }
 
+/// Replacing an old table with a new one. All actors in the table job will be rebuilt.
+/// Used for `ALTER TABLE` ([`Command::ReplaceTable`]) and sink into table ([`Command::CreateStreamingJob`]).
 #[derive(Debug, Clone)]
 pub struct ReplaceTablePlan {
     pub old_table_fragments: TableFragments,
     pub new_table_fragments: TableFragments,
     pub merge_updates: Vec<MergeUpdate>,
     pub dispatchers: HashMap<ActorId, Vec<Dispatcher>>,
+    /// For a table with connector, the `SourceExecutor` actor will also be rebuilt with new actor ids.
+    /// We need to reassign splits for it.
+    ///
+    /// Note that there's no `SourceBackfillExecutor` involved for table with connector, so we don't need to worry about
+    /// `backfill_splits`.
     pub init_split_assignment: SplitAssignment,
 }
 
@@ -166,6 +175,7 @@ pub enum Command {
         definition: String,
         ddl_type: DdlType,
         create_type: CreateType,
+        /// This is for create SINK into table.
         replace_table: Option<ReplaceTablePlan>,
     },
     /// `CancelStreamingJob` command generates a `Stop` barrier including the actors of the given
@@ -201,6 +211,22 @@ pub enum Command {
     /// `Throttle` command generates a `Throttle` barrier with the given throttle config to change
     /// the `rate_limit` of `FlowControl` Executor after `StreamScan` or Source.
     Throttle(ThrottleConfig),
+
+    /// `CreateSubscription` command generates a `CreateSubscriptionMutation` to notify
+    /// materialize executor to start storing old value for subscription.
+    CreateSubscription {
+        subscription_id: u32,
+        upstream_mv_table_id: TableId,
+        retention_second: u64,
+    },
+
+    /// `DropSubscription` command generates a `DropSubscriptionMutation` to notify
+    /// materialize executor to stop storing old value when there is no
+    /// subscription depending on it.
+    DropSubscription {
+        subscription_id: u32,
+        upstream_mv_table_id: TableId,
+    },
 }
 
 impl Command {
@@ -285,6 +311,8 @@ impl Command {
             Command::ReplaceTable(plan) => Some(plan.actor_changes()),
             Command::SourceSplitAssignment(_) => None,
             Command::Throttle(_) => None,
+            Command::CreateSubscription { .. } => None,
+            Command::DropSubscription { .. } => None,
         }
     }
 
@@ -360,7 +388,7 @@ pub struct CommandContext {
     /// Differs from [`TracedEpoch`], this span focuses on the lifetime of the corresponding
     /// barrier, including the process of waiting for the barrier to be sent, flowing through the
     /// stream graph on compute nodes, and finishing its `post_collect` stuffs.
-    pub span: tracing::Span,
+    pub _span: tracing::Span,
 }
 
 impl CommandContext {
@@ -383,7 +411,7 @@ impl CommandContext {
             command,
             kind,
             barrier_manager_context,
-            span,
+            _span: span,
         }
     }
 
@@ -663,6 +691,22 @@ impl CommandContext {
                     tracing::debug!("update mutation: {mutation:?}");
                     Some(mutation)
                 }
+
+                Command::CreateSubscription {
+                    upstream_mv_table_id,
+                    subscription_id,
+                    ..
+                } => Some(Mutation::CreateSubscription(CreateSubscriptionMutation {
+                    upstream_mv_table_id: upstream_mv_table_id.table_id,
+                    subscription_id: *subscription_id,
+                })),
+                Command::DropSubscription {
+                    upstream_mv_table_id,
+                    subscription_id,
+                } => Some(Mutation::DropSubscription(DropSubscriptionMutation {
+                    upstream_mv_table_id: upstream_mv_table_id.table_id,
+                    subscription_id: *subscription_id,
+                })),
             };
 
         mutation
@@ -1071,8 +1115,34 @@ impl CommandContext {
                     )
                     .await;
             }
+
+            Command::CreateSubscription {
+                subscription_id, ..
+            } => match &self.barrier_manager_context.metadata_manager {
+                MetadataManager::V1(mgr) => {
+                    mgr.catalog_manager
+                        .finish_create_subscription_procedure(*subscription_id)
+                        .await?;
+                }
+                MetadataManager::V2(mgr) => {
+                    mgr.catalog_controller
+                        .finish_create_subscription_catalog(*subscription_id)
+                        .await?;
+                }
+            },
+            Command::DropSubscription { .. } => {}
         }
 
         Ok(())
     }
+
+    pub fn get_truncate_epoch(&self, retention_second: u64) -> Epoch {
+        let Some(truncate_timestamptz) = Timestamptz::from_secs(
+            self.prev_epoch.value().as_timestamptz().timestamp() - retention_second as i64,
+        ) else {
+            warn!(retention_second, prev_epoch = ?self.prev_epoch.value(), "invalid retention second value");
+            return self.prev_epoch.value();
+        };
+        Epoch::from_unix_millis(truncate_timestamptz.timestamp_millis() as u64)
+    }
 }
diff --git a/src/meta/src/barrier/info.rs b/src/meta/src/barrier/info.rs
index 2c887eae02faa..aa8882d438dce 100644
--- a/src/meta/src/barrier/info.rs
+++ b/src/meta/src/barrier/info.rs
@@ -14,9 +14,11 @@
 
 use std::collections::{HashMap, HashSet};
 
+use risingwave_common::catalog::TableId;
 use risingwave_pb::common::PbWorkerNode;
 use tracing::warn;
 
+use crate::barrier::Command;
 use crate::manager::{ActiveStreamingWorkerNodes, ActorInfos, WorkerId};
 use crate::model::ActorId;
 
@@ -48,11 +50,18 @@ pub struct InflightActorInfo {
 
     /// `actor_id` => `WorkerId`
     pub actor_location_map: HashMap<ActorId, WorkerId>,
+
+    /// `mv_table_id` => `subscription_id` => retention seconds
+    pub mv_depended_subscriptions: HashMap<TableId, HashMap<u32, u64>>,
 }
 
 impl InflightActorInfo {
     /// Resolve inflight actor info from given nodes and actors that are loaded from meta store. It will be used during recovery to rebuild all streaming actors.
-    pub fn resolve(active_nodes: &ActiveStreamingWorkerNodes, actor_infos: ActorInfos) -> Self {
+    pub fn resolve(
+        active_nodes: &ActiveStreamingWorkerNodes,
+        actor_infos: ActorInfos,
+        mv_depended_subscriptions: HashMap<TableId, HashMap<u32, u64>>,
+    ) -> Self {
         let node_map = active_nodes.current().clone();
 
         let actor_map = actor_infos
@@ -77,6 +86,7 @@ impl InflightActorInfo {
             actor_map,
             actor_map_to_send,
             actor_location_map,
+            mv_depended_subscriptions,
         }
     }
 
@@ -96,8 +106,8 @@ impl InflightActorInfo {
 
     /// Apply some actor changes before issuing a barrier command, if the command contains any new added actors, we should update
     /// the info correspondingly.
-    pub fn pre_apply(&mut self, changes: Option<CommandActorChanges>) {
-        if let Some(CommandActorChanges { to_add, .. }) = changes {
+    pub fn pre_apply(&mut self, command: &Command) {
+        if let Some(CommandActorChanges { to_add, .. }) = command.actor_changes() {
             for actor_desc in to_add {
                 assert!(self.node_map.contains_key(&actor_desc.node_id));
                 assert!(
@@ -124,12 +134,27 @@ impl InflightActorInfo {
                 );
             }
         };
+        if let Command::CreateSubscription {
+            subscription_id,
+            upstream_mv_table_id,
+            retention_second,
+        } = command
+        {
+            if let Some(prev_retiontion) = self
+                .mv_depended_subscriptions
+                .entry(*upstream_mv_table_id)
+                .or_default()
+                .insert(*subscription_id, *retention_second)
+            {
+                warn!(subscription_id, ?upstream_mv_table_id, mv_depended_subscriptions = ?self.mv_depended_subscriptions, prev_retiontion, "add an existing subscription id");
+            }
+        }
     }
 
     /// Apply some actor changes after the barrier command is collected, if the command contains any actors that are dropped, we should
     /// remove that from the snapshot correspondingly.
-    pub fn post_apply(&mut self, changes: Option<CommandActorChanges>) {
-        if let Some(CommandActorChanges { to_remove, .. }) = changes {
+    pub fn post_apply(&mut self, command: &Command) {
+        if let Some(CommandActorChanges { to_remove, .. }) = command.actor_changes() {
             for actor_id in to_remove {
                 let node_id = self
                     .actor_location_map
@@ -145,6 +170,25 @@ impl InflightActorInfo {
             self.actor_map_to_send
                 .retain(|_, actor_ids| !actor_ids.is_empty());
         }
+        if let Command::DropSubscription {
+            subscription_id,
+            upstream_mv_table_id,
+        } = command
+        {
+            let removed = match self.mv_depended_subscriptions.get_mut(upstream_mv_table_id) {
+                Some(subscriptions) => {
+                    let removed = subscriptions.remove(subscription_id).is_some();
+                    if removed && subscriptions.is_empty() {
+                        self.mv_depended_subscriptions.remove(upstream_mv_table_id);
+                    }
+                    removed
+                }
+                None => false,
+            };
+            if !removed {
+                warn!(subscription_id, ?upstream_mv_table_id, mv_depended_subscriptions = ?self.mv_depended_subscriptions, "remove a non-existing subscription id");
+            }
+        }
     }
 
     /// Returns actor list to collect in the target worker node.
diff --git a/src/meta/src/barrier/mod.rs b/src/meta/src/barrier/mod.rs
index 05dbdddfc3320..6d5657a158e48 100644
--- a/src/meta/src/barrier/mod.rs
+++ b/src/meta/src/barrier/mod.rs
@@ -30,6 +30,7 @@ use risingwave_common::catalog::TableId;
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::system_param::PAUSE_ON_NEXT_BOOTSTRAP_KEY;
 use risingwave_common::util::epoch::{Epoch, INVALID_EPOCH};
+use risingwave_hummock_sdk::change_log::build_table_change_log_delta;
 use risingwave_hummock_sdk::table_watermark::{
     merge_multiple_new_table_watermarks, TableWatermarks,
 };
@@ -425,6 +426,7 @@ enum CompletingCommand {
         // that has finished but not checkpointed. If there is any, we will force checkpoint on the next barrier
         join_handle: JoinHandle<MetaResult<BarrierCompleteOutput>>,
     },
+    #[expect(dead_code)]
     Err(MetaError),
 }
 
@@ -1082,16 +1084,20 @@ impl GlobalBarrierManagerContext {
         &self,
         active_nodes: &ActiveStreamingWorkerNodes,
     ) -> MetaResult<InflightActorInfo> {
+        let subscriptions = self
+            .metadata_manager
+            .get_mv_depended_subscriptions()
+            .await?;
         let info = match &self.metadata_manager {
             MetadataManager::V1(mgr) => {
                 let all_actor_infos = mgr.fragment_manager.load_all_actors().await;
 
-                InflightActorInfo::resolve(active_nodes, all_actor_infos)
+                InflightActorInfo::resolve(active_nodes, all_actor_infos, subscriptions)
             }
             MetadataManager::V2(mgr) => {
                 let all_actor_infos = mgr.catalog_controller.load_all_actors().await?;
 
-                InflightActorInfo::resolve(active_nodes, all_actor_infos)
+                InflightActorInfo::resolve(active_nodes, all_actor_infos, subscriptions)
             }
         };
 
@@ -1144,11 +1150,12 @@ pub type BarrierManagerRef = GlobalBarrierManagerContext;
 fn collect_commit_epoch_info(
     resps: Vec<BarrierCompleteResponse>,
     command_ctx: &CommandContext,
-    _epochs: &Vec<u64>,
+    epochs: &Vec<u64>,
 ) -> CommitEpochInfo {
     let mut sst_to_worker: HashMap<HummockSstableObjectId, WorkerId> = HashMap::new();
     let mut synced_ssts: Vec<ExtendedSstableInfo> = vec![];
     let mut table_watermarks = Vec::with_capacity(resps.len());
+    let mut old_value_ssts = Vec::with_capacity(resps.len());
     for resp in resps {
         let ssts_iter = resp.synced_sstables.into_iter().map(|grouped| {
             let sst_info = grouped.sst.expect("field not None");
@@ -1161,6 +1168,7 @@ fn collect_commit_epoch_info(
         });
         synced_ssts.extend(ssts_iter);
         table_watermarks.push(resp.table_watermarks);
+        old_value_ssts.extend(resp.old_value_sstables);
     }
     let new_table_fragment_info = if let Command::CreateStreamingJob {
         table_fragments, ..
@@ -1179,6 +1187,24 @@ fn collect_commit_epoch_info(
         None
     };
 
+    let table_new_change_log = build_table_change_log_delta(
+        old_value_ssts.into_iter(),
+        synced_ssts.iter().map(|sst| &sst.sst_info),
+        epochs,
+        command_ctx
+            .info
+            .mv_depended_subscriptions
+            .iter()
+            .filter_map(|(mv_table_id, subscriptions)| {
+                subscriptions.values().max().map(|max_retention| {
+                    (
+                        mv_table_id.table_id,
+                        command_ctx.get_truncate_epoch(*max_retention).0,
+                    )
+                })
+            }),
+    );
+
     CommitEpochInfo::new(
         synced_ssts,
         merge_multiple_new_table_watermarks(
@@ -1199,5 +1225,6 @@ fn collect_commit_epoch_info(
         ),
         sst_to_worker,
         new_table_fragment_info,
+        table_new_change_log,
     )
 }
diff --git a/src/meta/src/barrier/recovery.rs b/src/meta/src/barrier/recovery.rs
index a3f19e555bf75..f17b4e163901c 100644
--- a/src/meta/src/barrier/recovery.rs
+++ b/src/meta/src/barrier/recovery.rs
@@ -67,6 +67,7 @@ impl GlobalBarrierManagerContext {
     async fn clean_dirty_streaming_jobs(&self) -> MetaResult<()> {
         match &self.metadata_manager {
             MetadataManager::V1(mgr) => {
+                mgr.catalog_manager.clean_dirty_subscription().await?;
                 // Please look at `CatalogManager::clean_dirty_tables` for more details.
                 mgr.catalog_manager
                     .clean_dirty_tables(mgr.fragment_manager.clone())
@@ -97,6 +98,7 @@ impl GlobalBarrierManagerContext {
                     .await;
             }
             MetadataManager::V2(mgr) => {
+                mgr.catalog_controller.clean_dirty_subscription().await?;
                 let ReleaseContext { source_ids, .. } =
                     mgr.catalog_controller.clean_dirty_creating_jobs().await?;
 
@@ -283,7 +285,7 @@ impl GlobalBarrierManagerContext {
                     tracing::debug!("recovering stream job {}", id);
                     finished.await.ok().context("failed to finish command")??;
                     tracing::debug!(id, "finished stream job");
-                    catalog_controller.finish_streaming_job(id).await?;
+                    catalog_controller.finish_streaming_job(id, None).await?;
                 };
                 if let Err(e) = &res {
                     tracing::error!(
diff --git a/src/meta/src/barrier/rpc.rs b/src/meta/src/barrier/rpc.rs
index d23859609e05e..23d2d0b5d3c2a 100644
--- a/src/meta/src/barrier/rpc.rs
+++ b/src/meta/src/barrier/rpc.rs
@@ -27,11 +27,12 @@ use itertools::Itertools;
 use risingwave_common::hash::ActorId;
 use risingwave_common::util::tracing::TracingContext;
 use risingwave_pb::common::{ActorInfo, WorkerNode};
-use risingwave_pb::stream_plan::{Barrier, BarrierMutation, StreamActor};
+use risingwave_pb::stream_plan::{Barrier, BarrierMutation};
 use risingwave_pb::stream_service::{
     streaming_control_stream_request, streaming_control_stream_response, BarrierCompleteResponse,
-    BroadcastActorInfoTableRequest, BuildActorsRequest, DropActorsRequest, InjectBarrierRequest,
-    StreamingControlStreamRequest, StreamingControlStreamResponse, UpdateActorsRequest,
+    BroadcastActorInfoTableRequest, BuildActorInfo, BuildActorsRequest, DropActorsRequest,
+    InjectBarrierRequest, StreamingControlStreamRequest, StreamingControlStreamResponse,
+    UpdateActorsRequest,
 };
 use risingwave_rpc_client::error::RpcError;
 use risingwave_rpc_client::StreamClient;
@@ -427,7 +428,7 @@ impl StreamRpcManager {
         worker_nodes: &HashMap<WorkerId, WorkerNode>,
         broadcast_worker_ids: impl Iterator<Item = WorkerId>,
         actor_infos_to_broadcast: impl Iterator<Item = ActorInfo>,
-        node_actors_to_create: impl Iterator<Item = (WorkerId, Vec<StreamActor>)>,
+        node_actors_to_create: impl Iterator<Item = (WorkerId, Vec<BuildActorInfo>)>,
     ) -> MetaResult<()> {
         let actor_infos = actor_infos_to_broadcast.collect_vec();
         let mut node_actors_to_create = node_actors_to_create.collect::<HashMap<_, _>>();
@@ -446,7 +447,7 @@ impl StreamRpcManager {
                         .await?;
                     if let Some(actors) = actors {
                         let request_id = Self::new_request_id();
-                        let actor_ids = actors.iter().map(|actor| actor.actor_id).collect_vec();
+                        let actor_ids = actors.iter().map(|actor| actor.actor.as_ref().unwrap().actor_id).collect_vec();
                         tracing::debug!(request_id = request_id.as_str(), actors = ?actor_ids, "update actors");
                         client
                             .update_actors(UpdateActorsRequest { request_id, actors })
diff --git a/src/meta/src/barrier/state.rs b/src/meta/src/barrier/state.rs
index f24bbb5637220..f3b49b8fa2709 100644
--- a/src/meta/src/barrier/state.rs
+++ b/src/meta/src/barrier/state.rs
@@ -76,10 +76,9 @@ impl BarrierManagerState {
     /// Returns the inflight actor infos that have included the newly added actors in the given command. The dropped actors
     /// will be removed from the state after the info get resolved.
     pub fn apply_command(&mut self, command: &Command) -> InflightActorInfo {
-        let changes = command.actor_changes();
-        self.inflight_actor_infos.pre_apply(changes.clone());
+        self.inflight_actor_infos.pre_apply(command);
         let info = self.inflight_actor_infos.clone();
-        self.inflight_actor_infos.post_apply(changes);
+        self.inflight_actor_infos.post_apply(command);
 
         info
     }
diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs
index 111e8e5ab9fe3..200736725cd07 100644
--- a/src/meta/src/controller/catalog.rs
+++ b/src/meta/src/controller/catalog.rs
@@ -18,10 +18,9 @@ use std::sync::Arc;
 
 use anyhow::anyhow;
 use itertools::Itertools;
-use risingwave_common::catalog::{
-    is_subscription_internal_table, TableOption, DEFAULT_SCHEMA_NAME, SYSTEM_SCHEMAS,
-};
-use risingwave_common::util::stream_graph_visitor::visit_stream_node_cont;
+use risingwave_common::catalog::{TableOption, DEFAULT_SCHEMA_NAME, SYSTEM_SCHEMAS};
+use risingwave_common::hash::ParallelUnitMapping;
+use risingwave_common::util::stream_graph_visitor::visit_stream_node_cont_mut;
 use risingwave_common::{bail, current_cluster_version};
 use risingwave_connector::source::UPSTREAM_SOURCE_KEY;
 use risingwave_meta_model_v2::object::ObjectType;
@@ -29,14 +28,16 @@ use risingwave_meta_model_v2::prelude::*;
 use risingwave_meta_model_v2::table::TableType;
 use risingwave_meta_model_v2::{
     actor, connection, database, fragment, function, index, object, object_dependency, schema,
-    sink, source, streaming_job, subscription, table, user_privilege, view, ActorId,
+    secret, sink, source, streaming_job, subscription, table, user_privilege, view, ActorId,
     ActorUpstreamActors, ColumnCatalogArray, ConnectionId, CreateType, DatabaseId, FragmentId,
     FunctionId, I32Array, IndexId, JobStatus, ObjectId, PrivateLinkService, Property, SchemaId,
-    SinkId, SourceId, StreamNode, StreamSourceInfo, StreamingParallelism, TableId, UserId,
+    SecretId, SinkId, SourceId, StreamNode, StreamSourceInfo, StreamingParallelism, SubscriptionId,
+    TableId, UserId, ViewId,
 };
+use risingwave_pb::catalog::subscription::SubscriptionState;
 use risingwave_pb::catalog::table::PbTableType;
 use risingwave_pb::catalog::{
-    PbComment, PbConnection, PbDatabase, PbFunction, PbIndex, PbSchema, PbSink, PbSource,
+    PbComment, PbConnection, PbDatabase, PbFunction, PbIndex, PbSchema, PbSecret, PbSink, PbSource,
     PbSubscription, PbTable, PbView,
 };
 use risingwave_pb::meta::cancel_creating_jobs_request::PbCreatingJobInfo;
@@ -45,7 +46,9 @@ use risingwave_pb::meta::relation::PbRelationInfo;
 use risingwave_pb::meta::subscribe_response::{
     Info as NotificationInfo, Info, Operation as NotificationOperation, Operation,
 };
-use risingwave_pb::meta::{PbRelation, PbRelationGroup};
+use risingwave_pb::meta::{
+    FragmentParallelUnitMapping, PbFragmentWorkerSlotMapping, PbRelation, PbRelationGroup,
+};
 use risingwave_pb::stream_plan::stream_node::NodeBody;
 use risingwave_pb::stream_plan::FragmentTypeFlag;
 use risingwave_pb::user::PbUserInfo;
@@ -58,14 +61,15 @@ use sea_orm::{
 };
 use tokio::sync::{RwLock, RwLockReadGuard};
 
+use super::utils::check_subscription_name_duplicate;
 use crate::controller::rename::{alter_relation_rename, alter_relation_rename_refs};
 use crate::controller::utils::{
     check_connection_name_duplicate, check_database_name_duplicate,
     check_function_signature_duplicate, check_relation_name_duplicate, check_schema_name_duplicate,
-    ensure_object_id, ensure_object_not_refer, ensure_schema_empty, ensure_user_id,
-    get_fragment_mappings, get_fragment_mappings_by_jobs, get_referring_objects,
-    get_referring_objects_cascade, get_user_privilege, list_user_info_by_ids,
-    resolve_source_register_info_for_jobs, PartialObject,
+    check_secret_name_duplicate, ensure_object_id, ensure_object_not_refer, ensure_schema_empty,
+    ensure_user_id, get_fragment_mappings_by_jobs, get_parallel_unit_to_worker_map,
+    get_referring_objects, get_referring_objects_cascade, get_user_privilege,
+    list_user_info_by_ids, resolve_source_register_info_for_jobs, PartialObject,
 };
 use crate::controller::ObjectModel;
 use crate::manager::{Catalog, MetaSrvEnv, NotificationVersion, IGNORED_NOTIFICATION_VERSION};
@@ -274,8 +278,30 @@ impl CatalogController {
             .into_tuple()
             .all(&txn)
             .await?;
+
+        let parallel_unit_to_worker = get_parallel_unit_to_worker_map(&txn).await?;
+
         let fragment_mappings = get_fragment_mappings_by_jobs(&txn, streaming_jobs.clone()).await?;
 
+        let fragment_mappings = fragment_mappings
+            .into_iter()
+            .map(
+                |FragmentParallelUnitMapping {
+                     fragment_id,
+                     mapping,
+                 }| {
+                    PbFragmentWorkerSlotMapping {
+                        fragment_id,
+                        mapping: Some(
+                            ParallelUnitMapping::from_protobuf(&mapping.unwrap())
+                                .to_worker_slot(&parallel_unit_to_worker)
+                                .to_protobuf(),
+                        ),
+                    }
+                },
+            )
+            .collect();
+
         // The schema and objects in the database will be delete cascade.
         let res = Object::delete_by_id(database_id).exec(&txn).await?;
         if res.rows_affected == 0 {
@@ -295,6 +321,7 @@ impl CatalogController {
                 }),
             )
             .await;
+
         self.notify_fragment_mapping(NotificationOperation::Delete, fragment_mappings)
             .await;
         Ok((
@@ -402,6 +429,117 @@ impl CatalogController {
         Ok(version)
     }
 
+    pub async fn create_subscription_catalog(
+        &self,
+        pb_subscription: &mut PbSubscription,
+    ) -> MetaResult<()> {
+        let inner = self.inner.write().await;
+        let txn = inner.db.begin().await?;
+
+        ensure_user_id(pb_subscription.owner as _, &txn).await?;
+        ensure_object_id(ObjectType::Database, pb_subscription.database_id as _, &txn).await?;
+        ensure_object_id(ObjectType::Schema, pb_subscription.schema_id as _, &txn).await?;
+        check_subscription_name_duplicate(pb_subscription, &txn).await?;
+
+        let obj = Self::create_object(
+            &txn,
+            ObjectType::Subscription,
+            pb_subscription.owner as _,
+            Some(pb_subscription.database_id as _),
+            Some(pb_subscription.schema_id as _),
+        )
+        .await?;
+        pb_subscription.id = obj.oid as _;
+        let subscription: subscription::ActiveModel = pb_subscription.clone().into();
+        Subscription::insert(subscription).exec(&txn).await?;
+
+        // record object dependency.
+        ObjectDependency::insert(object_dependency::ActiveModel {
+            oid: Set(pb_subscription.dependent_table_id as _),
+            used_by: Set(pb_subscription.id as _),
+            ..Default::default()
+        })
+        .exec(&txn)
+        .await?;
+        txn.commit().await?;
+        Ok(())
+    }
+
+    pub async fn finish_create_subscription_catalog(&self, subscription_id: u32) -> MetaResult<()> {
+        let inner = self.inner.write().await;
+        let txn = inner.db.begin().await?;
+        let job_id = subscription_id as i32;
+
+        // update `created_at` as now() and `created_at_cluster_version` as current cluster version.
+        let res = Object::update_many()
+            .col_expr(object::Column::CreatedAt, Expr::current_timestamp().into())
+            .col_expr(
+                object::Column::CreatedAtClusterVersion,
+                current_cluster_version().into(),
+            )
+            .filter(object::Column::Oid.eq(job_id))
+            .exec(&txn)
+            .await?;
+        if res.rows_affected == 0 {
+            return Err(MetaError::catalog_id_not_found("subscription", job_id));
+        }
+
+        // mark the target subscription as `Create`.
+        let job = subscription::ActiveModel {
+            subscription_id: Set(job_id),
+            subscription_state: Set(SubscriptionState::Created.into()),
+            ..Default::default()
+        };
+        job.update(&txn).await?;
+        txn.commit().await?;
+
+        Ok(())
+    }
+
+    pub async fn notify_create_subscription(
+        &self,
+        subscription_id: u32,
+    ) -> MetaResult<NotificationVersion> {
+        let inner = self.inner.write().await;
+        let txn = inner.db.begin().await?;
+        let job_id = subscription_id as i32;
+        let (subscription, obj) = Subscription::find_by_id(job_id)
+            .find_also_related(Object)
+            .one(&txn)
+            .await?
+            .ok_or_else(|| MetaError::catalog_id_not_found("subscription", job_id))?;
+        txn.commit().await?;
+
+        let version = self
+            .notify_frontend(
+                Operation::Add,
+                Info::RelationGroup(PbRelationGroup {
+                    relations: vec![PbRelation {
+                        relation_info: PbRelationInfo::Subscription(
+                            ObjectModel(subscription, obj.unwrap()).into(),
+                        )
+                        .into(),
+                    }],
+                }),
+            )
+            .await;
+        Ok(version)
+    }
+
+    pub async fn clean_dirty_subscription(&self) -> MetaResult<()> {
+        let inner = self.inner.write().await;
+        let txn = inner.db.begin().await?;
+        let _res = Subscription::delete_many()
+            .filter(
+                subscription::Column::SubscriptionState
+                    .eq(Into::<i32>::into(SubscriptionState::Init)),
+            )
+            .exec(&txn)
+            .await?;
+        txn.commit().await?;
+        Ok(())
+    }
+
     pub async fn list_background_creating_mviews(&self) -> MetaResult<Vec<table::Model>> {
         let inner = self.inner.read().await;
         let tables = Table::find()
@@ -469,6 +607,29 @@ impl CatalogController {
             }
         }));
 
+        let subscription_dependencies: Vec<(SubscriptionId, TableId)> = Subscription::find()
+            .select_only()
+            .columns([
+                subscription::Column::SubscriptionId,
+                subscription::Column::DependentTableId,
+            ])
+            .join(JoinType::InnerJoin, subscription::Relation::Object.def())
+            .filter(
+                subscription::Column::SubscriptionState
+                    .eq(Into::<i32>::into(SubscriptionState::Created))
+                    .and(subscription::Column::DependentTableId.is_not_null()),
+            )
+            .into_tuple()
+            .all(&inner.db)
+            .await?;
+
+        obj_dependencies.extend(subscription_dependencies.into_iter().map(
+            |(subscription_id, table_id)| PbObjectDependencies {
+                object_id: subscription_id as _,
+                referenced_object_id: table_id as _,
+            },
+        ));
+
         Ok(obj_dependencies)
     }
 
@@ -741,7 +902,7 @@ impl CatalogController {
 
                     let mut pb_stream_node = stream_node.to_protobuf();
 
-                    visit_stream_node_cont(&mut pb_stream_node, |node| {
+                    visit_stream_node_cont_mut(&mut pb_stream_node, |node| {
                         if let Some(NodeBody::Union(_)) = node.node_body {
                             node.input.retain_mut(|input| {
                                 if let Some(NodeBody::Merge(merge_node)) = &mut input.node_body
@@ -814,171 +975,6 @@ impl CatalogController {
         Ok(true)
     }
 
-    /// `finish_streaming_job` marks job related objects as `Created` and notify frontend.
-    pub async fn finish_streaming_job(&self, job_id: ObjectId) -> MetaResult<NotificationVersion> {
-        let inner = self.inner.write().await;
-        let txn = inner.db.begin().await?;
-
-        let job_type = Object::find_by_id(job_id)
-            .select_only()
-            .column(object::Column::ObjType)
-            .into_tuple()
-            .one(&txn)
-            .await?
-            .ok_or_else(|| MetaError::catalog_id_not_found("streaming job", job_id))?;
-
-        // update `created_at` as now() and `created_at_cluster_version` as current cluster version.
-        let res = Object::update_many()
-            .col_expr(object::Column::CreatedAt, Expr::current_timestamp().into())
-            .col_expr(
-                object::Column::CreatedAtClusterVersion,
-                current_cluster_version().into(),
-            )
-            .filter(object::Column::Oid.eq(job_id))
-            .exec(&txn)
-            .await?;
-        if res.rows_affected == 0 {
-            return Err(MetaError::catalog_id_not_found("streaming job", job_id));
-        }
-
-        // mark the target stream job as `Created`.
-        let job = streaming_job::ActiveModel {
-            job_id: Set(job_id),
-            job_status: Set(JobStatus::Created),
-            ..Default::default()
-        };
-        job.update(&txn).await?;
-
-        // notify frontend: job, internal tables.
-        let internal_table_objs = Table::find()
-            .find_also_related(Object)
-            .filter(table::Column::BelongsToJobId.eq(job_id))
-            .all(&txn)
-            .await?;
-        let mut relations = internal_table_objs
-            .iter()
-            .map(|(table, obj)| PbRelation {
-                relation_info: Some(PbRelationInfo::Table(
-                    ObjectModel(table.clone(), obj.clone().unwrap()).into(),
-                )),
-            })
-            .collect_vec();
-
-        match job_type {
-            ObjectType::Table => {
-                let (table, obj) = Table::find_by_id(job_id)
-                    .find_also_related(Object)
-                    .one(&txn)
-                    .await?
-                    .ok_or_else(|| MetaError::catalog_id_not_found("table", job_id))?;
-                if let Some(source_id) = table.optional_associated_source_id {
-                    let (src, obj) = Source::find_by_id(source_id)
-                        .find_also_related(Object)
-                        .one(&txn)
-                        .await?
-                        .ok_or_else(|| MetaError::catalog_id_not_found("source", source_id))?;
-                    relations.push(PbRelation {
-                        relation_info: Some(PbRelationInfo::Source(
-                            ObjectModel(src, obj.unwrap()).into(),
-                        )),
-                    });
-                }
-                relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::Table(
-                        ObjectModel(table, obj.unwrap()).into(),
-                    )),
-                });
-            }
-            ObjectType::Sink => {
-                let (sink, obj) = Sink::find_by_id(job_id)
-                    .find_also_related(Object)
-                    .one(&txn)
-                    .await?
-                    .ok_or_else(|| MetaError::catalog_id_not_found("sink", job_id))?;
-                relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::Sink(
-                        ObjectModel(sink, obj.unwrap()).into(),
-                    )),
-                });
-            }
-            ObjectType::Subscription => {
-                let (mut subscription, obj) = Subscription::find_by_id(job_id)
-                    .find_also_related(Object)
-                    .one(&txn)
-                    .await?
-                    .ok_or_else(|| MetaError::catalog_id_not_found("subscription", job_id))?;
-                let log_store_names: Vec<_> = internal_table_objs
-                    .iter()
-                    .filter(|a| is_subscription_internal_table(&subscription.name, &a.0.name))
-                    .map(|a| &a.0.name)
-                    .collect();
-                if log_store_names.len() != 1 {
-                    bail!("A subscription can only have one log_store_name");
-                }
-                subscription.subscription_internal_table_name =
-                    log_store_names.get(0).cloned().cloned();
-                relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::Subscription(
-                        ObjectModel(subscription, obj.unwrap()).into(),
-                    )),
-                });
-            }
-            ObjectType::Index => {
-                let (index, obj) = Index::find_by_id(job_id)
-                    .find_also_related(Object)
-                    .one(&txn)
-                    .await?
-                    .ok_or_else(|| MetaError::catalog_id_not_found("index", job_id))?;
-                {
-                    let (table, obj) = Table::find_by_id(index.index_table_id)
-                        .find_also_related(Object)
-                        .one(&txn)
-                        .await?
-                        .ok_or_else(|| {
-                            MetaError::catalog_id_not_found("table", index.index_table_id)
-                        })?;
-                    relations.push(PbRelation {
-                        relation_info: Some(PbRelationInfo::Table(
-                            ObjectModel(table, obj.unwrap()).into(),
-                        )),
-                    });
-                }
-                relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::Index(
-                        ObjectModel(index, obj.unwrap()).into(),
-                    )),
-                });
-            }
-            ObjectType::Source => {
-                let (source, obj) = Source::find_by_id(job_id)
-                    .find_also_related(Object)
-                    .one(&txn)
-                    .await?
-                    .ok_or_else(|| MetaError::catalog_id_not_found("source", job_id))?;
-                relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::Source(
-                        ObjectModel(source, obj.unwrap()).into(),
-                    )),
-                });
-            }
-            _ => unreachable!("invalid job type: {:?}", job_type),
-        }
-
-        let fragment_mapping = get_fragment_mappings(&txn, job_id).await?;
-        txn.commit().await?;
-
-        self.notify_fragment_mapping(NotificationOperation::Add, fragment_mapping)
-            .await;
-        let version = self
-            .notify_frontend(
-                NotificationOperation::Add,
-                NotificationInfo::RelationGroup(PbRelationGroup { relations }),
-            )
-            .await;
-
-        Ok(version)
-    }
-
     pub async fn create_source(
         &self,
         mut pb_source: PbSource,
@@ -1104,6 +1100,88 @@ impl CatalogController {
         Ok(version)
     }
 
+    pub async fn create_secret(&self, mut pb_secret: PbSecret) -> MetaResult<NotificationVersion> {
+        let inner = self.inner.write().await;
+        let owner_id = pb_secret.owner as _;
+        let txn = inner.db.begin().await?;
+        ensure_user_id(owner_id, &txn).await?;
+        ensure_object_id(ObjectType::Database, pb_secret.database_id as _, &txn).await?;
+        ensure_object_id(ObjectType::Schema, pb_secret.schema_id as _, &txn).await?;
+        check_secret_name_duplicate(&pb_secret, &txn).await?;
+
+        let secret_obj = Self::create_object(
+            &txn,
+            ObjectType::Secret,
+            owner_id,
+            Some(pb_secret.database_id as _),
+            Some(pb_secret.schema_id as _),
+        )
+        .await?;
+        pb_secret.id = secret_obj.oid as _;
+        let secret: secret::ActiveModel = pb_secret.clone().into();
+        Secret::insert(secret).exec(&txn).await?;
+
+        txn.commit().await?;
+
+        let version = self
+            .notify_frontend(
+                NotificationOperation::Add,
+                NotificationInfo::Secret(pb_secret),
+            )
+            .await;
+        Ok(version)
+    }
+
+    pub async fn get_secret_by_id(&self, secret_id: SecretId) -> MetaResult<PbSecret> {
+        let inner = self.inner.read().await;
+        let (secret, obj) = Secret::find_by_id(secret_id)
+            .find_also_related(Object)
+            .one(&inner.db)
+            .await?
+            .ok_or_else(|| MetaError::catalog_id_not_found("secret", secret_id))?;
+        Ok(ObjectModel(secret, obj.unwrap()).into())
+    }
+
+    pub async fn drop_secret(&self, secret_id: SecretId) -> MetaResult<NotificationVersion> {
+        let inner = self.inner.write().await;
+        let txn = inner.db.begin().await?;
+        let (secret, secret_obj) = Secret::find_by_id(secret_id)
+            .find_also_related(Object)
+            .one(&txn)
+            .await?
+            .ok_or_else(|| MetaError::catalog_id_not_found("secret", secret_id))?;
+        ensure_object_not_refer(ObjectType::Secret, secret_id, &txn).await?;
+
+        // Find affect users with privileges on the connection.
+        let to_update_user_ids: Vec<UserId> = UserPrivilege::find()
+            .select_only()
+            .distinct()
+            .column(user_privilege::Column::UserId)
+            .filter(user_privilege::Column::Oid.eq(secret_id))
+            .into_tuple()
+            .all(&txn)
+            .await?;
+
+        let res = Object::delete_by_id(secret_id).exec(&txn).await?;
+        if res.rows_affected == 0 {
+            return Err(MetaError::catalog_id_not_found("secret", secret_id));
+        }
+        let user_infos = list_user_info_by_ids(to_update_user_ids, &txn).await?;
+
+        txn.commit().await?;
+
+        let pb_secret: PbSecret = ObjectModel(secret, secret_obj.unwrap()).into();
+
+        self.notify_users_update(user_infos).await;
+        let version = self
+            .notify_frontend(
+                NotificationOperation::Delete,
+                NotificationInfo::Secret(pb_secret),
+            )
+            .await;
+        Ok(version)
+    }
+
     pub async fn create_connection(
         &self,
         mut pb_connection: PbConnection,
@@ -1446,35 +1524,6 @@ impl CatalogController {
                 relations.push(PbRelationInfo::Subscription(
                     ObjectModel(subscription, obj).into(),
                 ));
-
-                // internal tables.
-                let internal_tables: Vec<TableId> = Table::find()
-                    .select_only()
-                    .column(table::Column::TableId)
-                    .filter(table::Column::BelongsToJobId.eq(object_id))
-                    .into_tuple()
-                    .all(&txn)
-                    .await?;
-
-                Object::update_many()
-                    .col_expr(
-                        object::Column::OwnerId,
-                        SimpleExpr::Value(Value::Int(Some(new_owner))),
-                    )
-                    .filter(object::Column::Oid.is_in(internal_tables.clone()))
-                    .exec(&txn)
-                    .await?;
-
-                let table_objs = Table::find()
-                    .find_also_related(Object)
-                    .filter(table::Column::TableId.is_in(internal_tables))
-                    .all(&txn)
-                    .await?;
-                for (table, table_obj) in table_objs {
-                    relations.push(PbRelationInfo::Table(
-                        ObjectModel(table, table_obj.unwrap()).into(),
-                    ));
-                }
             }
             ObjectType::View => {
                 let view = View::find_by_id(object_id)
@@ -1706,37 +1755,6 @@ impl CatalogController {
                 relations.push(PbRelationInfo::Subscription(
                     ObjectModel(subscription, obj).into(),
                 ));
-
-                // internal tables.
-                let internal_tables: Vec<TableId> = Table::find()
-                    .select_only()
-                    .column(table::Column::TableId)
-                    .filter(table::Column::BelongsToJobId.eq(object_id))
-                    .into_tuple()
-                    .all(&txn)
-                    .await?;
-
-                if !internal_tables.is_empty() {
-                    Object::update_many()
-                        .col_expr(
-                            object::Column::SchemaId,
-                            SimpleExpr::Value(Value::Int(Some(new_schema))),
-                        )
-                        .filter(object::Column::Oid.is_in(internal_tables.clone()))
-                        .exec(&txn)
-                        .await?;
-
-                    let table_objs = Table::find()
-                        .find_also_related(Object)
-                        .filter(table::Column::TableId.is_in(internal_tables))
-                        .all(&txn)
-                        .await?;
-                    for (table, table_obj) in table_objs {
-                        relations.push(PbRelationInfo::Table(
-                            ObjectModel(table, table_obj.unwrap()).into(),
-                        ));
-                    }
-                }
             }
             ObjectType::View => {
                 let view = View::find_by_id(object_id)
@@ -2063,6 +2081,7 @@ impl CatalogController {
 
         let (source_fragments, removed_actors) =
             resolve_source_register_info_for_jobs(&txn, to_drop_streaming_jobs.clone()).await?;
+
         let fragment_mappings =
             get_fragment_mappings_by_jobs(&txn, to_drop_streaming_jobs.clone()).await?;
 
@@ -2089,6 +2108,8 @@ impl CatalogController {
         }
         let user_infos = list_user_info_by_ids(to_update_user_ids, &txn).await?;
 
+        let parallel_unit_to_worker = get_parallel_unit_to_worker_map(&txn).await?;
+
         txn.commit().await?;
 
         // notify about them.
@@ -2163,6 +2184,26 @@ impl CatalogController {
                 NotificationInfo::RelationGroup(PbRelationGroup { relations }),
             )
             .await;
+
+        let fragment_mappings = fragment_mappings
+            .into_iter()
+            .map(
+                |FragmentParallelUnitMapping {
+                     fragment_id,
+                     mapping,
+                 }| {
+                    PbFragmentWorkerSlotMapping {
+                        fragment_id,
+                        mapping: Some(
+                            ParallelUnitMapping::from_protobuf(&mapping.unwrap())
+                                .to_worker_slot(&parallel_unit_to_worker)
+                                .to_protobuf(),
+                        ),
+                    }
+                },
+            )
+            .collect();
+
         self.notify_fragment_mapping(NotificationOperation::Delete, fragment_mappings)
             .await;
 
@@ -2492,6 +2533,11 @@ impl CatalogController {
         inner.list_databases().await
     }
 
+    pub async fn list_schemas(&self) -> MetaResult<Vec<PbSchema>> {
+        let inner = self.inner.read().await;
+        inner.list_schemas().await
+    }
+
     pub async fn list_all_state_tables(&self) -> MetaResult<Vec<PbTable>> {
         let inner = self.inner.read().await;
         inner.list_all_state_tables().await
@@ -2536,6 +2582,19 @@ impl CatalogController {
         Ok(table_ids)
     }
 
+    pub async fn list_view_ids(&self, schema_id: SchemaId) -> MetaResult<Vec<ViewId>> {
+        let inner = self.inner.read().await;
+        let view_ids: Vec<ViewId> = View::find()
+            .select_only()
+            .column(view::Column::ViewId)
+            .join(JoinType::InnerJoin, view::Relation::Object.def())
+            .filter(object::Column::SchemaId.eq(schema_id))
+            .into_tuple()
+            .all(&inner.db)
+            .await?;
+        Ok(view_ids)
+    }
+
     pub async fn list_tables_by_type(&self, table_type: TableType) -> MetaResult<Vec<PbTable>> {
         let inner = self.inner.read().await;
         let table_objs = Table::find()
@@ -2582,6 +2641,11 @@ impl CatalogController {
         inner.list_views().await
     }
 
+    pub async fn list_users(&self) -> MetaResult<Vec<PbUserInfo>> {
+        let inner = self.inner.read().await;
+        inner.list_users().await
+    }
+
     pub async fn get_table_by_name(
         &self,
         database_name: &str,
@@ -2614,6 +2678,25 @@ impl CatalogController {
             .collect())
     }
 
+    pub async fn get_subscription_by_id(
+        &self,
+        subscription_id: SubscriptionId,
+    ) -> MetaResult<PbSubscription> {
+        let inner = self.inner.read().await;
+        let subscription_objs = Subscription::find()
+            .find_also_related(Object)
+            .filter(subscription::Column::SubscriptionId.eq(subscription_id))
+            .all(&inner.db)
+            .await?;
+        let subscription: PbSubscription = subscription_objs
+            .into_iter()
+            .map(|(subscription, obj)| ObjectModel(subscription, obj.unwrap()).into())
+            .find_or_first(|_| true)
+            .ok_or_else(|| anyhow!("cant find subscription with id {}", subscription_id))?;
+
+        Ok(subscription)
+    }
+
     pub async fn find_creating_streaming_job_ids(
         &self,
         infos: Vec<PbCreatingJobInfo>,
@@ -2757,11 +2840,9 @@ impl CatalogController {
         let inner = self.inner.read().await;
 
         // created table ids.
-        let mut table_ids: Vec<TableId> = Table::find()
+        let mut table_ids: Vec<TableId> = StreamingJob::find()
             .select_only()
-            .column(table::Column::TableId)
-            .join(JoinType::LeftJoin, table::Relation::Object1.def())
-            .join(JoinType::LeftJoin, object::Relation::StreamingJob.def())
+            .column(streaming_job::Column::JobId)
             .filter(streaming_job::Column::JobStatus.eq(JobStatus::Created))
             .into_tuple()
             .all(&inner.db)
@@ -2805,6 +2886,7 @@ impl CatalogControllerInner {
         let views = self.list_views().await?;
         let functions = self.list_functions().await?;
         let connections = self.list_connections().await?;
+        let secrets = self.list_secrets().await?;
 
         let users = self.list_users().await?;
 
@@ -2820,6 +2902,7 @@ impl CatalogControllerInner {
                 views,
                 functions,
                 connections,
+                secrets,
             ),
             users,
         ))
@@ -2993,8 +3076,6 @@ impl CatalogControllerInner {
     async fn list_subscriptions(&self) -> MetaResult<Vec<PbSubscription>> {
         let subscription_objs = Subscription::find()
             .find_also_related(Object)
-            .join(JoinType::LeftJoin, object::Relation::StreamingJob.def())
-            .filter(streaming_job::Column::JobStatus.eq(JobStatus::Created))
             .all(&self.db)
             .await?;
 
@@ -3040,6 +3121,17 @@ impl CatalogControllerInner {
             .collect())
     }
 
+    async fn list_secrets(&self) -> MetaResult<Vec<PbSecret>> {
+        let secret_objs = Secret::find()
+            .find_also_related(Object)
+            .all(&self.db)
+            .await?;
+        Ok(secret_objs
+            .into_iter()
+            .map(|(secret, obj)| ObjectModel(secret, obj.unwrap()).into())
+            .collect())
+    }
+
     async fn list_functions(&self) -> MetaResult<Vec<PbFunction>> {
         let func_objs = Function::find()
             .find_also_related(Object)
@@ -3056,7 +3148,6 @@ impl CatalogControllerInner {
 #[cfg(test)]
 #[cfg(not(madsim))]
 mod tests {
-    use risingwave_meta_model_v2::ViewId;
 
     use super::*;
 
diff --git a/src/meta/src/controller/cluster.rs b/src/meta/src/controller/cluster.rs
index 01bf28643bb66..41654192a58c8 100644
--- a/src/meta/src/controller/cluster.rs
+++ b/src/meta/src/controller/cluster.rs
@@ -175,7 +175,7 @@ impl ClusterController {
         Ok(())
     }
 
-    pub async fn delete_worker(&self, host_address: HostAddress) -> MetaResult<PbWorkerNode> {
+    pub async fn delete_worker(&self, host_address: HostAddress) -> MetaResult<()> {
         let mut inner = self.inner.write().await;
         let worker = inner.delete_worker(host_address).await?;
         if worker.r#type() == PbWorkerType::ComputeNode {
@@ -190,10 +190,10 @@ impl ClusterController {
         // local notification.
         self.env
             .notification_manager()
-            .notify_local_subscribers(LocalNotification::WorkerNodeDeleted(worker.clone()))
+            .notify_local_subscribers(LocalNotification::WorkerNodeDeleted(worker))
             .await;
 
-        Ok(worker)
+        Ok(())
     }
 
     pub async fn update_schedulability(
@@ -262,11 +262,12 @@ impl ClusterController {
                 // 3. Delete expired workers.
                 let worker_infos = match Worker::find()
                     .select_only()
+                    .column(worker::Column::WorkerId)
                     .column(worker::Column::WorkerType)
                     .column(worker::Column::Host)
                     .column(worker::Column::Port)
                     .filter(worker::Column::WorkerId.is_in(worker_to_delete.clone()))
-                    .into_tuple::<(WorkerType, String, i32)>()
+                    .into_tuple::<(WorkerId, WorkerType, String, i32)>()
                     .all(&inner.db)
                     .await
                 {
@@ -276,33 +277,36 @@ impl ClusterController {
                         continue;
                     }
                 };
+                drop(inner);
 
-                if let Err(err) = Worker::delete_many()
-                    .filter(worker::Column::WorkerId.is_in(worker_to_delete))
-                    .exec(&inner.db)
-                    .await
-                {
-                    tracing::warn!(error = %err.as_report(), "Failed to delete expire workers from db");
-                    continue;
-                }
-
-                for (worker_type, host, port) in worker_infos {
-                    match worker_type {
-                        WorkerType::Frontend
-                        | WorkerType::ComputeNode
-                        | WorkerType::Compactor
-                        | WorkerType::RiseCtl => {
-                            cluster_controller
-                                .env
-                                .notification_manager()
-                                .delete_sender(
-                                    worker_type.into(),
-                                    WorkerKey(HostAddress { host, port }),
-                                )
-                                .await
+                for (worker_id, worker_type, host, port) in worker_infos {
+                    let host_addr = PbHostAddress { host, port };
+                    match cluster_controller.delete_worker(host_addr.clone()).await {
+                        Ok(_) => {
+                            tracing::warn!(
+                                worker_id,
+                                ?host_addr,
+                                %now,
+                                "Deleted expired worker"
+                            );
+                            match worker_type {
+                                WorkerType::Frontend
+                                | WorkerType::ComputeNode
+                                | WorkerType::Compactor
+                                | WorkerType::RiseCtl => {
+                                    cluster_controller
+                                        .env
+                                        .notification_manager()
+                                        .delete_sender(worker_type.into(), WorkerKey(host_addr))
+                                        .await
+                                }
+                                _ => {}
+                            };
                         }
-                        _ => {}
-                    };
+                        Err(err) => {
+                            tracing::warn!(error = %err.as_report(), "Failed to delete expire worker from db");
+                        }
+                    }
                 }
             }
         });
diff --git a/src/meta/src/controller/fragment.rs b/src/meta/src/controller/fragment.rs
index 552008914d76e..fa53773741484 100644
--- a/src/meta/src/controller/fragment.rs
+++ b/src/meta/src/controller/fragment.rs
@@ -18,6 +18,7 @@ use std::mem::swap;
 use anyhow::Context;
 use itertools::Itertools;
 use risingwave_common::bail;
+use risingwave_common::hash::ParallelUnitMapping;
 use risingwave_common::util::stream_graph_visitor::visit_stream_node;
 use risingwave_meta_model_v2::actor::ActorStatus;
 use risingwave_meta_model_v2::prelude::{Actor, ActorDispatcher, Fragment, Sink, StreamingJob};
@@ -34,7 +35,7 @@ use risingwave_pb::meta::table_fragments::actor_status::PbActorState;
 use risingwave_pb::meta::table_fragments::fragment::PbFragmentDistributionType;
 use risingwave_pb::meta::table_fragments::{PbActorStatus, PbFragment, PbState};
 use risingwave_pb::meta::{
-    FragmentParallelUnitMapping, PbFragmentParallelUnitMapping, PbTableFragments,
+    FragmentWorkerSlotMapping, PbFragmentWorkerSlotMapping, PbTableFragments,
 };
 use risingwave_pb::source::PbConnectorSplits;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
@@ -50,7 +51,8 @@ use sea_orm::{
 
 use crate::controller::catalog::{CatalogController, CatalogControllerInner};
 use crate::controller::utils::{
-    get_actor_dispatchers, FragmentDesc, PartialActorLocation, PartialFragmentStateTables,
+    get_actor_dispatchers, get_parallel_unit_to_worker_map, FragmentDesc, PartialActorLocation,
+    PartialFragmentStateTables,
 };
 use crate::manager::{ActorInfos, LocalNotification};
 use crate::model::TableParallelism;
@@ -61,7 +63,9 @@ impl CatalogControllerInner {
     /// List all fragment vnode mapping info for all CREATED streaming jobs.
     pub async fn all_running_fragment_mappings(
         &self,
-    ) -> MetaResult<impl Iterator<Item = FragmentParallelUnitMapping> + '_> {
+    ) -> MetaResult<impl Iterator<Item = FragmentWorkerSlotMapping> + '_> {
+        let txn = self.db.begin().await?;
+
         let fragment_mappings: Vec<(FragmentId, FragmentVnodeMapping)> = Fragment::find()
             .join(JoinType::InnerJoin, fragment::Relation::Object.def())
             .join(JoinType::InnerJoin, object::Relation::StreamingJob.def())
@@ -69,14 +73,24 @@ impl CatalogControllerInner {
             .columns([fragment::Column::FragmentId, fragment::Column::VnodeMapping])
             .filter(streaming_job::Column::JobStatus.eq(JobStatus::Created))
             .into_tuple()
-            .all(&self.db)
+            .all(&txn)
             .await?;
-        Ok(fragment_mappings.into_iter().map(|(fragment_id, mapping)| {
-            FragmentParallelUnitMapping {
-                fragment_id: fragment_id as _,
-                mapping: Some(mapping.to_protobuf()),
-            }
-        }))
+
+        let parallel_unit_to_worker = get_parallel_unit_to_worker_map(&txn).await?;
+
+        Ok(fragment_mappings
+            .into_iter()
+            .map(move |(fragment_id, mapping)| {
+                let worker_slot_mapping =
+                    ParallelUnitMapping::from_protobuf(&mapping.to_protobuf())
+                        .to_worker_slot(&parallel_unit_to_worker)
+                        .to_protobuf();
+
+                FragmentWorkerSlotMapping {
+                    fragment_id: fragment_id as _,
+                    mapping: Some(worker_slot_mapping),
+                }
+            }))
     }
 }
 
@@ -84,7 +98,7 @@ impl CatalogController {
     pub(crate) async fn notify_fragment_mapping(
         &self,
         operation: NotificationOperation,
-        fragment_mappings: Vec<PbFragmentParallelUnitMapping>,
+        fragment_mappings: Vec<PbFragmentWorkerSlotMapping>,
     ) {
         let fragment_ids = fragment_mappings
             .iter()
@@ -96,7 +110,7 @@ impl CatalogController {
                 .notification_manager()
                 .notify_frontend(
                     operation,
-                    NotificationInfo::ParallelUnitMapping(fragment_mapping),
+                    NotificationInfo::StreamingWorkerSlotMapping(fragment_mapping),
                 )
                 .await;
         }
@@ -936,15 +950,21 @@ impl CatalogController {
             .await?;
         }
 
+        let parallel_unit_to_worker = get_parallel_unit_to_worker_map(&txn).await?;
+
         txn.commit().await?;
 
         self.notify_fragment_mapping(
             NotificationOperation::Update,
             fragment_mapping
                 .into_iter()
-                .map(|(fragment_id, mapping)| PbFragmentParallelUnitMapping {
+                .map(|(fragment_id, mapping)| PbFragmentWorkerSlotMapping {
                     fragment_id: fragment_id as _,
-                    mapping: Some(mapping.to_protobuf()),
+                    mapping: Some(
+                        ParallelUnitMapping::from_protobuf(&mapping.to_protobuf())
+                            .to_worker_slot(&parallel_unit_to_worker)
+                            .to_protobuf(),
+                    ),
                 })
                 .collect(),
         )
@@ -1346,7 +1366,6 @@ impl CatalogController {
 #[cfg(test)]
 mod tests {
     use std::collections::{BTreeMap, HashMap};
-    use std::default::Default;
 
     use itertools::Itertools;
     use risingwave_common::hash::{ParallelUnitId, ParallelUnitMapping};
diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs
index a271ab85bb928..ac61ceb67b77f 100644
--- a/src/meta/src/controller/mod.rs
+++ b/src/meta/src/controller/mod.rs
@@ -12,17 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::BTreeMap;
+
 use anyhow::anyhow;
 use risingwave_common::util::epoch::Epoch;
 use risingwave_meta_model_v2::{
-    connection, database, function, index, object, schema, sink, source, subscription, table, view,
+    connection, database, function, index, object, schema, secret, sink, source, subscription,
+    table, view,
 };
 use risingwave_pb::catalog::connection::PbInfo as PbConnectionInfo;
 use risingwave_pb::catalog::source::PbOptionalAssociatedTableId;
+use risingwave_pb::catalog::subscription::PbSubscriptionState;
 use risingwave_pb::catalog::table::{PbOptionalAssociatedSourceId, PbTableType};
 use risingwave_pb::catalog::{
     PbConnection, PbCreateType, PbDatabase, PbFunction, PbHandleConflictBehavior, PbIndex,
-    PbSchema, PbSink, PbSinkType, PbSource, PbStreamJobStatus, PbSubscription, PbTable, PbView,
+    PbSchema, PbSecret, PbSink, PbSinkType, PbSource, PbStreamJobStatus, PbSubscription, PbTable,
+    PbView,
 };
 use sea_orm::{DatabaseConnection, ModelTrait};
 
@@ -81,6 +86,19 @@ impl From<ObjectModel<database::Model>> for PbDatabase {
     }
 }
 
+impl From<ObjectModel<secret::Model>> for PbSecret {
+    fn from(value: ObjectModel<secret::Model>) -> Self {
+        Self {
+            id: value.0.secret_id as _,
+            name: value.0.name,
+            database_id: value.1.database_id.unwrap() as _,
+            value: value.0.value,
+            owner: value.1.owner_id as _,
+            schema_id: value.1.schema_id.unwrap() as _,
+        }
+    }
+}
+
 impl From<ObjectModel<schema::Model>> for PbSchema {
     fn from(value: ObjectModel<schema::Model>) -> Self {
         Self {
@@ -183,6 +201,10 @@ impl From<ObjectModel<source::Model>> for PbSource {
 
 impl From<ObjectModel<sink::Model>> for PbSink {
     fn from(value: ObjectModel<sink::Model>) -> Self {
+        let mut secret_ref_map: BTreeMap<String, u32> = BTreeMap::new();
+        if let Some(secret_ref) = value.0.secret_ref {
+            secret_ref_map = secret_ref.into_inner();
+        }
         Self {
             id: value.0.sink_id as _,
             schema_id: value.1.schema_id.unwrap() as _,
@@ -212,6 +234,7 @@ impl From<ObjectModel<sink::Model>> for PbSink {
             initialized_at_cluster_version: value.1.initialized_at_cluster_version,
             created_at_cluster_version: value.1.created_at_cluster_version,
             create_type: PbCreateType::Foreground as _,
+            secret_ref: secret_ref_map,
         }
     }
 }
@@ -223,11 +246,8 @@ impl From<ObjectModel<subscription::Model>> for PbSubscription {
             schema_id: value.1.schema_id.unwrap() as _,
             database_id: value.1.database_id.unwrap() as _,
             name: value.0.name,
-            plan_pk: value.0.plan_pk.to_protobuf(),
-            dependent_relations: vec![], // todo: deprecate it.
-            distribution_key: value.0.distribution_key.0,
             owner: value.1.owner_id as _,
-            properties: value.0.properties.0,
+            retention_seconds: value.0.retention_seconds as _,
             definition: value.0.definition,
             initialized_at_epoch: Some(
                 Epoch::from_unix_millis(value.1.initialized_at.timestamp_millis() as _).0,
@@ -235,12 +255,10 @@ impl From<ObjectModel<subscription::Model>> for PbSubscription {
             created_at_epoch: Some(
                 Epoch::from_unix_millis(value.1.created_at.timestamp_millis() as _).0,
             ),
-            stream_job_status: PbStreamJobStatus::Created as _, // todo: deprecate it.
-            column_catalogs: value.0.columns.to_protobuf(),
-            subscription_from_name: value.0.subscription_from_name,
             initialized_at_cluster_version: value.1.initialized_at_cluster_version,
             created_at_cluster_version: value.1.created_at_cluster_version,
-            subscription_internal_table_name: value.0.subscription_internal_table_name,
+            dependent_table_id: value.0.dependent_table_id as _,
+            subscription_state: PbSubscriptionState::Init as _,
         }
     }
 }
diff --git a/src/meta/src/controller/rename.rs b/src/meta/src/controller/rename.rs
index 860981762cde4..84bbce4dc1ee3 100644
--- a/src/meta/src/controller/rename.rs
+++ b/src/meta/src/controller/rename.rs
@@ -170,7 +170,7 @@ impl QueryRewriter<'_> {
     ///
     /// So that we DON'T have to:
     /// 1. rewrite the select and expr part like `schema.table.column`, `table.column`,
-    /// `alias.column` etc.
+    ///    `alias.column` etc.
     /// 2. handle the case that the old name is used as alias.
     /// 3. handle the case that the new name is used as alias.
     fn visit_table_factor(&self, table_factor: &mut TableFactor) {
diff --git a/src/meta/src/controller/session_params.rs b/src/meta/src/controller/session_params.rs
index 4a27967fa2b0a..566170a0ef4d2 100644
--- a/src/meta/src/controller/session_params.rs
+++ b/src/meta/src/controller/session_params.rs
@@ -151,7 +151,7 @@ mod tests {
         use sea_orm::QueryFilter;
 
         let env = MetaSrvEnv::for_test_with_sql_meta_store().await;
-        let meta_store = env.meta_store().as_sql();
+        let meta_store = env.meta_store_ref().as_sql();
         let init_params = SessionConfig::default();
 
         // init system parameter controller as first launch.
diff --git a/src/meta/src/controller/streaming_job.rs b/src/meta/src/controller/streaming_job.rs
index 271860ee49561..3066ce223785e 100644
--- a/src/meta/src/controller/streaming_job.rs
+++ b/src/meta/src/controller/streaming_job.rs
@@ -16,11 +16,11 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::num::NonZeroUsize;
 
 use itertools::Itertools;
-use risingwave_common::bail;
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::hash::{ActorMapping, ParallelUnitId, ParallelUnitMapping};
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_common::util::stream_graph_visitor::visit_stream_node;
+use risingwave_common::{bail, current_cluster_version};
 use risingwave_meta_model_v2::actor::ActorStatus;
 use risingwave_meta_model_v2::actor_dispatcher::DispatcherType;
 use risingwave_meta_model_v2::object::ObjectType;
@@ -30,9 +30,9 @@ use risingwave_meta_model_v2::prelude::{
 };
 use risingwave_meta_model_v2::{
     actor, actor_dispatcher, fragment, index, object, object_dependency, sink, source,
-    streaming_job, subscription, table, ActorId, ActorUpstreamActors, CreateType, DatabaseId,
-    ExprNodeArray, FragmentId, I32Array, IndexId, JobStatus, ObjectId, SchemaId, SourceId,
-    StreamNode, StreamingParallelism, TableId, TableVersion, UserId,
+    streaming_job, table, ActorId, ActorUpstreamActors, CreateType, DatabaseId, ExprNodeArray,
+    FragmentId, I32Array, IndexId, JobStatus, ObjectId, SchemaId, SourceId, StreamNode,
+    StreamingParallelism, TableId, TableVersion, UserId,
 };
 use risingwave_pb::catalog::source::PbOptionalAssociatedTableId;
 use risingwave_pb::catalog::table::{PbOptionalAssociatedSourceId, PbTableVersion};
@@ -43,16 +43,17 @@ use risingwave_pb::meta::subscribe_response::{
 };
 use risingwave_pb::meta::table_fragments::PbActorStatus;
 use risingwave_pb::meta::{
-    FragmentParallelUnitMapping, PbRelation, PbRelationGroup, PbTableFragments,
+    FragmentWorkerSlotMapping, PbFragmentWorkerSlotMapping, PbRelation, PbRelationGroup,
+    PbTableFragments, Relation,
 };
 use risingwave_pb::source::{PbConnectorSplit, PbConnectorSplits};
 use risingwave_pb::stream_plan::stream_fragment_graph::Parallelism;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
-use risingwave_pb::stream_plan::update_mutation::PbMergeUpdate;
+use risingwave_pb::stream_plan::update_mutation::{MergeUpdate, PbMergeUpdate};
 use risingwave_pb::stream_plan::{
     PbDispatcher, PbDispatcherType, PbFragmentTypeFlag, PbStreamActor,
 };
-use sea_orm::sea_query::SimpleExpr;
+use sea_orm::sea_query::{Expr, Query, SimpleExpr};
 use sea_orm::ActiveValue::Set;
 use sea_orm::{
     ActiveEnum, ActiveModelTrait, ColumnTrait, DatabaseTransaction, EntityTrait, IntoActiveModel,
@@ -65,7 +66,7 @@ use crate::controller::catalog::CatalogController;
 use crate::controller::rename::ReplaceTableExprRewriter;
 use crate::controller::utils::{
     check_relation_name_duplicate, check_sink_into_table_cycle, ensure_object_id, ensure_user_id,
-    get_fragment_actor_ids, get_fragment_mappings,
+    get_fragment_actor_ids, get_fragment_mappings, get_parallel_unit_to_worker_map,
 };
 use crate::controller::ObjectModel;
 use crate::manager::{NotificationVersion, SinkId, StreamingJob};
@@ -123,6 +124,39 @@ impl CatalogController {
         )
         .await?;
 
+        // check if any dependent relation is in altering status.
+        let dependent_relations = streaming_job.dependent_relations();
+        if !dependent_relations.is_empty() {
+            let altering_cnt = ObjectDependency::find()
+                .join(
+                    JoinType::InnerJoin,
+                    object_dependency::Relation::Object1.def(),
+                )
+                .join(JoinType::InnerJoin, object::Relation::StreamingJob.def())
+                .filter(
+                    object_dependency::Column::Oid
+                        .is_in(dependent_relations.iter().map(|id| *id as ObjectId))
+                        .and(object::Column::ObjType.eq(ObjectType::Table))
+                        .and(streaming_job::Column::JobStatus.ne(JobStatus::Created))
+                        .and(
+                            // It means the referring table is just dummy for altering.
+                            object::Column::Oid.not_in_subquery(
+                                Query::select()
+                                    .column(table::Column::TableId)
+                                    .from(Table)
+                                    .to_owned(),
+                            ),
+                        ),
+                )
+                .count(&txn)
+                .await?;
+            if altering_cnt != 0 {
+                return Err(MetaError::permission_denied(
+                    "some dependent relations are being altered",
+                ));
+            }
+        }
+
         match streaming_job {
             StreamingJob::MaterializedView(table) => {
                 let job_id = Self::create_streaming_job_obj(
@@ -171,22 +205,6 @@ impl CatalogController {
                 let sink: sink::ActiveModel = sink.clone().into();
                 Sink::insert(sink).exec(&txn).await?;
             }
-            StreamingJob::Subscription(subscription) => {
-                let job_id = Self::create_streaming_job_obj(
-                    &txn,
-                    ObjectType::Subscription,
-                    subscription.owner as _,
-                    Some(subscription.database_id as _),
-                    Some(subscription.schema_id as _),
-                    create_type,
-                    ctx,
-                    streaming_parallelism,
-                )
-                .await?;
-                subscription.id = job_id as _;
-                let subscription: subscription::ActiveModel = subscription.clone().into();
-                subscription.insert(&txn).await?;
-            }
             StreamingJob::Table(src, table, _) => {
                 let job_id = Self::create_streaming_job_obj(
                     &txn,
@@ -271,7 +289,6 @@ impl CatalogController {
         }
 
         // record object dependency.
-        let dependent_relations = streaming_job.dependent_relations();
         if !dependent_relations.is_empty() {
             ObjectDependency::insert_many(dependent_relations.into_iter().map(|id| {
                 object_dependency::ActiveModel {
@@ -556,12 +573,33 @@ impl CatalogController {
             return Err(MetaError::permission_denied("table version is stale"));
         }
 
+        // 2. check concurrent replace.
+        let referring_cnt = ObjectDependency::find()
+            .join(
+                JoinType::InnerJoin,
+                object_dependency::Relation::Object1.def(),
+            )
+            .join(JoinType::InnerJoin, object::Relation::StreamingJob.def())
+            .filter(
+                object_dependency::Column::Oid
+                    .eq(id as ObjectId)
+                    .and(object::Column::ObjType.eq(ObjectType::Table))
+                    .and(streaming_job::Column::JobStatus.ne(JobStatus::Created)),
+            )
+            .count(&txn)
+            .await?;
+        if referring_cnt != 0 {
+            return Err(MetaError::permission_denied(
+                "table is being altered or referenced by some creating jobs",
+            ));
+        }
+
         let parallelism = match specified_parallelism {
             None => StreamingParallelism::Adaptive,
             Some(n) => StreamingParallelism::Fixed(n.get() as _),
         };
 
-        // 2. create streaming object for new replace table.
+        // 3. create streaming object for new replace table.
         let obj_id = Self::create_streaming_job_obj(
             &txn,
             ObjectType::Table,
@@ -574,7 +612,7 @@ impl CatalogController {
         )
         .await?;
 
-        // 3. record dependency for new replace table.
+        // 4. record dependency for new replace table.
         ObjectDependency::insert(object_dependency::ActiveModel {
             oid: Set(id as _),
             used_by: Set(obj_id as _),
@@ -588,6 +626,186 @@ impl CatalogController {
         Ok(obj_id)
     }
 
+    /// `finish_streaming_job` marks job related objects as `Created` and notify frontend.
+    pub async fn finish_streaming_job(
+        &self,
+        job_id: ObjectId,
+        replace_table_job_info: Option<(crate::manager::StreamingJob, Vec<MergeUpdate>, u32)>,
+    ) -> MetaResult<NotificationVersion> {
+        let inner = self.inner.write().await;
+        let txn = inner.db.begin().await?;
+
+        let job_type = Object::find_by_id(job_id)
+            .select_only()
+            .column(object::Column::ObjType)
+            .into_tuple()
+            .one(&txn)
+            .await?
+            .ok_or_else(|| MetaError::catalog_id_not_found("streaming job", job_id))?;
+
+        // update `created_at` as now() and `created_at_cluster_version` as current cluster version.
+        let res = Object::update_many()
+            .col_expr(object::Column::CreatedAt, Expr::current_timestamp().into())
+            .col_expr(
+                object::Column::CreatedAtClusterVersion,
+                current_cluster_version().into(),
+            )
+            .filter(object::Column::Oid.eq(job_id))
+            .exec(&txn)
+            .await?;
+        if res.rows_affected == 0 {
+            return Err(MetaError::catalog_id_not_found("streaming job", job_id));
+        }
+
+        // mark the target stream job as `Created`.
+        let job = streaming_job::ActiveModel {
+            job_id: Set(job_id),
+            job_status: Set(JobStatus::Created),
+            ..Default::default()
+        };
+        job.update(&txn).await?;
+
+        // notify frontend: job, internal tables.
+        let internal_table_objs = Table::find()
+            .find_also_related(Object)
+            .filter(table::Column::BelongsToJobId.eq(job_id))
+            .all(&txn)
+            .await?;
+        let mut relations = internal_table_objs
+            .iter()
+            .map(|(table, obj)| PbRelation {
+                relation_info: Some(PbRelationInfo::Table(
+                    ObjectModel(table.clone(), obj.clone().unwrap()).into(),
+                )),
+            })
+            .collect_vec();
+
+        match job_type {
+            ObjectType::Table => {
+                let (table, obj) = Table::find_by_id(job_id)
+                    .find_also_related(Object)
+                    .one(&txn)
+                    .await?
+                    .ok_or_else(|| MetaError::catalog_id_not_found("table", job_id))?;
+                if let Some(source_id) = table.optional_associated_source_id {
+                    let (src, obj) = Source::find_by_id(source_id)
+                        .find_also_related(Object)
+                        .one(&txn)
+                        .await?
+                        .ok_or_else(|| MetaError::catalog_id_not_found("source", source_id))?;
+                    relations.push(PbRelation {
+                        relation_info: Some(PbRelationInfo::Source(
+                            ObjectModel(src, obj.unwrap()).into(),
+                        )),
+                    });
+                }
+                relations.push(PbRelation {
+                    relation_info: Some(PbRelationInfo::Table(
+                        ObjectModel(table, obj.unwrap()).into(),
+                    )),
+                });
+            }
+            ObjectType::Sink => {
+                let (sink, obj) = Sink::find_by_id(job_id)
+                    .find_also_related(Object)
+                    .one(&txn)
+                    .await?
+                    .ok_or_else(|| MetaError::catalog_id_not_found("sink", job_id))?;
+                relations.push(PbRelation {
+                    relation_info: Some(PbRelationInfo::Sink(
+                        ObjectModel(sink, obj.unwrap()).into(),
+                    )),
+                });
+            }
+            ObjectType::Index => {
+                let (index, obj) = Index::find_by_id(job_id)
+                    .find_also_related(Object)
+                    .one(&txn)
+                    .await?
+                    .ok_or_else(|| MetaError::catalog_id_not_found("index", job_id))?;
+                {
+                    let (table, obj) = Table::find_by_id(index.index_table_id)
+                        .find_also_related(Object)
+                        .one(&txn)
+                        .await?
+                        .ok_or_else(|| {
+                            MetaError::catalog_id_not_found("table", index.index_table_id)
+                        })?;
+                    relations.push(PbRelation {
+                        relation_info: Some(PbRelationInfo::Table(
+                            ObjectModel(table, obj.unwrap()).into(),
+                        )),
+                    });
+                }
+                relations.push(PbRelation {
+                    relation_info: Some(PbRelationInfo::Index(
+                        ObjectModel(index, obj.unwrap()).into(),
+                    )),
+                });
+            }
+            ObjectType::Source => {
+                let (source, obj) = Source::find_by_id(job_id)
+                    .find_also_related(Object)
+                    .one(&txn)
+                    .await?
+                    .ok_or_else(|| MetaError::catalog_id_not_found("source", job_id))?;
+                relations.push(PbRelation {
+                    relation_info: Some(PbRelationInfo::Source(
+                        ObjectModel(source, obj.unwrap()).into(),
+                    )),
+                });
+            }
+            _ => unreachable!("invalid job type: {:?}", job_type),
+        }
+
+        let fragment_mapping = get_fragment_mappings(&txn, job_id).await?;
+
+        let replace_table_mapping_update = match replace_table_job_info {
+            Some((streaming_job, merge_updates, dummy_id)) => {
+                let incoming_sink_id = job_id;
+
+                let (relations, fragment_mapping) = Self::finish_replace_streaming_job_inner(
+                    dummy_id as ObjectId,
+                    merge_updates,
+                    None,
+                    Some(incoming_sink_id as _),
+                    None,
+                    &txn,
+                    streaming_job,
+                )
+                .await?;
+
+                Some((relations, fragment_mapping))
+            }
+            None => None,
+        };
+
+        txn.commit().await?;
+
+        self.notify_fragment_mapping(NotificationOperation::Add, fragment_mapping)
+            .await;
+
+        let mut version = self
+            .notify_frontend(
+                NotificationOperation::Add,
+                NotificationInfo::RelationGroup(PbRelationGroup { relations }),
+            )
+            .await;
+
+        if let Some((relations, fragment_mapping)) = replace_table_mapping_update {
+            self.notify_fragment_mapping(NotificationOperation::Add, fragment_mapping)
+                .await;
+            version = self
+                .notify_frontend(
+                    NotificationOperation::Update,
+                    NotificationInfo::RelationGroup(PbRelationGroup { relations }),
+                )
+                .await;
+        }
+
+        Ok(version)
+    }
+
     pub async fn finish_replace_streaming_job(
         &self,
         dummy_id: ObjectId,
@@ -597,13 +815,53 @@ impl CatalogController {
         creating_sink_id: Option<SinkId>,
         dropping_sink_id: Option<SinkId>,
     ) -> MetaResult<NotificationVersion> {
+        let inner = self.inner.write().await;
+        let txn = inner.db.begin().await?;
+
+        let (relations, fragment_mapping) = Self::finish_replace_streaming_job_inner(
+            dummy_id,
+            merge_updates,
+            table_col_index_mapping,
+            creating_sink_id,
+            dropping_sink_id,
+            &txn,
+            streaming_job,
+        )
+        .await?;
+
+        txn.commit().await?;
+
+        // FIXME: Do not notify frontend currently, because frontend nodes might refer to old table
+        // catalog and need to access the old fragment. Let frontend nodes delete the old fragment
+        // when they receive table catalog change.
+        // self.notify_fragment_mapping(NotificationOperation::Delete, old_fragment_mappings)
+        //     .await;
+        self.notify_fragment_mapping(NotificationOperation::Add, fragment_mapping)
+            .await;
+        let version = self
+            .notify_frontend(
+                NotificationOperation::Update,
+                NotificationInfo::RelationGroup(PbRelationGroup { relations }),
+            )
+            .await;
+
+        Ok(version)
+    }
+
+    pub async fn finish_replace_streaming_job_inner(
+        dummy_id: ObjectId,
+        merge_updates: Vec<PbMergeUpdate>,
+        table_col_index_mapping: Option<ColIndexMapping>,
+        creating_sink_id: Option<SinkId>,
+        dropping_sink_id: Option<SinkId>,
+        txn: &DatabaseTransaction,
+        streaming_job: StreamingJob,
+    ) -> MetaResult<(Vec<Relation>, Vec<PbFragmentWorkerSlotMapping>)> {
         // Question: The source catalog should be remain unchanged?
         let StreamingJob::Table(_, table, ..) = streaming_job else {
             unreachable!("unexpected job: {streaming_job:?}")
         };
 
-        let inner = self.inner.write().await;
-        let txn = inner.db.begin().await?;
         let job_id = table.id as ObjectId;
 
         let mut table = table::ActiveModel::from(table);
@@ -621,7 +879,7 @@ impl CatalogController {
         }
 
         table.incoming_sinks = Set(incoming_sinks.into());
-        let table = table.update(&txn).await?;
+        let table = table.update(txn).await?;
 
         // Update state table fragment id.
         let fragment_table_ids: Vec<(FragmentId, I32Array)> = Fragment::find()
@@ -632,7 +890,7 @@ impl CatalogController {
             ])
             .filter(fragment::Column::JobId.eq(dummy_id))
             .into_tuple()
-            .all(&txn)
+            .all(txn)
             .await?;
         for (fragment_id, state_table_ids) in fragment_table_ids {
             for state_table_id in state_table_ids.into_inner() {
@@ -641,7 +899,7 @@ impl CatalogController {
                     fragment_id: Set(Some(fragment_id)),
                     ..Default::default()
                 }
-                .update(&txn)
+                .update(txn)
                 .await?;
             }
         }
@@ -650,12 +908,12 @@ impl CatalogController {
         // 1. replace old fragments/actors with new ones.
         Fragment::delete_many()
             .filter(fragment::Column::JobId.eq(job_id))
-            .exec(&txn)
+            .exec(txn)
             .await?;
         Fragment::update_many()
             .col_expr(fragment::Column::JobId, SimpleExpr::from(job_id))
             .filter(fragment::Column::JobId.eq(dummy_id))
-            .exec(&txn)
+            .exec(txn)
             .await?;
 
         // 2. update merges.
@@ -686,7 +944,7 @@ impl CatalogController {
                         actor::Column::UpstreamActorIds,
                     ])
                     .into_tuple::<(ActorId, FragmentId, ActorUpstreamActors)>()
-                    .one(&txn)
+                    .one(txn)
                     .await?
                     .ok_or_else(|| {
                         MetaError::catalog_id_not_found("actor", merge_update.actor_id)
@@ -709,7 +967,7 @@ impl CatalogController {
                 upstream_actor_ids: Set(upstream_actors),
                 ..Default::default()
             }
-            .update(&txn)
+            .update(txn)
             .await?;
 
             to_update_fragment_ids.insert(fragment_id);
@@ -724,7 +982,7 @@ impl CatalogController {
                         fragment::Column::UpstreamFragmentId,
                     ])
                     .into_tuple::<(FragmentId, StreamNode, I32Array)>()
-                    .one(&txn)
+                    .one(txn)
                     .await?
                     .map(|(id, node, upstream)| (id, node.to_protobuf(), upstream))
                     .ok_or_else(|| MetaError::catalog_id_not_found("fragment", fragment_id))?;
@@ -748,18 +1006,18 @@ impl CatalogController {
                 upstream_fragment_id: Set(upstream_fragment_id),
                 ..Default::default()
             }
-            .update(&txn)
+            .update(txn)
             .await?;
         }
 
         // 3. remove dummy object.
-        Object::delete_by_id(dummy_id).exec(&txn).await?;
+        Object::delete_by_id(dummy_id).exec(txn).await?;
 
         // 4. update catalogs and notify.
         let mut relations = vec![];
         let table_obj = table
             .find_related(Object)
-            .one(&txn)
+            .one(txn)
             .await?
             .ok_or_else(|| MetaError::catalog_id_not_found("object", table.table_id))?;
         relations.push(PbRelation {
@@ -775,7 +1033,7 @@ impl CatalogController {
                 .columns([index::Column::IndexId, index::Column::IndexItems])
                 .filter(index::Column::PrimaryTableId.eq(job_id))
                 .into_tuple()
-                .all(&txn)
+                .all(txn)
                 .await?;
             for (index_id, nodes) in index_items {
                 let mut pb_nodes = nodes.to_protobuf();
@@ -787,11 +1045,11 @@ impl CatalogController {
                     index_items: Set(pb_nodes.into()),
                     ..Default::default()
                 }
-                .update(&txn)
+                .update(txn)
                 .await?;
                 let index_obj = index
                     .find_related(Object)
-                    .one(&txn)
+                    .one(txn)
                     .await?
                     .ok_or_else(|| MetaError::catalog_id_not_found("object", index.index_id))?;
                 relations.push(PbRelation {
@@ -801,25 +1059,10 @@ impl CatalogController {
                 });
             }
         }
-        let fragment_mapping = get_fragment_mappings(&txn, job_id).await?;
 
-        txn.commit().await?;
-
-        // FIXME: Do not notify frontend currently, because frontend nodes might refer to old table
-        // catalog and need to access the old fragment. Let frontend nodes delete the old fragment
-        // when they receive table catalog change.
-        // self.notify_fragment_mapping(NotificationOperation::Delete, old_fragment_mappings)
-        //     .await;
-        self.notify_fragment_mapping(NotificationOperation::Add, fragment_mapping)
-            .await;
-        let version = self
-            .notify_frontend(
-                NotificationOperation::Update,
-                NotificationInfo::RelationGroup(PbRelationGroup { relations }),
-            )
-            .await;
+        let fragment_mapping: Vec<_> = get_fragment_mappings(txn, job_id as _).await?;
 
-        Ok(version)
+        Ok((relations, fragment_mapping))
     }
 
     /// `try_abort_replacing_streaming_job` is used to abort the replacing streaming job, the input `job_id` is the dummy job id.
@@ -1033,6 +1276,8 @@ impl CatalogController {
 
         let txn = inner.db.begin().await?;
 
+        let parallel_unit_to_worker = get_parallel_unit_to_worker_map(&txn).await?;
+
         let mut fragment_mapping_to_notify = vec![];
 
         // for assert only
@@ -1062,10 +1307,6 @@ impl CatalogController {
                 .exec(&txn)
                 .await?;
 
-            // newly created actor
-            let mut new_actors = vec![];
-            let mut new_actor_dispatchers = vec![];
-
             for (
                 PbStreamActor {
                     actor_id,
@@ -1085,6 +1326,7 @@ impl CatalogController {
             ) in newly_created_actors
             {
                 let mut actor_upstreams = BTreeMap::<FragmentId, BTreeSet<ActorId>>::new();
+                let mut new_actor_dispatchers = vec![];
 
                 if let Some(nodes) = &mut nodes {
                     visit_stream_node(nodes, |node| {
@@ -1123,7 +1365,7 @@ impl CatalogController {
                     .get(&actor_id)
                     .map(|splits| splits.iter().map(PbConnectorSplit::from).collect_vec());
 
-                new_actors.push(actor::ActiveModel {
+                Actor::insert(actor::ActiveModel {
                     actor_id: Set(actor_id as _),
                     fragment_id: Set(fragment_id as _),
                     status: Set(ActorStatus::Running),
@@ -1133,7 +1375,9 @@ impl CatalogController {
                     upstream_actor_ids: Set(actor_upstreams),
                     vnode_bitmap: Set(vnode_bitmap.as_ref().map(|bitmap| bitmap.into())),
                     expr_context: Set(expr_context.as_ref().unwrap().into()),
-                });
+                })
+                .exec(&txn)
+                .await?;
 
                 for PbDispatcher {
                     r#type: dispatcher_type,
@@ -1157,16 +1401,11 @@ impl CatalogController {
                         downstream_actor_ids: Set(downstream_actor_id.into()),
                     })
                 }
-            }
-
-            if !new_actors.is_empty() {
-                Actor::insert_many(new_actors).exec(&txn).await?;
-            }
-
-            if !new_actor_dispatchers.is_empty() {
-                ActorDispatcher::insert_many(new_actor_dispatchers)
-                    .exec(&txn)
-                    .await?;
+                if !new_actor_dispatchers.is_empty() {
+                    ActorDispatcher::insert_many(new_actor_dispatchers)
+                        .exec(&txn)
+                        .await?;
+                }
             }
 
             // actor update
@@ -1214,9 +1453,13 @@ impl CatalogController {
             fragment.vnode_mapping = Set((&vnode_mapping).into());
             fragment.update(&txn).await?;
 
-            fragment_mapping_to_notify.push(FragmentParallelUnitMapping {
+            let worker_slot_mapping = ParallelUnitMapping::from_protobuf(&vnode_mapping)
+                .to_worker_slot(&parallel_unit_to_worker)
+                .to_protobuf();
+
+            fragment_mapping_to_notify.push(FragmentWorkerSlotMapping {
                 fragment_id: fragment_id as u32,
-                mapping: Some(vnode_mapping),
+                mapping: Some(worker_slot_mapping),
             });
 
             // for downstream and upstream
diff --git a/src/meta/src/controller/utils.rs b/src/meta/src/controller/utils.rs
index 173634ab03cc7..b98788248a115 100644
--- a/src/meta/src/controller/utils.rs
+++ b/src/meta/src/controller/utils.rs
@@ -16,6 +16,7 @@ use std::collections::{BTreeSet, HashMap, HashSet};
 
 use anyhow::anyhow;
 use itertools::Itertools;
+use risingwave_common::hash::ParallelUnitMapping;
 use risingwave_meta_model_migration::WithQuery;
 use risingwave_meta_model_v2::actor::ActorStatus;
 use risingwave_meta_model_v2::fragment::DistributionType;
@@ -23,12 +24,12 @@ use risingwave_meta_model_v2::object::ObjectType;
 use risingwave_meta_model_v2::prelude::*;
 use risingwave_meta_model_v2::{
     actor, actor_dispatcher, connection, database, fragment, function, index, object,
-    object_dependency, schema, sink, source, table, user, user_privilege, view, ActorId,
-    DataTypeArray, DatabaseId, FragmentId, FragmentVnodeMapping, I32Array, ObjectId, PrivilegeId,
-    SchemaId, SourceId, StreamNode, UserId,
+    object_dependency, schema, secret, sink, source, subscription, table, user, user_privilege,
+    view, worker_property, ActorId, DataTypeArray, DatabaseId, FragmentId, FragmentVnodeMapping,
+    I32Array, ObjectId, PrivilegeId, SchemaId, SourceId, StreamNode, UserId, WorkerId,
 };
-use risingwave_pb::catalog::{PbConnection, PbFunction};
-use risingwave_pb::meta::PbFragmentParallelUnitMapping;
+use risingwave_pb::catalog::{PbConnection, PbFunction, PbSecret, PbSubscription};
+use risingwave_pb::meta::{PbFragmentParallelUnitMapping, PbFragmentWorkerSlotMapping};
 use risingwave_pb::stream_plan::stream_node::NodeBody;
 use risingwave_pb::stream_plan::{PbFragmentTypeFlag, PbStreamNode, StreamSource};
 use risingwave_pb::user::grant_privilege::{PbAction, PbActionWithGrantOption, PbObject};
@@ -410,6 +411,54 @@ where
     Ok(())
 }
 
+pub async fn check_secret_name_duplicate<C>(pb_secret: &PbSecret, db: &C) -> MetaResult<()>
+where
+    C: ConnectionTrait,
+{
+    let count = Secret::find()
+        .inner_join(Object)
+        .filter(
+            object::Column::DatabaseId
+                .eq(pb_secret.database_id as DatabaseId)
+                .and(object::Column::SchemaId.eq(pb_secret.schema_id as SchemaId))
+                .and(secret::Column::Name.eq(&pb_secret.name)),
+        )
+        .count(db)
+        .await?;
+    if count > 0 {
+        assert_eq!(count, 1);
+        return Err(MetaError::catalog_duplicated("secret", &pb_secret.name));
+    }
+    Ok(())
+}
+
+pub async fn check_subscription_name_duplicate<C>(
+    pb_subscription: &PbSubscription,
+    db: &C,
+) -> MetaResult<()>
+where
+    C: ConnectionTrait,
+{
+    let count = Subscription::find()
+        .inner_join(Object)
+        .filter(
+            object::Column::DatabaseId
+                .eq(pb_subscription.database_id as DatabaseId)
+                .and(object::Column::SchemaId.eq(pb_subscription.schema_id as SchemaId))
+                .and(subscription::Column::Name.eq(&pb_subscription.name)),
+        )
+        .count(db)
+        .await?;
+    if count > 0 {
+        assert_eq!(count, 1);
+        return Err(MetaError::catalog_duplicated(
+            "subscription",
+            &pb_subscription.name,
+        ));
+    }
+    Ok(())
+}
+
 /// `check_user_name_duplicate` checks whether the user is already existed in the cluster.
 pub async fn check_user_name_duplicate<C>(name: &str, db: &C) -> MetaResult<()>
 where
@@ -727,14 +776,14 @@ where
             let obj = match object.obj_type {
                 ObjectType::Database => PbObject::DatabaseId(oid),
                 ObjectType::Schema => PbObject::SchemaId(oid),
-                ObjectType::Table => PbObject::TableId(oid),
+                ObjectType::Table | ObjectType::Index => PbObject::TableId(oid),
                 ObjectType::Source => PbObject::SourceId(oid),
                 ObjectType::Sink => PbObject::SinkId(oid),
                 ObjectType::View => PbObject::ViewId(oid),
                 ObjectType::Function => PbObject::FunctionId(oid),
-                ObjectType::Index => unreachable!("index is not supported yet"),
                 ObjectType::Connection => unreachable!("connection is not supported yet"),
                 ObjectType::Subscription => PbObject::SubscriptionId(oid),
+                ObjectType::Secret => unreachable!("secret is not supported yet"),
             };
             PbGrantPrivilege {
                 action_with_opts: vec![PbActionWithGrantOption {
@@ -789,10 +838,12 @@ where
 pub async fn get_fragment_mappings<C>(
     db: &C,
     job_id: ObjectId,
-) -> MetaResult<Vec<PbFragmentParallelUnitMapping>>
+) -> MetaResult<Vec<PbFragmentWorkerSlotMapping>>
 where
     C: ConnectionTrait,
 {
+    let parallel_unit_to_worker = get_parallel_unit_to_worker_map(db).await?;
+
     let fragment_mappings: Vec<(FragmentId, FragmentVnodeMapping)> = Fragment::find()
         .select_only()
         .columns([fragment::Column::FragmentId, fragment::Column::VnodeMapping])
@@ -803,9 +854,13 @@ where
 
     Ok(fragment_mappings
         .into_iter()
-        .map(|(fragment_id, mapping)| PbFragmentParallelUnitMapping {
+        .map(|(fragment_id, mapping)| PbFragmentWorkerSlotMapping {
             fragment_id: fragment_id as _,
-            mapping: Some(mapping.to_protobuf()),
+            mapping: Some(
+                ParallelUnitMapping::from_protobuf(&mapping.to_protobuf())
+                    .to_worker_slot(&parallel_unit_to_worker)
+                    .to_protobuf(),
+            ),
         })
         .collect())
 }
@@ -922,3 +977,30 @@ where
 
     Ok((source_fragment_ids, actors.into_iter().collect()))
 }
+
+pub(crate) async fn get_parallel_unit_to_worker_map<C>(db: &C) -> MetaResult<HashMap<u32, u32>>
+where
+    C: ConnectionTrait,
+{
+    let worker_parallel_units = WorkerProperty::find()
+        .select_only()
+        .columns([
+            worker_property::Column::WorkerId,
+            worker_property::Column::ParallelUnitIds,
+        ])
+        .into_tuple::<(WorkerId, I32Array)>()
+        .all(db)
+        .await?;
+
+    let parallel_unit_to_worker = worker_parallel_units
+        .into_iter()
+        .flat_map(|(worker_id, parallel_unit_ids)| {
+            parallel_unit_ids
+                .into_inner()
+                .into_iter()
+                .map(move |parallel_unit_id| (parallel_unit_id as u32, worker_id as u32))
+        })
+        .collect::<HashMap<_, _>>();
+
+    Ok(parallel_unit_to_worker)
+}
diff --git a/src/meta/src/dashboard/mod.rs b/src/meta/src/dashboard/mod.rs
index 6d05666819437..1229554032614 100644
--- a/src/meta/src/dashboard/mod.rs
+++ b/src/meta/src/dashboard/mod.rs
@@ -24,6 +24,7 @@ use axum::http::{Method, StatusCode};
 use axum::response::{IntoResponse, Response};
 use axum::routing::get;
 use axum::Router;
+use risingwave_common::util::StackTraceResponseExt;
 use risingwave_rpc_client::ComputeClientPool;
 use tokio::net::TcpListener;
 use tower::ServiceBuilder;
@@ -54,7 +55,7 @@ pub(super) mod handlers {
     use itertools::Itertools;
     use risingwave_common_heap_profiling::COLLAPSED_SUFFIX;
     use risingwave_pb::catalog::table::TableType;
-    use risingwave_pb::catalog::{Sink, Source, Table, View};
+    use risingwave_pb::catalog::{PbDatabase, PbSchema, Sink, Source, Subscription, Table, View};
     use risingwave_pb::common::{WorkerNode, WorkerType};
     use risingwave_pb::meta::list_object_dependencies_response::PbObjectDependencies;
     use risingwave_pb::meta::PbTableFragments;
@@ -62,6 +63,7 @@ pub(super) mod handlers {
         GetBackPressureResponse, HeapProfilingResponse, ListHeapProfilingResponse,
         StackTraceResponse,
     };
+    use risingwave_pb::user::PbUserInfo;
     use serde_json::json;
     use thiserror_ext::AsReport;
 
@@ -139,6 +141,21 @@ pub(super) mod handlers {
         list_table_catalogs_inner(&srv.metadata_manager, TableType::Index).await
     }
 
+    pub async fn list_subscription(
+        Extension(srv): Extension<Service>,
+    ) -> Result<Json<Vec<Subscription>>> {
+        let subscriptions = match &srv.metadata_manager {
+            MetadataManager::V1(mgr) => mgr.catalog_manager.list_subscriptions().await,
+            MetadataManager::V2(mgr) => mgr
+                .catalog_controller
+                .list_subscriptions()
+                .await
+                .map_err(err)?,
+        };
+
+        Ok(Json(subscriptions))
+    }
+
     pub async fn list_internal_tables(
         Extension(srv): Extension<Service>,
     ) -> Result<Json<Vec<Table>>> {
@@ -194,6 +211,37 @@ pub(super) mod handlers {
         Ok(Json(table_fragments))
     }
 
+    pub async fn list_users(Extension(srv): Extension<Service>) -> Result<Json<Vec<PbUserInfo>>> {
+        let users = match &srv.metadata_manager {
+            MetadataManager::V1(mgr) => mgr.catalog_manager.list_users().await,
+            MetadataManager::V2(mgr) => mgr.catalog_controller.list_users().await.map_err(err)?,
+        };
+
+        Ok(Json(users))
+    }
+
+    pub async fn list_databases(
+        Extension(srv): Extension<Service>,
+    ) -> Result<Json<Vec<PbDatabase>>> {
+        let databases = match &srv.metadata_manager {
+            MetadataManager::V1(mgr) => mgr.catalog_manager.list_databases().await,
+            MetadataManager::V2(mgr) => {
+                mgr.catalog_controller.list_databases().await.map_err(err)?
+            }
+        };
+
+        Ok(Json(databases))
+    }
+
+    pub async fn list_schemas(Extension(srv): Extension<Service>) -> Result<Json<Vec<PbSchema>>> {
+        let schemas = match &srv.metadata_manager {
+            MetadataManager::V1(mgr) => mgr.catalog_manager.list_schemas().await,
+            MetadataManager::V2(mgr) => mgr.catalog_controller.list_schemas().await.map_err(err)?,
+        };
+
+        Ok(Json(schemas))
+    }
+
     pub async fn list_object_dependencies(
         Extension(srv): Extension<Service>,
     ) -> Result<Json<Vec<PbObjectDependencies>>> {
@@ -213,20 +261,13 @@ pub(super) mod handlers {
         worker_nodes: impl IntoIterator<Item = &WorkerNode>,
         compute_clients: &ComputeClientPool,
     ) -> Result<Json<StackTraceResponse>> {
-        let mut all = Default::default();
-
-        fn merge(a: &mut StackTraceResponse, b: StackTraceResponse) {
-            a.actor_traces.extend(b.actor_traces);
-            a.rpc_traces.extend(b.rpc_traces);
-            a.compaction_task_traces.extend(b.compaction_task_traces);
-            a.inflight_barrier_traces.extend(b.inflight_barrier_traces);
-        }
+        let mut all = StackTraceResponse::default();
 
         for worker_node in worker_nodes {
             let client = compute_clients.get(worker_node).await.map_err(err)?;
             let result = client.stack_trace().await.map_err(err)?;
 
-            merge(&mut all, result);
+            all.merge_other(result);
         }
 
         Ok(all.into())
@@ -391,9 +432,13 @@ impl DashboardService {
             .route("/materialized_views", get(list_materialized_views))
             .route("/tables", get(list_tables))
             .route("/indexes", get(list_indexes))
+            .route("/subscriptions", get(list_subscription))
             .route("/internal_tables", get(list_internal_tables))
             .route("/sources", get(list_sources))
             .route("/sinks", get(list_sinks))
+            .route("/users", get(list_users))
+            .route("/databases", get(list_databases))
+            .route("/schemas", get(list_schemas))
             .route("/object_dependencies", get(list_object_dependencies))
             .route("/metrics/cluster", get(prometheus::list_prometheus_cluster))
             .route(
diff --git a/src/meta/src/hummock/compaction/compaction_config.rs b/src/meta/src/hummock/compaction/compaction_config.rs
index 6048e301c97d6..de91bf4f79ded 100644
--- a/src/meta/src/hummock/compaction/compaction_config.rs
+++ b/src/meta/src/hummock/compaction/compaction_config.rs
@@ -17,8 +17,6 @@ use risingwave_common::config::CompactionConfig as CompactionConfigOpt;
 use risingwave_pb::hummock::compaction_config::CompactionMode;
 use risingwave_pb::hummock::CompactionConfig;
 
-const MAX_LEVEL: u64 = 6;
-
 pub struct CompactionConfigBuilder {
     config: CompactionConfig,
 }
@@ -29,7 +27,7 @@ impl CompactionConfigBuilder {
             config: CompactionConfig {
                 max_bytes_for_level_base: compaction_config::max_bytes_for_level_base(),
                 max_bytes_for_level_multiplier: compaction_config::max_bytes_for_level_multiplier(),
-                max_level: MAX_LEVEL,
+                max_level: compaction_config::max_level() as u64,
                 max_compaction_bytes: compaction_config::max_compaction_bytes(),
                 sub_level_max_compaction_bytes: compaction_config::sub_level_max_compaction_bytes(),
                 level0_tier_compact_file_number: compaction_config::level0_tier_compact_file_number(
@@ -66,6 +64,7 @@ impl CompactionConfigBuilder {
                     compaction_config::level0_overlapping_sub_level_compact_level_count(),
                 tombstone_reclaim_ratio: compaction_config::tombstone_reclaim_ratio(),
                 enable_emergency_picker: compaction_config::enable_emergency_picker(),
+                max_l0_compact_level_count: compaction_config::max_l0_compact_level_count(),
             },
         }
     }
@@ -94,6 +93,7 @@ impl CompactionConfigBuilder {
             .max_space_reclaim_bytes(opt.max_space_reclaim_bytes)
             .level0_max_compact_file_number(opt.level0_max_compact_file_number)
             .tombstone_reclaim_ratio(opt.tombstone_reclaim_ratio)
+            .max_level(opt.max_level as u64)
     }
 
     pub fn build(self) -> CompactionConfig {
diff --git a/src/meta/src/hummock/compaction/mod.rs b/src/meta/src/hummock/compaction/mod.rs
index 49eeaa5778ab7..bf4b608fe59ad 100644
--- a/src/meta/src/hummock/compaction/mod.rs
+++ b/src/meta/src/hummock/compaction/mod.rs
@@ -16,12 +16,12 @@
 
 pub mod compaction_config;
 mod overlap_strategy;
-use risingwave_common::catalog::TableOption;
+use risingwave_common::catalog::{TableId, TableOption};
 use risingwave_pb::hummock::compact_task::{self, TaskType};
 
 mod picker;
 pub mod selector;
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeSet, HashMap, HashSet};
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
@@ -92,6 +92,7 @@ impl CompactStatus {
     pub fn get_compact_task(
         &mut self,
         levels: &Levels,
+        member_table_ids: &BTreeSet<TableId>,
         task_id: HummockCompactionTaskId,
         group: &CompactionGroup,
         stats: &mut LocalSelectorStatistic,
@@ -106,6 +107,7 @@ impl CompactStatus {
             task_id,
             group,
             levels,
+            member_table_ids,
             &mut self.level_handlers,
             stats,
             table_id_to_options.clone(),
@@ -121,6 +123,7 @@ impl CompactStatus {
                     task_id,
                     group,
                     levels,
+                    member_table_ids,
                     &mut self.level_handlers,
                     stats,
                     table_id_to_options,
@@ -193,12 +196,6 @@ pub fn create_compaction_task(
 ) -> CompactionTask {
     let target_file_size = if input.target_level == 0 {
         compaction_config.target_file_size_base
-    } else if input.target_level == base_level {
-        // This is just a temporary optimization measure. We hope to reduce the size of SST as much
-        // as possible to reduce the amount of data blocked by a single task during compaction,
-        // but too many files will increase computing overhead.
-        // TODO: remove it after can reduce configuration `target_file_size_base`.
-        compaction_config.target_file_size_base / 4
     } else {
         assert!(input.target_level >= base_level);
         let step = (input.target_level - base_level) / 2;
diff --git a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs
index 6617b9496fe10..bf05afc6c6e88 100644
--- a/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/base_level_compaction_picker.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::cell::RefCell;
 use std::sync::Arc;
 
 use itertools::Itertools;
@@ -28,6 +29,10 @@ use crate::hummock::compaction::picker::TrivialMovePicker;
 use crate::hummock::compaction::{create_overlap_strategy, CompactionDeveloperConfig};
 use crate::hummock::level_handler::LevelHandler;
 
+std::thread_local! {
+    static LOG_COUNTER: RefCell<usize> = const { RefCell::new(0) };
+}
+
 pub struct LevelCompactionPicker {
     target_level: usize,
     config: Arc<CompactionConfig>,
@@ -161,6 +166,7 @@ impl LevelCompactionPicker {
             self.config.level0_max_compact_file_number,
             overlap_strategy.clone(),
             self.developer_config.enable_check_task_level_overlap,
+            self.config.max_l0_compact_level_count as usize,
         );
 
         let mut max_vnode_partition_idx = 0;
@@ -253,13 +259,21 @@ impl LevelCompactionPicker {
                 stats,
             ) {
                 if l0.total_file_size > target_level.total_file_size * 8 {
-                    tracing::warn!("skip task with level count: {}, file count: {}, select size: {}, target size: {}, target level size: {}",
-                        result.input_levels.len(),
-                        result.total_file_count,
-                        result.select_input_size,
-                        result.target_input_size,
-                        target_level.total_file_size,
-                    );
+                    let log_counter = LOG_COUNTER.with_borrow_mut(|counter| {
+                        *counter += 1;
+                        *counter
+                    });
+
+                    // reduce log
+                    if log_counter % 100 == 0 {
+                        tracing::warn!("skip task with level count: {}, file count: {}, select size: {}, target size: {}, target level size: {}",
+                            result.input_levels.len(),
+                            result.total_file_count,
+                            result.select_input_size,
+                            result.target_input_size,
+                            target_level.total_file_size,
+                        );
+                    }
                 }
                 continue;
             }
@@ -272,14 +286,11 @@ impl LevelCompactionPicker {
 
 #[cfg(test)]
 pub mod tests {
-    use itertools::Itertools;
 
     use super::*;
     use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
     use crate::hummock::compaction::selector::tests::*;
-    use crate::hummock::compaction::{
-        CompactionDeveloperConfig, CompactionMode, TierCompactionPicker,
-    };
+    use crate::hummock::compaction::{CompactionMode, TierCompactionPicker};
 
     fn create_compaction_picker_for_test() -> LevelCompactionPicker {
         let config = Arc::new(
@@ -315,7 +326,6 @@ pub mod tests {
                     generate_table(1, 1, 201, 210, 1),
                 ],
             )],
-            member_table_ids: vec![1],
             ..Default::default()
         };
         let mut local_stats = LocalPickerStatistic::default();
@@ -408,7 +418,6 @@ pub mod tests {
                 total_file_size: 0,
                 uncompressed_file_size: 0,
             }),
-            member_table_ids: vec![1],
             ..Default::default()
         };
         push_tables_level0_nonoverlapping(&mut levels, vec![generate_table(1, 1, 50, 140, 2)]);
@@ -471,7 +480,6 @@ pub mod tests {
                 total_file_size: 0,
                 uncompressed_file_size: 0,
             }),
-            member_table_ids: vec![1],
             ..Default::default()
         };
         push_tables_level0_nonoverlapping(
@@ -575,7 +583,6 @@ pub mod tests {
         let levels = Levels {
             l0: Some(l0),
             levels: vec![generate_level(1, vec![generate_table(3, 1, 0, 100000, 1)])],
-            member_table_ids: vec![1],
             ..Default::default()
         };
         let levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
@@ -653,7 +660,6 @@ pub mod tests {
         let levels = Levels {
             l0: Some(l0),
             levels: vec![generate_level(1, vec![generate_table(3, 1, 0, 100000, 1)])],
-            member_table_ids: vec![1],
             ..Default::default()
         };
         let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
@@ -720,7 +726,6 @@ pub mod tests {
         let levels = Levels {
             l0: Some(l0),
             levels: vec![generate_level(1, vec![generate_table(3, 1, 1, 100, 1)])],
-            member_table_ids: vec![1],
             ..Default::default()
         };
         let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
diff --git a/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs b/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs
index 29119ae283b0a..c7dd27a6b1907 100644
--- a/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs
+++ b/src/meta/src/hummock/compaction/picker/compaction_task_validator.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 
 use risingwave_pb::hummock::CompactionConfig;
 
-use super::{CompactionInput, LocalPickerStatistic, MAX_COMPACT_LEVEL_COUNT};
+use super::{CompactionInput, LocalPickerStatistic};
 
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum ValidationRuleType {
@@ -89,14 +89,8 @@ struct TierCompactionTaskValidationRule {
 
 impl CompactionTaskValidationRule for TierCompactionTaskValidationRule {
     fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool {
-        // Limit sstable file count to avoid using too much memory.
-        let overlapping_max_compact_file_numer = std::cmp::min(
-            self.config.level0_max_compact_file_number,
-            MAX_COMPACT_LEVEL_COUNT as u64,
-        );
-
-        if input.total_file_count >= overlapping_max_compact_file_numer
-            || input.input_levels.len() >= MAX_COMPACT_LEVEL_COUNT
+        if input.total_file_count >= self.config.level0_max_compact_file_number
+            || input.input_levels.len() >= self.config.max_l0_compact_level_count as usize
         {
             return true;
         }
@@ -130,7 +124,7 @@ impl CompactionTaskValidationRule for IntraCompactionTaskValidationRule {
     fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool {
         if (input.total_file_count >= self.config.level0_max_compact_file_number
             && input.input_levels.len() > 1)
-            || input.input_levels.len() >= MAX_COMPACT_LEVEL_COUNT
+            || input.input_levels.len() >= self.config.max_l0_compact_level_count as usize
         {
             return true;
         }
@@ -178,7 +172,7 @@ struct BaseCompactionTaskValidationRule {
 impl CompactionTaskValidationRule for BaseCompactionTaskValidationRule {
     fn validate(&self, input: &CompactionInput, stats: &mut LocalPickerStatistic) -> bool {
         if input.total_file_count >= self.config.level0_max_compact_file_number
-            || input.input_levels.len() >= MAX_COMPACT_LEVEL_COUNT
+            || input.input_levels.len() >= self.config.max_l0_compact_level_count as usize
         {
             return true;
         }
diff --git a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs
index a6c4344532d77..5cc65bd38a1c8 100644
--- a/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/intra_compaction_picker.rs
@@ -44,19 +44,9 @@ impl CompactionPicker for IntraCompactionPicker {
         if l0.sub_levels.is_empty() {
             return None;
         }
-        if l0.sub_levels[0].level_type != LevelType::Nonoverlapping as i32
-            && l0.sub_levels[0].table_infos.len() > 1
-        {
-            stats.skip_by_overlapping += 1;
-            return None;
-        }
-
-        let is_l0_pending_compact =
-            level_handlers[0].is_level_all_pending_compact(&l0.sub_levels[0]);
 
-        if is_l0_pending_compact {
-            stats.skip_by_pending_files += 1;
-            return None;
+        if let Some(ret) = self.pick_l0_trivial_move_file(l0, level_handlers, stats) {
+            return Some(ret);
         }
 
         let vnode_partition_count = self.config.split_weight_by_vnode;
@@ -67,18 +57,13 @@ impl CompactionPicker for IntraCompactionPicker {
             return Some(ret);
         }
 
-        if let Some(ret) = self.pick_l0_intra(l0, &level_handlers[0], vnode_partition_count, stats)
-        {
-            return Some(ret);
-        }
-
-        self.pick_l0_trivial_move_file(l0, level_handlers, stats)
+        self.pick_l0_intra(l0, &level_handlers[0], vnode_partition_count, stats)
     }
 }
 
 impl IntraCompactionPicker {
     #[cfg(test)]
-    pub fn new(
+    pub fn for_test(
         config: Arc<CompactionConfig>,
         developer_config: Arc<CompactionDeveloperConfig>,
     ) -> IntraCompactionPicker {
@@ -94,6 +79,7 @@ impl IntraCompactionPicker {
         compaction_task_validator: Arc<CompactionTaskValidator>,
         developer_config: Arc<CompactionDeveloperConfig>,
     ) -> IntraCompactionPicker {
+        assert!(config.level0_sub_level_compact_level_count > 1);
         IntraCompactionPicker {
             config,
             compaction_task_validator,
@@ -158,6 +144,7 @@ impl IntraCompactionPicker {
                 self.config.level0_max_compact_file_number,
                 overlap_strategy.clone(),
                 self.developer_config.enable_check_task_level_overlap,
+                self.config.max_l0_compact_level_count as usize,
             );
 
             let l0_select_tables_vec = non_overlap_sub_level_picker
@@ -421,7 +408,7 @@ pub mod tests {
                 .level0_sub_level_compact_level_count(1)
                 .build(),
         );
-        IntraCompactionPicker::new(config, Arc::new(CompactionDeveloperConfig::default()))
+        IntraCompactionPicker::for_test(config, Arc::new(CompactionDeveloperConfig::default()))
     }
 
     #[test]
@@ -441,7 +428,6 @@ pub mod tests {
                 total_file_size: 0,
                 uncompressed_file_size: 0,
             }),
-            member_table_ids: vec![1],
             ..Default::default()
         };
         push_tables_level0_nonoverlapping(
@@ -486,7 +472,6 @@ pub mod tests {
                 generate_table(1, 1, 100, 210, 2),
                 generate_table(2, 1, 200, 250, 2),
             ])),
-            member_table_ids: vec![1],
             ..Default::default()
         };
         let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
@@ -524,7 +509,6 @@ pub mod tests {
             let levels = Levels {
                 l0: Some(l0),
                 levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
-                member_table_ids: vec![1],
                 ..Default::default()
             };
             let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
@@ -535,8 +519,10 @@ pub mod tests {
                     .level0_overlapping_sub_level_compact_level_count(4)
                     .build(),
             );
-            let mut picker =
-                IntraCompactionPicker::new(config, Arc::new(CompactionDeveloperConfig::default()));
+            let mut picker = IntraCompactionPicker::for_test(
+                config,
+                Arc::new(CompactionDeveloperConfig::default()),
+            );
             let mut local_stats = LocalPickerStatistic::default();
             let ret = picker
                 .pick_compaction(&levels, &levels_handler, &mut local_stats)
@@ -571,7 +557,6 @@ pub mod tests {
             let levels = Levels {
                 l0: Some(l0),
                 levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
-                member_table_ids: vec![1],
                 ..Default::default()
             };
             let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
@@ -581,8 +566,10 @@ pub mod tests {
                     .level0_sub_level_compact_level_count(1)
                     .build(),
             );
-            let mut picker =
-                IntraCompactionPicker::new(config, Arc::new(CompactionDeveloperConfig::default()));
+            let mut picker = IntraCompactionPicker::for_test(
+                config,
+                Arc::new(CompactionDeveloperConfig::default()),
+            );
             let mut local_stats = LocalPickerStatistic::default();
             let ret = picker
                 .pick_compaction(&levels, &levels_handler, &mut local_stats)
@@ -641,7 +628,6 @@ pub mod tests {
             let levels = Levels {
                 l0: Some(l0),
                 levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
-                member_table_ids: vec![1],
                 ..Default::default()
             };
             let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
@@ -651,8 +637,10 @@ pub mod tests {
                     .level0_sub_level_compact_level_count(1)
                     .build(),
             );
-            let mut picker =
-                IntraCompactionPicker::new(config, Arc::new(CompactionDeveloperConfig::default()));
+            let mut picker = IntraCompactionPicker::for_test(
+                config,
+                Arc::new(CompactionDeveloperConfig::default()),
+            );
             let mut local_stats = LocalPickerStatistic::default();
             let ret = picker
                 .pick_compaction(&levels, &levels_handler, &mut local_stats)
@@ -705,7 +693,7 @@ pub mod tests {
                 .build(),
         );
         let mut picker =
-            IntraCompactionPicker::new(config, Arc::new(CompactionDeveloperConfig::default()));
+            IntraCompactionPicker::for_test(config, Arc::new(CompactionDeveloperConfig::default()));
 
         // Cannot trivial move because there is only 1 sub-level.
         let l0 = generate_l0_overlapping_sublevels(vec![vec![
@@ -715,7 +703,6 @@ pub mod tests {
         let levels = Levels {
             l0: Some(l0),
             levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
-            member_table_ids: vec![1],
             ..Default::default()
         };
         levels_handler[1].add_pending_task(100, 1, levels.levels[0].get_table_infos());
@@ -736,7 +723,6 @@ pub mod tests {
         let mut levels = Levels {
             l0: Some(l0),
             levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
-            member_table_ids: vec![1],
             ..Default::default()
         };
         assert!(picker
@@ -792,4 +778,71 @@ pub mod tests {
             .unwrap();
         assert_eq!(ret.input_levels.len(), 2);
     }
+
+    #[test]
+    fn test_priority() {
+        let config = Arc::new(
+            CompactionConfigBuilder::new()
+                .level0_max_compact_file_number(20)
+                .sub_level_max_compaction_bytes(1)
+                .level0_sub_level_compact_level_count(2)
+                .build(),
+        );
+        let mut table_infos = vec![];
+        for epoch in 1..3 {
+            let base = epoch * 100;
+            let mut ssts = vec![];
+            for i in 1..50 {
+                let left = (i as usize) * 100;
+                let right = left + 100;
+                ssts.push(generate_table(base + i, 1, left, right, epoch));
+            }
+            table_infos.push(ssts);
+        }
+
+        let mut l0 = generate_l0_nonoverlapping_multi_sublevels(table_infos);
+        // trivial-move
+        l0.sub_levels[1]
+            .table_infos
+            .push(generate_table(9999, 900000000, 0, 100, 1));
+
+        l0.sub_levels[0].total_file_size = 1;
+        l0.sub_levels[1].total_file_size = 1;
+
+        let mut picker = IntraCompactionPicker::new_with_validator(
+            config,
+            Arc::new(CompactionTaskValidator::unused()),
+            Arc::new(CompactionDeveloperConfig::default()),
+        );
+        let mut levels_handler = vec![LevelHandler::new(0), LevelHandler::new(1)];
+        let mut local_stats = LocalPickerStatistic::default();
+
+        let levels = Levels {
+            l0: Some(l0),
+            levels: vec![generate_level(1, vec![generate_table(100, 1, 0, 1000, 1)])],
+            ..Default::default()
+        };
+
+        let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
+        assert!(is_l0_trivial_move(ret.as_ref().unwrap()));
+        ret.as_ref()
+            .unwrap()
+            .add_pending_task(1, &mut levels_handler);
+        let ret = picker.pick_compaction(&levels, &levels_handler, &mut local_stats);
+        assert!(ret.is_some());
+        let input = ret.as_ref().unwrap();
+        assert_eq!(input.input_levels.len(), 2);
+        assert_ne!(
+            levels.l0.as_ref().unwrap().get_sub_levels()[0]
+                .table_infos
+                .len(),
+            input.input_levels[0].table_infos.len()
+        );
+        assert_ne!(
+            levels.l0.as_ref().unwrap().get_sub_levels()[1]
+                .table_infos
+                .len(),
+            input.input_levels[1].table_infos.len()
+        );
+    }
 }
diff --git a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs
index f46a99bc80c0d..2d1b2c95b20aa 100644
--- a/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/manual_compaction_picker.rs
@@ -326,10 +326,10 @@ impl CompactionPicker for ManualCompactionPicker {
 
 #[cfg(test)]
 pub mod tests {
-    use std::collections::{HashMap, HashSet};
+    use std::collections::{BTreeSet, HashMap};
 
     use risingwave_pb::hummock::compact_task;
-    pub use risingwave_pb::hummock::{KeyRange, Level, LevelType};
+    pub use risingwave_pb::hummock::KeyRange;
 
     use super::*;
     use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
@@ -1198,6 +1198,7 @@ pub mod tests {
                     1,
                     &group_config,
                     &levels,
+                    &BTreeSet::new(),
                     &mut levels_handler,
                     &mut local_stats,
                     HashMap::default(),
@@ -1235,6 +1236,7 @@ pub mod tests {
                     2,
                     &group_config,
                     &levels,
+                    &BTreeSet::new(),
                     &mut levels_handler,
                     &mut local_stats,
                     HashMap::default(),
@@ -1308,6 +1310,7 @@ pub mod tests {
                     1,
                     &group_config,
                     &levels,
+                    &BTreeSet::new(),
                     &mut levels_handler,
                     &mut local_stats,
                     HashMap::default(),
@@ -1347,6 +1350,7 @@ pub mod tests {
                     1,
                     &group_config,
                     &levels,
+                    &BTreeSet::new(),
                     &mut levels_handler,
                     &mut local_stats,
                     HashMap::default(),
diff --git a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs
index 01c3c050e574c..57dd5469d42ae 100644
--- a/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/min_overlap_compaction_picker.rs
@@ -20,7 +20,7 @@ use risingwave_hummock_sdk::prost_key_range::KeyRangeExt;
 use risingwave_pb::hummock::hummock_version::Levels;
 use risingwave_pb::hummock::{InputLevel, Level, LevelType, SstableInfo};
 
-use super::{CompactionInput, CompactionPicker, LocalPickerStatistic, MAX_COMPACT_LEVEL_COUNT};
+use super::{CompactionInput, CompactionPicker, LocalPickerStatistic};
 use crate::hummock::compaction::overlap_strategy::OverlapStrategy;
 use crate::hummock::level_handler::LevelHandler;
 
@@ -182,29 +182,52 @@ pub struct SubLevelSstables {
 pub struct NonOverlapSubLevelPicker {
     min_compaction_bytes: u64,
     max_compaction_bytes: u64,
-    min_depth: usize,
+    min_expected_level_count: usize,
     max_file_count: u64,
     overlap_strategy: Arc<dyn OverlapStrategy>,
-
     enable_check_task_level_overlap: bool,
+    max_expected_level_count: usize,
 }
 
 impl NonOverlapSubLevelPicker {
     pub fn new(
         min_compaction_bytes: u64,
         max_compaction_bytes: u64,
-        min_depth: usize,
+        min_expected_level_count: usize,
         max_file_count: u64,
         overlap_strategy: Arc<dyn OverlapStrategy>,
         enable_check_task_level_overlap: bool,
+        max_expected_level_count: usize,
     ) -> Self {
         Self {
             min_compaction_bytes,
             max_compaction_bytes,
-            min_depth,
+            min_expected_level_count,
             max_file_count,
             overlap_strategy,
             enable_check_task_level_overlap,
+            max_expected_level_count,
+        }
+    }
+
+    #[cfg(test)]
+    pub fn for_test(
+        min_compaction_bytes: u64,
+        max_compaction_bytes: u64,
+        min_expected_level_count: usize,
+        max_file_count: u64,
+        overlap_strategy: Arc<dyn OverlapStrategy>,
+        enable_check_task_level_overlap: bool,
+        max_expected_level_count: usize,
+    ) -> Self {
+        Self {
+            min_compaction_bytes,
+            max_compaction_bytes,
+            min_expected_level_count,
+            max_file_count,
+            overlap_strategy,
+            enable_check_task_level_overlap,
+            max_expected_level_count,
         }
     }
 
@@ -220,11 +243,9 @@ impl NonOverlapSubLevelPicker {
         sst: &SstableInfo,
     ) -> SubLevelSstables {
         let mut ret = SubLevelSstables {
-            total_file_count: 1,
-            total_file_size: sst.file_size,
             sstable_infos: vec![vec![]; levels.len()],
+            ..Default::default()
         };
-        ret.sstable_infos[0].extend(vec![sst.clone()]);
 
         let mut pick_levels_range = Vec::default();
         let mut max_select_level_count = 0;
@@ -241,7 +262,7 @@ impl NonOverlapSubLevelPicker {
                 .iter()
                 .filter(|ssts| !ssts.is_empty())
                 .count()
-                > MAX_COMPACT_LEVEL_COUNT
+                > self.max_expected_level_count
             {
                 break;
             }
@@ -293,14 +314,6 @@ impl NonOverlapSubLevelPicker {
                     add_files_count += 1;
                 }
 
-                // When size / file count has exceeded the limit, we need to abandon this plan, it cannot be expanded to the last sub_level
-                if max_select_level_count > 1
-                    && (add_files_size >= self.max_compaction_bytes
-                        || add_files_count >= self.max_file_count as usize)
-                {
-                    break 'expand_new_level;
-                }
-
                 overlap_levels.push((reverse_index, overlap_files_range.clone()));
                 select_level_count += 1;
             }
@@ -309,32 +322,75 @@ impl NonOverlapSubLevelPicker {
                 max_select_level_count = select_level_count;
                 pick_levels_range = overlap_levels;
             }
-        }
 
-        for (reverse_index, sst_range) in pick_levels_range {
-            let level_ssts = &levels[reverse_index].table_infos;
-            ret.sstable_infos[reverse_index] = level_ssts[sst_range].to_vec();
-            ret.total_file_count += ret.sstable_infos[reverse_index].len();
-            ret.total_file_size += ret.sstable_infos[reverse_index]
-                .iter()
-                .map(|sst| sst.file_size)
-                .sum::<u64>();
+            // When size / file count has exceeded the limit, we need to abandon this plan, it cannot be expanded to the last sub_level
+            if max_select_level_count >= self.min_expected_level_count
+                && (add_files_size >= self.max_compaction_bytes
+                    || add_files_count >= self.max_file_count as usize)
+            {
+                break 'expand_new_level;
+            }
         }
 
-        // sort sst per level due to reverse expand
-        ret.sstable_infos.iter_mut().for_each(|level_ssts| {
-            level_ssts.sort_by(|sst1, sst2| {
-                let a = sst1.key_range.as_ref().unwrap();
-                let b = sst2.key_range.as_ref().unwrap();
-                a.compare(b)
+        if !pick_levels_range.is_empty() {
+            for (reverse_index, sst_range) in pick_levels_range {
+                let level_ssts = &levels[reverse_index].table_infos;
+                ret.sstable_infos[reverse_index] = level_ssts[sst_range].to_vec();
+                ret.total_file_count += ret.sstable_infos[reverse_index].len();
+                ret.total_file_size += ret.sstable_infos[reverse_index]
+                    .iter()
+                    .map(|sst| sst.file_size)
+                    .sum::<u64>();
+            }
+
+            // sort sst per level due to reverse expand
+            ret.sstable_infos.iter_mut().for_each(|level_ssts| {
+                level_ssts.sort_by(|sst1, sst2| {
+                    let a = sst1.key_range.as_ref().unwrap();
+                    let b = sst2.key_range.as_ref().unwrap();
+                    a.compare(b)
+                });
             });
-        });
+        } else {
+            ret.total_file_count = 1;
+            ret.total_file_size = sst.file_size;
+            ret.sstable_infos[0].extend(vec![sst.clone()]);
+        }
 
         if self.enable_check_task_level_overlap {
             self.verify_task_level_overlap(&ret, levels);
         }
 
         ret.sstable_infos.retain(|ssts| !ssts.is_empty());
+
+        // To check whether the task is expected
+        if ret.total_file_size > self.max_compaction_bytes
+            || ret.total_file_count as u64 > self.max_file_count
+            || ret.sstable_infos.len() > self.max_expected_level_count
+        {
+            // rotate the sstables to meet the `max_file_count` and `max_compaction_bytes` and `max_expected_level_count`
+            let mut total_file_count = 0;
+            let mut total_file_size = 0;
+            let mut total_level_count = 0;
+            for (index, sstables) in ret.sstable_infos.iter().enumerate() {
+                total_file_count += sstables.len();
+                total_file_size += sstables.iter().map(|sst| sst.file_size).sum::<u64>();
+                total_level_count += 1;
+
+                // Atleast `min_expected_level_count`` level should be selected
+                if total_level_count >= self.min_expected_level_count
+                    && (total_file_count as u64 >= self.max_file_count
+                        || total_file_size >= self.max_compaction_bytes
+                        || total_level_count >= self.max_expected_level_count)
+                {
+                    ret.total_file_count = total_file_count;
+                    ret.total_file_size = total_file_size;
+                    ret.sstable_infos.truncate(index + 1);
+                    break;
+                }
+            }
+        }
+
         ret
     }
 
@@ -343,7 +399,7 @@ impl NonOverlapSubLevelPicker {
         l0: &[Level],
         level_handler: &LevelHandler,
     ) -> Vec<SubLevelSstables> {
-        if l0.len() < self.min_depth {
+        if l0.len() < self.min_expected_level_count {
             return vec![];
         }
 
@@ -354,7 +410,7 @@ impl NonOverlapSubLevelPicker {
             }
 
             let ret = self.pick_sub_level(l0, level_handler, sst);
-            if ret.sstable_infos.len() < self.min_depth
+            if ret.sstable_infos.len() < self.min_expected_level_count
                 && ret.total_file_size < self.min_compaction_bytes
             {
                 continue;
@@ -370,7 +426,7 @@ impl NonOverlapSubLevelPicker {
         let mut unexpected = vec![];
 
         for selected_task in scores {
-            if selected_task.sstable_infos.len() > MAX_COMPACT_LEVEL_COUNT {
+            if selected_task.sstable_infos.len() > self.max_expected_level_count {
                 unexpected.push(selected_task);
             } else {
                 expected.push(selected_task);
@@ -388,10 +444,11 @@ impl NonOverlapSubLevelPicker {
                 .then_with(|| a.total_file_size.cmp(&b.total_file_size))
         });
 
+        // For unexpected task the number of levels is as small as possible
         unexpected.sort_by(|a, b| {
-            b.sstable_infos
+            a.sstable_infos
                 .len()
-                .cmp(&a.sstable_infos.len())
+                .cmp(&b.sstable_infos.len())
                 .then_with(|| a.total_file_count.cmp(&b.total_file_count))
                 .then_with(|| a.total_file_size.cmp(&b.total_file_size))
         });
@@ -477,7 +534,7 @@ impl NonOverlapSubLevelPicker {
 pub mod tests {
     use std::collections::BTreeSet;
 
-    pub use risingwave_pb::hummock::{Level, LevelType};
+    use risingwave_common::config::default::compaction_config;
 
     use super::*;
     use crate::hummock::compaction::overlap_strategy::RangeOverlapStrategy;
@@ -682,6 +739,7 @@ pub mod tests {
                 10000,
                 Arc::new(RangeOverlapStrategy::default()),
                 true,
+                compaction_config::max_l0_compact_level_count() as usize,
             );
             let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]);
             assert_eq!(6, ret.len());
@@ -696,6 +754,7 @@ pub mod tests {
                 10000,
                 Arc::new(RangeOverlapStrategy::default()),
                 true,
+                compaction_config::max_l0_compact_level_count() as usize,
             );
             let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]);
             assert_eq!(6, ret.len());
@@ -710,6 +769,7 @@ pub mod tests {
                 5,
                 Arc::new(RangeOverlapStrategy::default()),
                 true,
+                compaction_config::max_l0_compact_level_count() as usize,
             );
             let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]);
             assert_eq!(6, ret.len());
@@ -785,6 +845,7 @@ pub mod tests {
                 10000,
                 Arc::new(RangeOverlapStrategy::default()),
                 true,
+                compaction_config::max_l0_compact_level_count() as usize,
             );
             let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]);
             assert_eq!(6, ret.len());
@@ -800,6 +861,7 @@ pub mod tests {
                 10000,
                 Arc::new(RangeOverlapStrategy::default()),
                 true,
+                compaction_config::max_l0_compact_level_count() as usize,
             );
             let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]);
             assert_eq!(6, ret.len());
@@ -815,6 +877,7 @@ pub mod tests {
                 max_file_count,
                 Arc::new(RangeOverlapStrategy::default()),
                 true,
+                compaction_config::max_l0_compact_level_count() as usize,
             );
             let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]);
             assert_eq!(6, ret.len());
@@ -838,6 +901,7 @@ pub mod tests {
                 10000,
                 Arc::new(RangeOverlapStrategy::default()),
                 true,
+                compaction_config::max_l0_compact_level_count() as usize,
             );
             let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]);
             assert_eq!(3, ret.len());
@@ -908,4 +972,129 @@ pub mod tests {
         assert_eq!(select_files.len(), 1);
         assert_eq!(target_files.len(), 1);
     }
+
+    #[test]
+    fn test_pick_unexpected_task() {
+        let levels = vec![
+            Level {
+                level_idx: 1,
+                level_type: LevelType::Nonoverlapping as i32,
+                table_infos: vec![generate_table(0, 1, 50, 100, 2)], // 50
+                total_file_size: 50,
+                ..Default::default()
+            },
+            Level {
+                level_idx: 2,
+                level_type: LevelType::Nonoverlapping as i32,
+                table_infos: vec![
+                    generate_table(1, 1, 101, 150, 1), // 50
+                ],
+                total_file_size: 50,
+                ..Default::default()
+            },
+            Level {
+                level_idx: 3,
+                level_type: LevelType::Nonoverlapping as i32,
+                table_infos: vec![
+                    generate_table(2, 1, 151, 200, 2), // 50
+                ],
+                total_file_size: 50,
+                ..Default::default()
+            },
+            Level {
+                level_idx: 4,
+                level_type: LevelType::Nonoverlapping as i32,
+                table_infos: vec![
+                    generate_table(3, 1, 50, 300, 2), // 250
+                ],
+                total_file_size: 250,
+                ..Default::default()
+            },
+        ];
+
+        let levels_handlers = vec![
+            LevelHandler::new(0),
+            LevelHandler::new(1),
+            LevelHandler::new(2),
+            LevelHandler::new(3),
+            LevelHandler::new(4),
+        ];
+
+        {
+            // no limit
+            let picker = NonOverlapSubLevelPicker::new(
+                0,
+                10000,
+                1,
+                10000,
+                Arc::new(RangeOverlapStrategy::default()),
+                true,
+                compaction_config::max_l0_compact_level_count() as usize,
+            );
+            let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]);
+            {
+                let plan = &ret[0];
+                assert_eq!(4, plan.sstable_infos.len());
+            }
+        }
+
+        {
+            // limit size
+            let picker = NonOverlapSubLevelPicker::new(
+                0,
+                150,
+                1,
+                10000,
+                Arc::new(RangeOverlapStrategy::default()),
+                true,
+                compaction_config::max_l0_compact_level_count() as usize,
+            );
+            let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]);
+            {
+                let plan = &ret[0];
+                assert_eq!(3, plan.sstable_infos.len());
+            }
+        }
+
+        {
+            // limit count
+            let picker = NonOverlapSubLevelPicker::new(
+                0,
+                10000,
+                1,
+                3,
+                Arc::new(RangeOverlapStrategy::default()),
+                true,
+                compaction_config::max_l0_compact_level_count() as usize,
+            );
+            let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]);
+            {
+                let plan = &ret[0];
+                assert_eq!(3, plan.sstable_infos.len());
+            }
+        }
+
+        {
+            // limit expected level count
+            let max_expected_level_count = 3;
+            let picker = NonOverlapSubLevelPicker::for_test(
+                0,
+                10000,
+                1,
+                3,
+                Arc::new(RangeOverlapStrategy::default()),
+                true,
+                max_expected_level_count,
+            );
+            let ret = picker.pick_l0_multi_non_overlap_level(&levels, &levels_handlers[0]);
+            {
+                let plan = &ret[0];
+                assert_eq!(max_expected_level_count, plan.sstable_infos.len());
+
+                assert_eq!(0, plan.sstable_infos[0][0].sst_id);
+                assert_eq!(1, plan.sstable_infos[1][0].sst_id);
+                assert_eq!(2, plan.sstable_infos[2][0].sst_id);
+            }
+        }
+    }
 }
diff --git a/src/meta/src/hummock/compaction/picker/mod.rs b/src/meta/src/hummock/compaction/picker/mod.rs
index 7e33123272684..6d464b9a33bcd 100644
--- a/src/meta/src/hummock/compaction/picker/mod.rs
+++ b/src/meta/src/hummock/compaction/picker/mod.rs
@@ -43,8 +43,6 @@ pub use ttl_reclaim_compaction_picker::{TtlPickerState, TtlReclaimCompactionPick
 
 use crate::hummock::level_handler::LevelHandler;
 
-pub const MAX_COMPACT_LEVEL_COUNT: usize = 42;
-
 #[derive(Default, Debug)]
 pub struct LocalPickerStatistic {
     pub skip_by_write_amp_limit: u64,
@@ -97,18 +95,3 @@ pub trait CompactionPicker {
         stats: &mut LocalPickerStatistic,
     ) -> Option<CompactionInput>;
 }
-
-#[derive(Default, Clone, Debug)]
-pub struct PartitionLevelInfo {
-    pub level_id: u32,
-    pub sub_level_id: u64,
-    pub left_idx: usize,
-    pub right_idx: usize,
-    pub total_file_size: u64,
-}
-
-#[derive(Default, Clone, Debug)]
-pub struct LevelPartition {
-    pub sub_levels: Vec<PartitionLevelInfo>,
-    pub total_file_size: u64,
-}
diff --git a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs
index 5d05fedbe5338..b94f7587dd04e 100644
--- a/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/space_reclaim_compaction_picker.rs
@@ -24,7 +24,7 @@ use crate::hummock::level_handler::LevelHandler;
 // key_range and selects the appropriate files to generate compaction
 pub struct SpaceReclaimCompactionPicker {
     // config
-    pub max_space_reclaim_bytes: u64,
+    pub _max_space_reclaim_bytes: u64,
 
     // for filter
     pub all_table_ids: HashSet<u32>,
@@ -40,7 +40,7 @@ pub struct SpaceReclaimPickerState {
 impl SpaceReclaimCompactionPicker {
     pub fn new(max_space_reclaim_bytes: u64, all_table_ids: HashSet<u32>) -> Self {
         Self {
-            max_space_reclaim_bytes,
+            _max_space_reclaim_bytes: max_space_reclaim_bytes,
             all_table_ids,
         }
     }
@@ -168,10 +168,11 @@ impl SpaceReclaimCompactionPicker {
 #[cfg(test)]
 mod test {
 
-    use std::collections::HashMap;
+    use std::collections::{BTreeSet, HashMap};
     use std::sync::Arc;
 
     use itertools::Itertools;
+    use risingwave_common::catalog::TableId;
     use risingwave_pb::hummock::compact_task;
     pub use risingwave_pb::hummock::{Level, LevelType};
 
@@ -234,11 +235,12 @@ mod test {
         }
 
         assert_eq!(levels.len(), 4);
-        let mut levels = Levels {
+        let levels = Levels {
             levels,
             l0: Some(l0),
             ..Default::default()
         };
+        let mut member_table_ids = BTreeSet::new();
         let mut levels_handler = (0..5).map(LevelHandler::new).collect_vec();
         let mut local_stats = LocalSelectorStatistic::default();
 
@@ -252,6 +254,7 @@ mod test {
                     1,
                     &group_config,
                     &levels,
+                    &member_table_ids,
                     &mut levels_handler,
                     &mut local_stats,
                     HashMap::default(),
@@ -268,6 +271,7 @@ mod test {
                     1,
                     &group_config,
                     &levels,
+                    &member_table_ids,
                     &mut levels_handler,
                     &mut local_stats,
                     HashMap::default(),
@@ -308,6 +312,7 @@ mod test {
                     1,
                     &group_config,
                     &levels,
+                    &member_table_ids,
                     &mut levels_handler,
                     &mut local_stats,
                     HashMap::default(),
@@ -334,6 +339,7 @@ mod test {
                     1,
                     &group_config,
                     &levels,
+                    &member_table_ids,
                     &mut levels_handler,
                     &mut local_stats,
                     HashMap::default(),
@@ -351,12 +357,17 @@ mod test {
                 }
             }
 
-            levels.member_table_ids = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
+            member_table_ids = BTreeSet::from_iter(
+                [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+                    .into_iter()
+                    .map(TableId::new),
+            );
             // pick space reclaim
             let task = selector.pick_compaction(
                 1,
                 &group_config,
                 &levels,
+                &member_table_ids,
                 &mut levels_handler,
                 &mut local_stats,
                 HashMap::default(),
@@ -372,13 +383,15 @@ mod test {
                 }
             }
 
-            levels.member_table_ids = vec![2, 3, 4, 5, 6, 7, 8, 9];
+            member_table_ids =
+                BTreeSet::from_iter([2, 3, 4, 5, 6, 7, 8, 9].into_iter().map(TableId::new));
             // pick space reclaim
             let task = selector
                 .pick_compaction(
                     1,
                     &group_config,
                     &levels,
+                    &member_table_ids,
                     &mut levels_handler,
                     &mut local_stats,
                     HashMap::default(),
@@ -407,7 +420,7 @@ mod test {
             // rebuild selector
             selector = SpaceReclaimCompactionSelector::default();
             // cut range [3,4] [6] [8,9,10]
-            levels.member_table_ids = vec![0, 1, 2, 5, 7];
+            member_table_ids = BTreeSet::from_iter([0, 1, 2, 5, 7].into_iter().map(TableId::new));
             let expect_task_file_count = [2, 1, 4];
             let expect_task_sst_id_range = [vec![3, 4], vec![6], vec![8, 9, 10, 11]];
             for (index, x) in expect_task_file_count.iter().enumerate() {
@@ -417,6 +430,7 @@ mod test {
                         1,
                         &group_config,
                         &levels,
+                        &member_table_ids,
                         &mut levels_handler,
                         &mut local_stats,
                         HashMap::default(),
@@ -458,12 +472,13 @@ mod test {
             // rebuild selector
             selector = SpaceReclaimCompactionSelector::default();
             // cut range [3,4] [6] [8,9,10]
-            levels.member_table_ids = vec![0, 1, 2, 5, 7];
+
+            member_table_ids = BTreeSet::from_iter([0, 1, 2, 5, 7].into_iter().map(TableId::new));
             let expect_task_file_count = [2, 1, 5];
             let expect_task_sst_id_range = [vec![3, 4], vec![6], vec![7, 8, 9, 10, 11]];
             for (index, x) in expect_task_file_count.iter().enumerate() {
                 if index == expect_task_file_count.len() - 1 {
-                    levels.member_table_ids = vec![2, 5];
+                    member_table_ids = BTreeSet::from_iter([2, 5].into_iter().map(TableId::new));
                 }
 
                 // // pick space reclaim
@@ -472,6 +487,7 @@ mod test {
                         1,
                         &group_config,
                         &levels,
+                        &member_table_ids,
                         &mut levels_handler,
                         &mut local_stats,
                         HashMap::default(),
diff --git a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs
index 9ed22ba551fcc..ea25211b44749 100644
--- a/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/tier_compaction_picker.rs
@@ -21,7 +21,6 @@ use super::{
     CompactionInput, CompactionPicker, CompactionTaskValidator, LocalPickerStatistic,
     ValidationRuleType,
 };
-use crate::hummock::compaction::picker::MAX_COMPACT_LEVEL_COUNT;
 use crate::hummock::level_handler::LevelHandler;
 
 pub struct TierCompactionPicker {
@@ -87,10 +86,7 @@ impl TierCompactionPicker {
             let mut compaction_bytes = level.total_file_size;
             let mut compact_file_count = level.table_infos.len() as u64;
             // Limit sstable file count to avoid using too much memory.
-            let overlapping_max_compact_file_numer = std::cmp::min(
-                self.config.level0_max_compact_file_number,
-                MAX_COMPACT_LEVEL_COUNT as u64,
-            );
+            let overlapping_max_compact_file_numer = self.config.level0_max_compact_file_number;
 
             for other in &l0.sub_levels[idx + 1..] {
                 if compaction_bytes > max_compaction_bytes {
@@ -295,7 +291,6 @@ pub mod tests {
                 sub_levels: vec![l1, l2],
             }),
             levels: vec![],
-            member_table_ids: vec![1],
             ..Default::default()
         };
         let config = Arc::new(
diff --git a/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs
index f07a8872c9b03..c7be0347fce61 100644
--- a/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/tombstone_reclaim_compaction_picker.rs
@@ -154,7 +154,6 @@ pub mod tests {
                     ],
                 ),
             ],
-            member_table_ids: vec![1],
             ..Default::default()
         };
         let levels_handler = vec![
diff --git a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs
index f690b0f80112c..bc1fc2ce304be 100644
--- a/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs
+++ b/src/meta/src/hummock/compaction/picker/ttl_reclaim_compaction_picker.rs
@@ -201,6 +201,7 @@ impl TtlReclaimCompactionPicker {
 
 #[cfg(test)]
 mod test {
+    use std::collections::BTreeSet;
     use std::sync::Arc;
 
     use itertools::Itertools;
@@ -377,6 +378,7 @@ mod test {
                     1,
                     &group_config,
                     &levels,
+                    &BTreeSet::new(),
                     &mut levels_handler,
                     &mut local_stats,
                     table_id_to_options,
@@ -427,6 +429,7 @@ mod test {
                     1,
                     &group_config,
                     &levels,
+                    &BTreeSet::new(),
                     &mut levels_handler,
                     &mut local_stats,
                     table_id_to_options.clone(),
@@ -460,6 +463,7 @@ mod test {
                     1,
                     &group_config,
                     &levels,
+                    &BTreeSet::new(),
                     &mut levels_handler,
                     &mut local_stats,
                     table_id_to_options.clone(),
@@ -516,6 +520,7 @@ mod test {
                     1,
                     &group_config,
                     &levels,
+                    &BTreeSet::new(),
                     &mut levels_handler,
                     &mut local_stats,
                     table_id_to_options,
@@ -557,6 +562,7 @@ mod test {
                 1,
                 &group_config,
                 &levels,
+                &BTreeSet::new(),
                 &mut levels_handler,
                 &mut local_stats,
                 HashMap::default(),
@@ -619,6 +625,7 @@ mod test {
                         1,
                         &group_config,
                         &levels,
+                        &BTreeSet::new(),
                         &mut levels_handler,
                         &mut local_stats,
                         table_id_to_options.clone(),
@@ -711,6 +718,7 @@ mod test {
                         1,
                         &group_config,
                         &levels,
+                        &BTreeSet::new(),
                         &mut levels_handler,
                         &mut local_stats,
                         table_id_to_options.clone(),
diff --git a/src/meta/src/hummock/compaction/selector/emergency_selector.rs b/src/meta/src/hummock/compaction/selector/emergency_selector.rs
index c386aee5c8644..685ab1487d51c 100644
--- a/src/meta/src/hummock/compaction/selector/emergency_selector.rs
+++ b/src/meta/src/hummock/compaction/selector/emergency_selector.rs
@@ -37,6 +37,7 @@ impl CompactionSelector for EmergencySelector {
         task_id: HummockCompactionTaskId,
         group: &CompactionGroup,
         levels: &Levels,
+        _member_table_ids: &std::collections::BTreeSet<risingwave_common::catalog::TableId>,
         level_handlers: &mut [LevelHandler],
         selector_stats: &mut LocalSelectorStatistic,
         _table_id_to_options: HashMap<u32, TableOption>,
diff --git a/src/meta/src/hummock/compaction/selector/level_selector.rs b/src/meta/src/hummock/compaction/selector/level_selector.rs
index 5c118269cfee3..38d1e35e22502 100644
--- a/src/meta/src/hummock/compaction/selector/level_selector.rs
+++ b/src/meta/src/hummock/compaction/selector/level_selector.rs
@@ -314,14 +314,9 @@ impl DynamicLevelSelectorCore {
             if level_idx < ctx.base_level || level_idx >= self.config.max_level as usize {
                 continue;
             }
-            let upper_level = if level_idx == ctx.base_level {
-                0
-            } else {
-                level_idx - 1
-            };
-            let total_size = level.total_file_size
-                + handlers[upper_level].get_pending_output_file_size(level.level_idx)
-                - handlers[level_idx].get_pending_output_file_size(level.level_idx + 1);
+            let output_file_size =
+                handlers[level_idx].get_pending_output_file_size(level.level_idx + 1);
+            let total_size = level.total_file_size.saturating_sub(output_file_size);
             if total_size == 0 {
                 continue;
             }
@@ -427,6 +422,7 @@ impl CompactionSelector for DynamicLevelSelector {
         task_id: HummockCompactionTaskId,
         compaction_group: &CompactionGroup,
         levels: &Levels,
+        _member_table_ids: &std::collections::BTreeSet<risingwave_common::catalog::TableId>,
         level_handlers: &mut [LevelHandler],
         selector_stats: &mut LocalSelectorStatistic,
         _table_id_to_options: HashMap<u32, TableOption>,
@@ -483,7 +479,7 @@ impl CompactionSelector for DynamicLevelSelector {
 
 #[cfg(test)]
 pub mod tests {
-    use std::collections::HashMap;
+    use std::collections::{BTreeSet, HashMap};
     use std::sync::Arc;
 
     use itertools::Itertools;
@@ -588,7 +584,7 @@ pub mod tests {
             .max_compaction_bytes(10000)
             .level0_tier_compact_file_number(4)
             .compaction_mode(CompactionMode::Range as i32)
-            .level0_sub_level_compact_level_count(1)
+            .level0_sub_level_compact_level_count(3)
             .build();
         let group_config = CompactionGroup::new(1, config.clone());
         let levels = vec![
@@ -605,7 +601,6 @@ pub mod tests {
                 3,
                 10,
             ))),
-            member_table_ids: vec![1],
             ..Default::default()
         };
 
@@ -617,6 +612,7 @@ pub mod tests {
                 1,
                 &group_config,
                 &levels,
+                &BTreeSet::new(),
                 &mut levels_handlers,
                 &mut local_stats,
                 HashMap::default(),
@@ -644,6 +640,7 @@ pub mod tests {
                 1,
                 &group_config,
                 &levels,
+                &BTreeSet::new(),
                 &mut levels_handlers,
                 &mut local_stats,
                 HashMap::default(),
@@ -663,6 +660,7 @@ pub mod tests {
                 2,
                 &group_config,
                 &levels,
+                &BTreeSet::new(),
                 &mut levels_handlers,
                 &mut local_stats,
                 HashMap::default(),
@@ -699,6 +697,7 @@ pub mod tests {
             2,
             &group_config,
             &levels,
+            &BTreeSet::new(),
             &mut levels_handlers,
             &mut local_stats,
             HashMap::default(),
diff --git a/src/meta/src/hummock/compaction/selector/manual_selector.rs b/src/meta/src/hummock/compaction/selector/manual_selector.rs
index 427efadf3914d..62c94c8f888df 100644
--- a/src/meta/src/hummock/compaction/selector/manual_selector.rs
+++ b/src/meta/src/hummock/compaction/selector/manual_selector.rs
@@ -79,6 +79,7 @@ impl CompactionSelector for ManualCompactionSelector {
         task_id: HummockCompactionTaskId,
         group: &CompactionGroup,
         levels: &Levels,
+        _member_table_ids: &std::collections::BTreeSet<risingwave_common::catalog::TableId>,
         level_handlers: &mut [LevelHandler],
         _selector_stats: &mut LocalSelectorStatistic,
         _table_id_to_options: HashMap<u32, TableOption>,
diff --git a/src/meta/src/hummock/compaction/selector/mod.rs b/src/meta/src/hummock/compaction/selector/mod.rs
index a342a661ecb7b..aca2457da62dc 100644
--- a/src/meta/src/hummock/compaction/selector/mod.rs
+++ b/src/meta/src/hummock/compaction/selector/mod.rs
@@ -24,13 +24,13 @@ mod space_reclaim_selector;
 mod tombstone_compaction_selector;
 mod ttl_selector;
 
-use std::collections::HashMap;
+use std::collections::{BTreeSet, HashMap};
 use std::sync::Arc;
 
 pub use emergency_selector::EmergencySelector;
 pub use level_selector::{DynamicLevelSelector, DynamicLevelSelectorCore};
 pub use manual_selector::{ManualCompactionOption, ManualCompactionSelector};
-use risingwave_common::catalog::TableOption;
+use risingwave_common::catalog::{TableId, TableOption};
 use risingwave_hummock_sdk::HummockCompactionTaskId;
 use risingwave_pb::hummock::compact_task;
 use risingwave_pb::hummock::hummock_version::Levels;
@@ -53,6 +53,7 @@ pub trait CompactionSelector: Sync + Send {
         task_id: HummockCompactionTaskId,
         group: &CompactionGroup,
         levels: &Levels,
+        member_table_ids: &BTreeSet<TableId>,
         level_handlers: &mut [LevelHandler],
         selector_stats: &mut LocalSelectorStatistic,
         table_id_to_options: HashMap<u32, TableOption>,
diff --git a/src/meta/src/hummock/compaction/selector/space_reclaim_selector.rs b/src/meta/src/hummock/compaction/selector/space_reclaim_selector.rs
index c48cb0fe605c5..b284a6a538b3e 100644
--- a/src/meta/src/hummock/compaction/selector/space_reclaim_selector.rs
+++ b/src/meta/src/hummock/compaction/selector/space_reclaim_selector.rs
@@ -17,10 +17,10 @@
 // COPYING file in the root directory) and Apache 2.0 License
 // (found in the LICENSE.Apache file in the root directory).
 
-use std::collections::HashMap;
+use std::collections::{BTreeSet, HashMap};
 use std::sync::Arc;
 
-use risingwave_common::catalog::TableOption;
+use risingwave_common::catalog::{TableId, TableOption};
 use risingwave_hummock_sdk::HummockCompactionTaskId;
 use risingwave_pb::hummock::compact_task;
 use risingwave_pb::hummock::hummock_version::Levels;
@@ -44,6 +44,7 @@ impl CompactionSelector for SpaceReclaimCompactionSelector {
         task_id: HummockCompactionTaskId,
         group: &CompactionGroup,
         levels: &Levels,
+        member_table_ids: &BTreeSet<TableId>,
         level_handlers: &mut [LevelHandler],
         _selector_stats: &mut LocalSelectorStatistic,
         _table_id_to_options: HashMap<u32, TableOption>,
@@ -53,7 +54,10 @@ impl CompactionSelector for SpaceReclaimCompactionSelector {
             DynamicLevelSelectorCore::new(group.compaction_config.clone(), developer_config);
         let mut picker = SpaceReclaimCompactionPicker::new(
             group.compaction_config.max_space_reclaim_bytes,
-            levels.member_table_ids.iter().cloned().collect(),
+            member_table_ids
+                .iter()
+                .map(|table_id| table_id.table_id)
+                .collect(),
         );
         let ctx = dynamic_level_core.calculate_level_base_size(levels);
         let state = self.state.entry(group.group_id).or_default();
diff --git a/src/meta/src/hummock/compaction/selector/tombstone_compaction_selector.rs b/src/meta/src/hummock/compaction/selector/tombstone_compaction_selector.rs
index 5cbae609caa86..b05802513733d 100644
--- a/src/meta/src/hummock/compaction/selector/tombstone_compaction_selector.rs
+++ b/src/meta/src/hummock/compaction/selector/tombstone_compaction_selector.rs
@@ -42,6 +42,7 @@ impl CompactionSelector for TombstoneCompactionSelector {
         task_id: HummockCompactionTaskId,
         group: &CompactionGroup,
         levels: &Levels,
+        _member_table_ids: &std::collections::BTreeSet<risingwave_common::catalog::TableId>,
         level_handlers: &mut [LevelHandler],
         _selector_stats: &mut LocalSelectorStatistic,
         _table_id_to_options: HashMap<u32, TableOption>,
diff --git a/src/meta/src/hummock/compaction/selector/ttl_selector.rs b/src/meta/src/hummock/compaction/selector/ttl_selector.rs
index ed099ede4158f..0e9497b06b17d 100644
--- a/src/meta/src/hummock/compaction/selector/ttl_selector.rs
+++ b/src/meta/src/hummock/compaction/selector/ttl_selector.rs
@@ -44,6 +44,7 @@ impl CompactionSelector for TtlCompactionSelector {
         task_id: HummockCompactionTaskId,
         group: &CompactionGroup,
         levels: &Levels,
+        _member_table_ids: &std::collections::BTreeSet<risingwave_common::catalog::TableId>,
         level_handlers: &mut [LevelHandler],
         _selector_stats: &mut LocalSelectorStatistic,
         table_id_to_options: HashMap<u32, TableOption>,
diff --git a/src/meta/src/hummock/compactor_manager.rs b/src/meta/src/hummock/compactor_manager.rs
index fcbac63817fff..ab0c868f703c4 100644
--- a/src/meta/src/hummock/compactor_manager.rs
+++ b/src/meta/src/hummock/compactor_manager.rs
@@ -101,6 +101,13 @@ impl Compactor {
         Ok(())
     }
 
+    pub fn cancel_tasks(&self, task_ids: &Vec<u64>) -> MetaResult<()> {
+        for task_id in task_ids {
+            self.cancel_task(*task_id)?;
+        }
+        Ok(())
+    }
+
     pub fn context_id(&self) -> HummockContextId {
         self.context_id
     }
@@ -111,12 +118,12 @@ impl Compactor {
 /// `CompactTaskAssignment`.
 ///
 /// A compact task can be in one of these states:
-/// - 1. Success: an assigned task is reported as success via `CompactStatus::report_compact_task`.
-///   It's the final state.
-/// - 2. Failed: an Failed task is reported as success via `CompactStatus::report_compact_task`.
-///   It's the final state.
-/// - 3. Cancelled: a task is reported as cancelled via `CompactStatus::report_compact_task`. It's
-///   the final state.
+/// 1. Success: an assigned task is reported as success via `CompactStatus::report_compact_task`.
+///    It's the final state.
+/// 2. Failed: an Failed task is reported as success via `CompactStatus::report_compact_task`.
+///    It's the final state.
+/// 3. Cancelled: a task is reported as cancelled via `CompactStatus::report_compact_task`. It's
+///    the final state.
 pub struct CompactorManagerInner {
     pub task_expired_seconds: u64,
     pub heartbeat_expired_seconds: u64,
@@ -131,7 +138,7 @@ impl CompactorManagerInner {
         use risingwave_meta_model_v2::compaction_task;
         use sea_orm::EntityTrait;
         // Retrieve the existing task assignments from metastore.
-        let task_assignment: Vec<CompactTaskAssignment> = match env.meta_store() {
+        let task_assignment: Vec<CompactTaskAssignment> = match env.meta_store_ref() {
             MetaStoreImpl::Kv(meta_store) => CompactTaskAssignment::list(meta_store).await?,
             MetaStoreImpl::Sql(sql_meta_store) => compaction_task::Entity::find()
                 .all(&sql_meta_store.conn)
diff --git a/src/meta/src/hummock/manager/checkpoint.rs b/src/meta/src/hummock/manager/checkpoint.rs
index 67ab7ebcf047d..70bbef6bd3db2 100644
--- a/src/meta/src/hummock/manager/checkpoint.rs
+++ b/src/meta/src/hummock/manager/checkpoint.rs
@@ -17,7 +17,6 @@ use std::ops::Bound::{Excluded, Included};
 use std::ops::{Deref, DerefMut};
 use std::sync::atomic::Ordering;
 
-use function_name::named;
 use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
     object_size_map, summarize_group_deltas,
 };
@@ -28,11 +27,11 @@ use risingwave_pb::hummock::hummock_version_checkpoint::{
 };
 use risingwave_pb::hummock::{PbHummockVersionArchive, PbHummockVersionCheckpoint};
 use thiserror_ext::AsReport;
+use tracing::warn;
 
 use crate::hummock::error::Result;
 use crate::hummock::manager::versioning::Versioning;
-use crate::hummock::manager::{read_lock, write_lock};
-use crate::hummock::metrics_utils::trigger_gc_stat;
+use crate::hummock::metrics_utils::{trigger_gc_stat, trigger_split_stat};
 use crate::hummock::HummockManager;
 
 #[derive(Default)]
@@ -121,11 +120,10 @@ impl HummockManager {
     /// Returns the diff between new and old checkpoint id.
     /// Note that this method must not be called concurrently, because internally it doesn't hold
     /// lock throughout the method.
-    #[named]
     pub async fn create_version_checkpoint(&self, min_delta_log_num: u64) -> Result<u64> {
         let timer = self.metrics.version_checkpoint_latency.start_timer();
         // 1. hold read lock and create new checkpoint
-        let versioning_guard = read_lock!(self, versioning).await;
+        let versioning_guard = self.versioning.read().await;
         let versioning: &Versioning = versioning_guard.deref();
         let current_version: &HummockVersion = &versioning.current_version;
         let old_checkpoint: &HummockVersionCheckpoint = &versioning.checkpoint;
@@ -136,9 +134,10 @@ impl HummockManager {
         }
         if cfg!(test) && new_checkpoint_id == old_checkpoint_id {
             drop(versioning_guard);
-            let mut versioning = write_lock!(self, versioning).await;
-            versioning.mark_objects_for_deletion();
-            let min_pinned_version_id = versioning.min_pinned_version_id();
+            let versioning = self.versioning.read().await;
+            let context_info = self.context_info.read().await;
+            versioning.mark_objects_for_deletion(&context_info, &self.delete_object_tracker);
+            let min_pinned_version_id = context_info.min_pinned_version_id();
             trigger_gc_stat(&self.metrics, &versioning.checkpoint, min_pinned_version_id);
             return Ok(0);
         }
@@ -159,19 +158,35 @@ impl HummockManager {
                     summary
                         .insert_table_infos
                         .iter()
-                        .map(|t| (t.object_id, t.file_size)),
+                        .map(|t| (t.object_id, t.file_size))
+                        .chain(
+                            version_delta
+                                .change_log_delta
+                                .values()
+                                .flat_map(|change_log| {
+                                    let new_log = change_log.new_log.as_ref().unwrap();
+                                    new_log
+                                        .new_value
+                                        .iter()
+                                        .chain(new_log.old_value.iter())
+                                        .map(|t| (t.object_id, t.file_size))
+                                }),
+                        ),
                 );
             }
-
             versions_object_ids.extend(version_delta.newly_added_object_ids());
         }
 
         // Object ids that once exist in any hummock version but not exist in the latest hummock version
         let removed_object_ids = &versions_object_ids - &current_version.get_object_ids();
-
         let total_file_size = removed_object_ids
             .iter()
-            .map(|t| object_sizes.get(t).copied().unwrap())
+            .map(|t| {
+                object_sizes.get(t).copied().unwrap_or_else(|| {
+                    warn!(object_id = t, "unable to get size of removed object id");
+                    0
+                })
+            })
             .sum::<u64>();
         stale_objects.insert(
             current_version.id,
@@ -207,17 +222,19 @@ impl HummockManager {
             }
         }
         // 3. hold write lock and update in memory state
-        let mut versioning_guard = write_lock!(self, versioning).await;
+        let mut versioning_guard = self.versioning.write().await;
         let versioning = versioning_guard.deref_mut();
+        let context_info = self.context_info.read().await;
         assert!(new_checkpoint.version.id > versioning.checkpoint.version.id);
         versioning.checkpoint = new_checkpoint;
         // Not delete stale objects when archive is enabled
         if !self.env.opts.enable_hummock_data_archive {
-            versioning.mark_objects_for_deletion();
+            versioning.mark_objects_for_deletion(&context_info, &self.delete_object_tracker);
         }
 
-        let min_pinned_version_id = versioning.min_pinned_version_id();
+        let min_pinned_version_id = context_info.min_pinned_version_id();
         trigger_gc_stat(&self.metrics, &versioning.checkpoint, min_pinned_version_id);
+        trigger_split_stat(&self.metrics, &versioning.current_version);
         drop(versioning_guard);
         timer.observe_duration();
         self.metrics
@@ -242,9 +259,8 @@ impl HummockManager {
         self.pause_version_checkpoint.load(Ordering::Relaxed)
     }
 
-    #[named]
     pub async fn get_checkpoint_version(&self) -> HummockVersion {
-        let versioning_guard = read_lock!(self, versioning).await;
+        let versioning_guard = self.versioning.read().await;
         versioning_guard.checkpoint.version.clone()
     }
 }
diff --git a/src/meta/src/hummock/manager/commit_epoch.rs b/src/meta/src/hummock/manager/commit_epoch.rs
new file mode 100644
index 0000000000000..44117139dc192
--- /dev/null
+++ b/src/meta/src/hummock/manager/commit_epoch.rs
@@ -0,0 +1,451 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{BTreeMap, HashMap};
+
+use itertools::Itertools;
+use risingwave_common::catalog::TableId;
+use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
+use risingwave_hummock_sdk::table_stats::{
+    add_prost_table_stats_map, purge_prost_table_stats, PbTableStatsMap,
+};
+use risingwave_hummock_sdk::table_watermark::TableWatermarks;
+use risingwave_hummock_sdk::{
+    CompactionGroupId, ExtendedSstableInfo, HummockContextId, HummockEpoch, HummockSstableObjectId,
+};
+use risingwave_pb::hummock::compact_task::{self};
+use risingwave_pb::hummock::group_delta::DeltaType;
+use risingwave_pb::hummock::hummock_version_delta::ChangeLogDelta;
+use risingwave_pb::hummock::{GroupDelta, HummockSnapshot, IntraLevelDelta, StateTableInfoDelta};
+
+use crate::hummock::error::{Error, Result};
+use crate::hummock::manager::transaction::{
+    HummockVersionStatsTransaction, HummockVersionTransaction,
+};
+use crate::hummock::manager::versioning::Versioning;
+use crate::hummock::manager::HISTORY_TABLE_INFO_STATISTIC_TIME;
+use crate::hummock::metrics_utils::{
+    get_or_create_local_table_stat, trigger_local_table_stat, trigger_sst_stat,
+};
+use crate::hummock::sequence::next_sstable_object_id;
+use crate::hummock::{commit_multi_var, start_measure_real_process_timer, HummockManager};
+
+#[derive(Debug, Clone)]
+pub struct NewTableFragmentInfo {
+    pub table_id: TableId,
+    pub mv_table_id: Option<TableId>,
+    pub internal_table_ids: Vec<TableId>,
+}
+
+pub struct CommitEpochInfo {
+    pub sstables: Vec<ExtendedSstableInfo>,
+    pub new_table_watermarks: HashMap<TableId, TableWatermarks>,
+    pub sst_to_context: HashMap<HummockSstableObjectId, HummockContextId>,
+    pub new_table_fragment_info: Option<NewTableFragmentInfo>,
+    pub change_log_delta: HashMap<TableId, ChangeLogDelta>,
+}
+
+impl CommitEpochInfo {
+    pub fn new(
+        sstables: Vec<ExtendedSstableInfo>,
+        new_table_watermarks: HashMap<TableId, TableWatermarks>,
+        sst_to_context: HashMap<HummockSstableObjectId, HummockContextId>,
+        new_table_fragment_info: Option<NewTableFragmentInfo>,
+        change_log_delta: HashMap<TableId, ChangeLogDelta>,
+    ) -> Self {
+        Self {
+            sstables,
+            new_table_watermarks,
+            sst_to_context,
+            new_table_fragment_info,
+            change_log_delta,
+        }
+    }
+
+    #[cfg(any(test, feature = "test"))]
+    pub(crate) fn for_test(
+        sstables: Vec<impl Into<ExtendedSstableInfo>>,
+        sst_to_context: HashMap<HummockSstableObjectId, HummockContextId>,
+    ) -> Self {
+        Self::new(
+            sstables.into_iter().map(Into::into).collect(),
+            HashMap::new(),
+            sst_to_context,
+            None,
+            HashMap::new(),
+        )
+    }
+}
+
+impl HummockManager {
+    /// Caller should ensure `epoch` > `max_committed_epoch`
+    pub async fn commit_epoch(
+        &self,
+        epoch: HummockEpoch,
+        commit_info: CommitEpochInfo,
+    ) -> Result<Option<HummockSnapshot>> {
+        let CommitEpochInfo {
+            mut sstables,
+            new_table_watermarks,
+            sst_to_context,
+            new_table_fragment_info,
+            change_log_delta,
+        } = commit_info;
+        let mut versioning_guard = self.versioning.write().await;
+        let _timer = start_measure_real_process_timer!(self, "commit_epoch");
+        // Prevent commit new epochs if this flag is set
+        if versioning_guard.disable_commit_epochs {
+            return Ok(None);
+        }
+
+        let versioning: &mut Versioning = &mut versioning_guard;
+        self.commit_epoch_sanity_check(
+            epoch,
+            &sstables,
+            &sst_to_context,
+            &versioning.current_version,
+        )
+        .await?;
+
+        // Consume and aggregate table stats.
+        let mut table_stats_change = PbTableStatsMap::default();
+        for s in &mut sstables {
+            add_prost_table_stats_map(&mut table_stats_change, &std::mem::take(&mut s.table_stats));
+        }
+
+        let mut version = HummockVersionTransaction::new(
+            &mut versioning.current_version,
+            &mut versioning.hummock_version_deltas,
+            self.env.notification_manager(),
+            &self.metrics,
+        );
+        let mut new_version_delta = version.new_delta();
+
+        new_version_delta.max_committed_epoch = epoch;
+        new_version_delta.new_table_watermarks = new_table_watermarks;
+        new_version_delta.change_log_delta = change_log_delta;
+
+        let mut table_compaction_group_mapping = new_version_delta
+            .latest_version()
+            .state_table_info
+            .build_table_compaction_group_id();
+
+        let mut new_table_ids = None;
+
+        // Add new table
+        if let Some(new_fragment_table_info) = new_table_fragment_info {
+            let new_table_ids = new_table_ids.insert(HashMap::new());
+            if !new_fragment_table_info.internal_table_ids.is_empty() {
+                for table_id in &new_fragment_table_info.internal_table_ids {
+                    if let Some(info) = new_version_delta
+                        .latest_version()
+                        .state_table_info
+                        .info()
+                        .get(table_id)
+                    {
+                        return Err(Error::CompactionGroup(format!(
+                            "table {} already exist {:?}",
+                            table_id.table_id, info,
+                        )));
+                    }
+                }
+
+                for table_id in &new_fragment_table_info.internal_table_ids {
+                    table_compaction_group_mapping
+                        .insert(*table_id, StaticCompactionGroupId::StateDefault as u64);
+                    new_table_ids.insert(*table_id, StaticCompactionGroupId::StateDefault as u64);
+                }
+            }
+
+            if let Some(table_id) = new_fragment_table_info.mv_table_id {
+                if let Some(info) = new_version_delta
+                    .latest_version()
+                    .state_table_info
+                    .info()
+                    .get(&table_id)
+                {
+                    return Err(Error::CompactionGroup(format!(
+                        "table {} already exist {:?}",
+                        table_id.table_id, info,
+                    )));
+                }
+                let _ = table_compaction_group_mapping
+                    .insert(table_id, StaticCompactionGroupId::MaterializedView as u64);
+                new_table_ids.insert(table_id, StaticCompactionGroupId::MaterializedView as u64);
+            }
+        }
+
+        let mut incorrect_ssts = vec![];
+        let mut new_sst_id_number = 0;
+        for ExtendedSstableInfo {
+            compaction_group_id,
+            sst_info: sst,
+            ..
+        } in &mut sstables
+        {
+            let is_sst_belong_to_group_declared = match new_version_delta
+                .latest_version()
+                .levels
+                .get(compaction_group_id)
+            {
+                Some(_compaction_group) => sst.table_ids.iter().all(|t| {
+                    table_compaction_group_mapping
+                        .get(&TableId::new(*t))
+                        .map(|table_cg_id| table_cg_id == compaction_group_id)
+                        .unwrap_or(false)
+                }),
+                None => false,
+            };
+            if !is_sst_belong_to_group_declared {
+                let mut group_table_ids: BTreeMap<_, Vec<u32>> = BTreeMap::new();
+                for table_id in sst.get_table_ids() {
+                    match table_compaction_group_mapping.get(&TableId::new(*table_id)) {
+                        Some(compaction_group_id) => {
+                            group_table_ids
+                                .entry(*compaction_group_id)
+                                .or_default()
+                                .push(*table_id);
+                        }
+                        None => {
+                            tracing::warn!(
+                                "table {} in SST {} doesn't belong to any compaction group",
+                                table_id,
+                                sst.get_object_id(),
+                            );
+                        }
+                    }
+                }
+                let is_trivial_adjust = group_table_ids.len() == 1
+                    && group_table_ids.first_key_value().unwrap().1.len()
+                        == sst.get_table_ids().len();
+                if is_trivial_adjust {
+                    *compaction_group_id = *group_table_ids.first_key_value().unwrap().0;
+                    // is_sst_belong_to_group_declared = true;
+                } else {
+                    new_sst_id_number += group_table_ids.len();
+                    incorrect_ssts.push((std::mem::take(sst), group_table_ids));
+                    *compaction_group_id =
+                        StaticCompactionGroupId::NewCompactionGroup as CompactionGroupId;
+                }
+            }
+        }
+        let mut new_sst_id = next_sstable_object_id(&self.env, new_sst_id_number).await?;
+        let original_sstables = std::mem::take(&mut sstables);
+        sstables.reserve_exact(original_sstables.len() - incorrect_ssts.len() + new_sst_id_number);
+        let mut incorrect_ssts = incorrect_ssts.into_iter();
+        for original_sstable in original_sstables {
+            if original_sstable.compaction_group_id
+                == StaticCompactionGroupId::NewCompactionGroup as CompactionGroupId
+            {
+                let (sst, group_table_ids) = incorrect_ssts.next().unwrap();
+                let mut branch_groups = HashMap::new();
+                for (group_id, _match_ids) in group_table_ids {
+                    let mut branch_sst = sst.clone();
+                    branch_sst.sst_id = new_sst_id;
+                    sstables.push(ExtendedSstableInfo::with_compaction_group(
+                        group_id, branch_sst,
+                    ));
+                    branch_groups.insert(group_id, new_sst_id);
+                    new_sst_id += 1;
+                }
+            } else {
+                sstables.push(original_sstable);
+            }
+        }
+
+        let mut modified_compaction_groups = vec![];
+        // Append SSTs to a new version.
+        for (compaction_group_id, sstables) in &sstables
+            .into_iter()
+            // the sort is stable sort, and will not change the order within compaction group.
+            // Do a sort so that sst in the same compaction group can be consecutive
+            .sorted_by_key(
+                |ExtendedSstableInfo {
+                     compaction_group_id,
+                     ..
+                 }| *compaction_group_id,
+            )
+            .group_by(
+                |ExtendedSstableInfo {
+                     compaction_group_id,
+                     ..
+                 }| *compaction_group_id,
+            )
+        {
+            modified_compaction_groups.push(compaction_group_id);
+            let group_sstables = sstables
+                .into_iter()
+                .map(|ExtendedSstableInfo { sst_info, .. }| sst_info)
+                .collect_vec();
+
+            let group_deltas = &mut new_version_delta
+                .group_deltas
+                .entry(compaction_group_id)
+                .or_default()
+                .group_deltas;
+            let l0_sub_level_id = epoch;
+            let group_delta = GroupDelta {
+                delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta {
+                    level_idx: 0,
+                    inserted_table_infos: group_sstables.clone(),
+                    l0_sub_level_id,
+                    ..Default::default()
+                })),
+            };
+            group_deltas.push(group_delta);
+        }
+
+        // update state table info
+        new_version_delta.with_latest_version(|version, delta| {
+            if let Some(new_table_ids) = new_table_ids {
+                for (table_id, cg_id) in new_table_ids {
+                    delta.state_table_info_delta.insert(
+                        table_id,
+                        StateTableInfoDelta {
+                            committed_epoch: epoch,
+                            safe_epoch: epoch,
+                            compaction_group_id: cg_id,
+                        },
+                    );
+                }
+            }
+            for (table_id, info) in version.state_table_info.info() {
+                assert!(
+                    delta
+                        .state_table_info_delta
+                        .insert(
+                            *table_id,
+                            StateTableInfoDelta {
+                                committed_epoch: epoch,
+                                safe_epoch: info.safe_epoch,
+                                compaction_group_id: info.compaction_group_id,
+                            }
+                        )
+                        .is_none(),
+                    "newly added table exists previously: {:?}",
+                    table_id
+                );
+            }
+        });
+
+        new_version_delta.pre_apply();
+
+        // Apply stats changes.
+        let mut version_stats = HummockVersionStatsTransaction::new(
+            &mut versioning.version_stats,
+            self.env.notification_manager(),
+        );
+        add_prost_table_stats_map(&mut version_stats.table_stats, &table_stats_change);
+        if purge_prost_table_stats(&mut version_stats.table_stats, version.latest_version()) {
+            self.metrics.version_stats.reset();
+            versioning.local_metrics.clear();
+        }
+
+        trigger_local_table_stat(
+            &self.metrics,
+            &mut versioning.local_metrics,
+            &version_stats,
+            &table_stats_change,
+        );
+        for (table_id, stats) in &table_stats_change {
+            if stats.total_key_size == 0
+                && stats.total_value_size == 0
+                && stats.total_key_count == 0
+            {
+                continue;
+            }
+            let stats_value = std::cmp::max(0, stats.total_key_size + stats.total_value_size);
+            let table_metrics = get_or_create_local_table_stat(
+                &self.metrics,
+                *table_id,
+                &mut versioning.local_metrics,
+            );
+            table_metrics.inc_write_throughput(stats_value as u64);
+        }
+        commit_multi_var!(self.meta_store_ref(), version, version_stats)?;
+
+        let snapshot = HummockSnapshot {
+            committed_epoch: epoch,
+            current_epoch: epoch,
+        };
+        let prev_snapshot = self.latest_snapshot.swap(snapshot.clone().into());
+        assert!(prev_snapshot.committed_epoch < epoch);
+        assert!(prev_snapshot.current_epoch < epoch);
+
+        for compaction_group_id in &modified_compaction_groups {
+            trigger_sst_stat(
+                &self.metrics,
+                None,
+                &versioning.current_version,
+                *compaction_group_id,
+            );
+        }
+
+        tracing::trace!("new committed epoch {}", epoch);
+
+        let mut table_groups = HashMap::<u32, usize>::default();
+        for (table_id, info) in versioning.current_version.state_table_info.info() {
+            table_groups.insert(
+                table_id.table_id,
+                versioning
+                    .current_version
+                    .state_table_info
+                    .compaction_group_member_tables()
+                    .get(&info.compaction_group_id)
+                    .expect("should exist")
+                    .len(),
+            );
+        }
+        drop(versioning_guard);
+        // Don't trigger compactions if we enable deterministic compaction
+        if !self.env.opts.compaction_deterministic_test {
+            // commit_epoch may contains SSTs from any compaction group
+            for id in &modified_compaction_groups {
+                self.try_send_compaction_request(*id, compact_task::TaskType::Dynamic);
+            }
+            if !table_stats_change.is_empty() {
+                table_stats_change.retain(|table_id, _| {
+                    table_groups
+                        .get(table_id)
+                        .map(|table_count| *table_count > 1)
+                        .unwrap_or(false)
+                });
+            }
+            if !table_stats_change.is_empty() {
+                self.collect_table_write_throughput(table_stats_change);
+            }
+        }
+        if !modified_compaction_groups.is_empty() {
+            self.try_update_write_limits(&modified_compaction_groups)
+                .await;
+        }
+        #[cfg(test)]
+        {
+            self.check_state_consistency().await;
+        }
+        Ok(Some(snapshot))
+    }
+
+    fn collect_table_write_throughput(&self, table_stats: PbTableStatsMap) {
+        let mut table_infos = self.history_table_throughput.write();
+        for (table_id, stat) in table_stats {
+            let throughput = (stat.total_value_size + stat.total_key_size) as u64;
+            let entry = table_infos.entry(table_id).or_default();
+            entry.push_back(throughput);
+            if entry.len() > HISTORY_TABLE_INFO_STATISTIC_TIME {
+                entry.pop_front();
+            }
+        }
+    }
+}
diff --git a/src/meta/src/hummock/manager/compaction.rs b/src/meta/src/hummock/manager/compaction.rs
index 39dd44565865d..e406292edef2f 100644
--- a/src/meta/src/hummock/manager/compaction.rs
+++ b/src/meta/src/hummock/manager/compaction.rs
@@ -12,37 +12,211 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::cmp::min;
 use std::collections::{BTreeMap, HashMap, HashSet};
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
+use std::time::{Instant, SystemTime};
 
-use function_name::named;
+use anyhow::Context;
+use fail::fail_point;
 use futures::future::Shared;
+use futures::stream::FuturesUnordered;
+use futures::{FutureExt, StreamExt};
 use itertools::Itertools;
-use risingwave_hummock_sdk::{CompactionGroupId, HummockCompactionTaskId};
+use parking_lot::Mutex;
+use rand::seq::SliceRandom;
+use rand::thread_rng;
+use risingwave_common::util::epoch::Epoch;
+use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt;
+use risingwave_hummock_sdk::table_stats::{
+    add_prost_table_stats_map, purge_prost_table_stats, PbTableStatsMap,
+};
+use risingwave_hummock_sdk::version::HummockVersion;
+use risingwave_hummock_sdk::{
+    compact_task_to_string, statistics_compact_task, CompactionGroupId, HummockCompactionTaskId,
+    HummockVersionId,
+};
 use risingwave_pb::hummock::compact_task::{TaskStatus, TaskType};
+use risingwave_pb::hummock::group_delta::DeltaType;
 use risingwave_pb::hummock::hummock_version::Levels;
 use risingwave_pb::hummock::subscribe_compaction_event_request::{
-    self, Event as RequestEvent, PullTask,
+    self, Event as RequestEvent, HeartBeat, PullTask, ReportTask,
 };
 use risingwave_pb::hummock::subscribe_compaction_event_response::{
     Event as ResponseEvent, PullTaskAck,
 };
 use risingwave_pb::hummock::{
-    CompactStatus as PbCompactStatus, CompactTaskAssignment, CompactionConfig,
+    compact_task, CompactStatus as PbCompactStatus, CompactTask, CompactTaskAssignment,
+    CompactionConfig, GroupDelta, InputLevel, IntraLevelDelta, Level, SstableInfo,
+    StateTableInfoDelta, SubscribeCompactionEventRequest, TableOption, TableSchema,
 };
+use rw_futures_util::pending_on_none;
 use thiserror_ext::AsReport;
-use tokio::sync::mpsc::UnboundedReceiver;
-use tokio::sync::oneshot::Receiver as OneShotReceiver;
+use tokio::sync::mpsc::error::SendError;
+use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver};
+use tokio::sync::oneshot::{Receiver as OneShotReceiver, Sender};
+use tokio::task::JoinHandle;
+use tonic::Streaming;
+use tracing::warn;
 
 use crate::hummock::compaction::selector::level_selector::PickerInfo;
-use crate::hummock::compaction::selector::DynamicLevelSelectorCore;
+use crate::hummock::compaction::selector::{
+    DynamicLevelSelector, DynamicLevelSelectorCore, LocalSelectorStatistic, ManualCompactionOption,
+    ManualCompactionSelector, SpaceReclaimCompactionSelector, TombstoneCompactionSelector,
+    TtlCompactionSelector,
+};
 use crate::hummock::compaction::{CompactStatus, CompactionDeveloperConfig, CompactionSelector};
-use crate::hummock::manager::{init_selectors, read_lock};
-use crate::hummock::HummockManager;
+use crate::hummock::error::{Error, Result};
+use crate::hummock::manager::transaction::{
+    HummockVersionStatsTransaction, HummockVersionTransaction,
+};
+use crate::hummock::manager::versioning::Versioning;
+use crate::hummock::metrics_utils::{
+    build_compact_task_level_type_metrics_label, trigger_local_table_stat, trigger_sst_stat,
+};
+use crate::hummock::sequence::next_compaction_task_id;
+use crate::hummock::{commit_multi_var, start_measure_real_process_timer, HummockManager};
+use crate::manager::{MetadataManager, META_NODE_ID};
+use crate::model::BTreeMapTransaction;
 
 const MAX_SKIP_TIMES: usize = 8;
 const MAX_REPORT_COUNT: usize = 16;
 
+static CANCEL_STATUS_SET: LazyLock<HashSet<TaskStatus>> = LazyLock::new(|| {
+    [
+        TaskStatus::ManualCanceled,
+        TaskStatus::SendFailCanceled,
+        TaskStatus::AssignFailCanceled,
+        TaskStatus::HeartbeatCanceled,
+        TaskStatus::InvalidGroupCanceled,
+        TaskStatus::NoAvailMemoryResourceCanceled,
+        TaskStatus::NoAvailCpuResourceCanceled,
+        TaskStatus::HeartbeatProgressCanceled,
+    ]
+    .into_iter()
+    .collect()
+});
+
+type CompactionRequestChannelItem = (CompactionGroupId, compact_task::TaskType);
+
+fn init_selectors() -> HashMap<compact_task::TaskType, Box<dyn CompactionSelector>> {
+    let mut compaction_selectors: HashMap<compact_task::TaskType, Box<dyn CompactionSelector>> =
+        HashMap::default();
+    compaction_selectors.insert(
+        compact_task::TaskType::Dynamic,
+        Box::<DynamicLevelSelector>::default(),
+    );
+    compaction_selectors.insert(
+        compact_task::TaskType::SpaceReclaim,
+        Box::<SpaceReclaimCompactionSelector>::default(),
+    );
+    compaction_selectors.insert(
+        compact_task::TaskType::Ttl,
+        Box::<TtlCompactionSelector>::default(),
+    );
+    compaction_selectors.insert(
+        compact_task::TaskType::Tombstone,
+        Box::<TombstoneCompactionSelector>::default(),
+    );
+    compaction_selectors
+}
+
+impl<'a> HummockVersionTransaction<'a> {
+    fn apply_compact_task(&mut self, compact_task: &CompactTask) {
+        let mut version_delta = self.new_delta();
+        let trivial_move = CompactStatus::is_trivial_move_task(compact_task);
+        version_delta.trivial_move = trivial_move;
+
+        let group_deltas = &mut version_delta
+            .group_deltas
+            .entry(compact_task.compaction_group_id)
+            .or_default()
+            .group_deltas;
+        let mut removed_table_ids_map: BTreeMap<u32, Vec<u64>> = BTreeMap::default();
+
+        for level in &compact_task.input_ssts {
+            let level_idx = level.level_idx;
+            let mut removed_table_ids = level
+                .table_infos
+                .iter()
+                .map(|sst| sst.get_sst_id())
+                .collect_vec();
+
+            removed_table_ids_map
+                .entry(level_idx)
+                .or_default()
+                .append(&mut removed_table_ids);
+        }
+
+        for (level_idx, removed_table_ids) in removed_table_ids_map {
+            let group_delta = GroupDelta {
+                delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta {
+                    level_idx,
+                    removed_table_ids,
+                    ..Default::default()
+                })),
+            };
+            group_deltas.push(group_delta);
+        }
+
+        let group_delta = GroupDelta {
+            delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta {
+                level_idx: compact_task.target_level,
+                inserted_table_infos: compact_task.sorted_output_ssts.clone(),
+                l0_sub_level_id: compact_task.target_sub_level_id,
+                vnode_partition_count: compact_task.split_weight_by_vnode,
+                ..Default::default()
+            })),
+        };
+        group_deltas.push(group_delta);
+        version_delta.safe_epoch = std::cmp::max(
+            version_delta.latest_version().visible_table_safe_epoch(),
+            compact_task.watermark,
+        );
+        if version_delta.latest_version().visible_table_safe_epoch() < version_delta.safe_epoch {
+            version_delta.with_latest_version(|version, version_delta| {
+                for (table_id, info) in version.state_table_info.info() {
+                    let new_safe_epoch = min(version_delta.safe_epoch, info.committed_epoch);
+                    if new_safe_epoch > info.safe_epoch {
+                        if new_safe_epoch != version_delta.safe_epoch {
+                            warn!(
+                                new_safe_epoch,
+                                committed_epoch = info.committed_epoch,
+                                global_safe_epoch = version_delta.safe_epoch,
+                                table_id = table_id.table_id,
+                                "table has different safe epoch to global"
+                            );
+                        }
+                        version_delta.state_table_info_delta.insert(
+                            *table_id,
+                            StateTableInfoDelta {
+                                committed_epoch: info.committed_epoch,
+                                safe_epoch: new_safe_epoch,
+                                compaction_group_id: info.compaction_group_id,
+                            },
+                        );
+                    }
+                }
+            });
+        }
+        version_delta.pre_apply();
+    }
+}
+
 #[derive(Default)]
 pub struct Compaction {
     /// Compaction task that is already assigned to a compactor
@@ -50,38 +224,18 @@ pub struct Compaction {
     /// `CompactStatus` of each compaction group
     pub compaction_statuses: BTreeMap<CompactionGroupId, CompactStatus>,
 
-    pub deterministic_mode: bool,
+    pub _deterministic_mode: bool,
 }
 
 impl HummockManager {
-    #[named]
     pub async fn get_assigned_compact_task_num(&self) -> u64 {
-        read_lock!(self, compaction)
-            .await
-            .compact_task_assignment
-            .len() as u64
+        self.compaction.read().await.compact_task_assignment.len() as u64
     }
 
-    #[named]
-    pub async fn list_all_tasks_ids(&self) -> Vec<HummockCompactionTaskId> {
-        let compaction = read_lock!(self, compaction).await;
-
-        compaction
-            .compaction_statuses
-            .iter()
-            .flat_map(|(_, cs)| {
-                cs.level_handlers
-                    .iter()
-                    .flat_map(|lh| lh.pending_tasks_ids())
-            })
-            .collect_vec()
-    }
-
-    #[named]
     pub async fn list_compaction_status(
         &self,
     ) -> (Vec<PbCompactStatus>, Vec<CompactTaskAssignment>) {
-        let compaction = read_lock!(self, compaction).await;
+        let compaction = self.compaction.read().await;
         (
             compaction.compaction_statuses.values().map_into().collect(),
             compaction
@@ -92,14 +246,13 @@ impl HummockManager {
         )
     }
 
-    #[named]
     pub async fn get_compaction_scores(
         &self,
         compaction_group_id: CompactionGroupId,
     ) -> Vec<PickerInfo> {
         let (status, levels, group) = {
-            let compaction = read_lock!(self, compaction).await;
-            let versioning = read_lock!(self, versioning).await;
+            let compaction = self.compaction.read().await;
+            let versioning = self.versioning.read().await;
             let config_manager = self.compaction_group_manager.read().await;
             match (
                 compaction.compaction_statuses.get(&compaction_group_id),
@@ -119,12 +272,15 @@ impl HummockManager {
         let ctx = dynamic_level_core.get_priority_levels(&levels, &status.level_handlers);
         ctx.score_levels
     }
+}
 
-    pub async fn handle_pull_task_event(
+impl HummockManager {
+    async fn handle_pull_task_event(
         &self,
         context_id: u32,
         pull_task_count: usize,
         compaction_selectors: &mut HashMap<TaskType, Box<dyn CompactionSelector>>,
+        max_get_task_probe_times: usize,
     ) {
         assert_ne!(0, pull_task_count);
         if let Some(compactor) = self.compactor_manager.get_compactor(context_id) {
@@ -137,8 +293,13 @@ impl HummockManager {
                 let mut existed_groups = groups.clone();
                 let mut no_task_groups: HashSet<CompactionGroupId> = HashSet::default();
                 let mut failed_tasks = vec![];
+                let mut loop_times = 0;
 
-                while generated_task_count < pull_task_count && failed_tasks.is_empty() {
+                while generated_task_count < pull_task_count
+                    && failed_tasks.is_empty()
+                    && loop_times < max_get_task_probe_times
+                {
+                    loop_times += 1;
                     let compact_ret = self
                         .get_compact_tasks(
                             existed_groups.clone(),
@@ -149,11 +310,11 @@ impl HummockManager {
 
                     match compact_ret {
                         Ok((compact_tasks, unschedule_groups)) => {
+                            no_task_groups.extend(unschedule_groups);
                             if compact_tasks.is_empty() {
                                 break;
                             }
                             generated_task_count += compact_tasks.len();
-                            no_task_groups.extend(unschedule_groups);
                             for task in compact_tasks {
                                 let task_id = task.task_id;
                                 if let Err(e) =
@@ -203,7 +364,217 @@ impl HummockManager {
     }
 
     /// dedicated event runtime for CPU/IO bound event
-    pub async fn compact_task_dedicated_event_handler(
+    pub fn compaction_event_loop(
+        hummock_manager: Arc<Self>,
+        mut compactor_streams_change_rx: UnboundedReceiver<(
+            u32,
+            Streaming<SubscribeCompactionEventRequest>,
+        )>,
+    ) -> Vec<(JoinHandle<()>, Sender<()>)> {
+        let mut compactor_request_streams = FuturesUnordered::new();
+        let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
+        let (shutdown_tx_dedicated, shutdown_rx_dedicated) = tokio::sync::oneshot::channel();
+        let shutdown_rx_shared = shutdown_rx.shared();
+        let shutdown_rx_dedicated_shared = shutdown_rx_dedicated.shared();
+
+        let (tx, rx) = unbounded_channel();
+
+        let mut join_handle_vec = Vec::default();
+
+        let hummock_manager_dedicated = hummock_manager.clone();
+        let compact_task_event_handler_join_handle = tokio::spawn(async move {
+            Self::compact_task_dedicated_event_handler(
+                hummock_manager_dedicated,
+                rx,
+                shutdown_rx_dedicated_shared,
+            )
+            .await;
+        });
+
+        join_handle_vec.push((
+            compact_task_event_handler_join_handle,
+            shutdown_tx_dedicated,
+        ));
+
+        let join_handle = tokio::spawn(async move {
+            let push_stream =
+                |context_id: u32,
+                 stream: Streaming<SubscribeCompactionEventRequest>,
+                 compactor_request_streams: &mut FuturesUnordered<_>| {
+                    let future = stream
+                        .into_future()
+                        .map(move |stream_future| (context_id, stream_future));
+
+                    compactor_request_streams.push(future);
+                };
+
+            let mut event_loop_iteration_now = Instant::now();
+
+            loop {
+                let shutdown_rx_shared = shutdown_rx_shared.clone();
+                let hummock_manager = hummock_manager.clone();
+                hummock_manager
+                    .metrics
+                    .compaction_event_loop_iteration_latency
+                    .observe(event_loop_iteration_now.elapsed().as_millis() as _);
+                event_loop_iteration_now = Instant::now();
+
+                tokio::select! {
+                    _ = shutdown_rx_shared => { return; },
+
+                    compactor_stream = compactor_streams_change_rx.recv() => {
+                        if let Some((context_id, stream)) = compactor_stream {
+                            tracing::info!("compactor {} enters the cluster", context_id);
+                            push_stream(context_id, stream, &mut compactor_request_streams);
+                        }
+                    },
+
+                    result = pending_on_none(compactor_request_streams.next()) => {
+                        let mut compactor_alive = true;
+
+                        let (context_id, compactor_stream_req): (_, (std::option::Option<std::result::Result<SubscribeCompactionEventRequest, _>>, _)) = result;
+                        let (event, create_at, stream) = match compactor_stream_req {
+                            (Some(Ok(req)), stream) => {
+                                (req.event.unwrap(), req.create_at, stream)
+                            }
+
+                            (Some(Err(err)), _stream) => {
+                                tracing::warn!(error = %err.as_report(), "compactor stream {} poll with err, recv stream may be destroyed", context_id);
+                                continue
+                            }
+
+                            _ => {
+                                tracing::warn!("compactor stream {} poll err, recv stream may be destroyed", context_id);
+                                continue
+                            },
+                        };
+
+                        {
+                            let consumed_latency_ms = SystemTime::now()
+                                .duration_since(std::time::UNIX_EPOCH)
+                                .expect("Clock may have gone backwards")
+                                .as_millis()
+                                as u64
+                            - create_at;
+                            hummock_manager.metrics
+                                .compaction_event_consumed_latency
+                                .observe(consumed_latency_ms as _);
+                        }
+
+                        match event {
+                            RequestEvent::HeartBeat(HeartBeat {
+                                progress,
+                            }) => {
+                                let compactor_manager = hummock_manager.compactor_manager.clone();
+                                let cancel_tasks = compactor_manager.update_task_heartbeats(&progress).into_iter().map(|task|task.task_id).collect::<Vec<_>>();
+                                if !cancel_tasks.is_empty() {
+                                    tracing::info!(
+                                        "Tasks cancel with task_ids {:?} with context_id {} has expired due to lack of visible progress",
+                                        cancel_tasks,
+                                        context_id,
+                                    );
+
+                                    if let Err(e) = hummock_manager
+                                        .cancel_compact_tasks(cancel_tasks.clone(), TaskStatus::HeartbeatProgressCanceled)
+                                        .await
+                                    {
+                                        tracing::error!(
+                                            error = %e.as_report(),
+                                            "Attempt to remove compaction task due to elapsed heartbeat failed. We will continue to track its heartbeat
+                                            until we can successfully report its status."
+                                        );
+                                    }
+                                }
+
+                                if let Some(compactor) = compactor_manager.get_compactor(context_id) {
+                                    // Forcefully cancel the task so that it terminates
+                                    // early on the compactor
+                                    // node.
+                                    let _ = compactor.cancel_tasks(&cancel_tasks);
+                                    tracing::info!(
+                                        "CancelTask operation for task_id {:?} has been sent to node with context_id {}",
+                                        cancel_tasks,
+                                        context_id
+                                    );
+                                } else {
+                                    // Determine the validity of the compactor streaming rpc. When the compactor no longer exists in the manager, the stream will be removed.
+                                    // Tip: Connectivity to the compactor will be determined through the `send_event` operation. When send fails, it will be removed from the manager
+                                    compactor_alive = false;
+                                }
+                            },
+
+                            RequestEvent::Register(_) => {
+                                unreachable!()
+                            }
+
+                            e @ (RequestEvent::PullTask(_) | RequestEvent::ReportTask(_)) => {
+                                let _ = tx.send((context_id, e));
+                            }
+                        }
+
+                        if compactor_alive {
+                            push_stream(context_id, stream, &mut compactor_request_streams);
+                        } else {
+                            tracing::warn!("compactor stream {} error, send stream may be destroyed", context_id);
+                        }
+                    },
+                }
+            }
+        });
+
+        join_handle_vec.push((join_handle, shutdown_tx));
+
+        join_handle_vec
+    }
+
+    pub fn add_compactor_stream(
+        &self,
+        context_id: u32,
+        req_stream: Streaming<SubscribeCompactionEventRequest>,
+    ) {
+        self.compactor_streams_change_tx
+            .send((context_id, req_stream))
+            .unwrap();
+    }
+
+    pub async fn auto_pick_compaction_group_and_type(
+        &self,
+    ) -> Option<(CompactionGroupId, compact_task::TaskType)> {
+        let mut compaction_group_ids = self.compaction_group_ids().await;
+        compaction_group_ids.shuffle(&mut thread_rng());
+
+        for cg_id in compaction_group_ids {
+            if let Some(pick_type) = self.compaction_state.auto_pick_type(cg_id) {
+                return Some((cg_id, pick_type));
+            }
+        }
+
+        None
+    }
+
+    /// This method will return all compaction group id in a random order and task type. If there are any group block by `write_limit`, it will return a single array with `TaskType::Emergency`.
+    /// If these groups get different task-type, it will return all group id with `TaskType::Dynamic` if the first group get `TaskType::Dynamic`, otherwise it will return the single group with other task type.
+    async fn auto_pick_compaction_groups_and_type(
+        &self,
+    ) -> (Vec<CompactionGroupId>, compact_task::TaskType) {
+        let mut compaction_group_ids = self.compaction_group_ids().await;
+        compaction_group_ids.shuffle(&mut thread_rng());
+
+        let mut normal_groups = vec![];
+        for cg_id in compaction_group_ids {
+            if let Some(pick_type) = self.compaction_state.auto_pick_type(cg_id) {
+                if pick_type == TaskType::Dynamic {
+                    normal_groups.push(cg_id);
+                } else if normal_groups.is_empty() {
+                    return (vec![cg_id], pick_type);
+                }
+            }
+        }
+        (normal_groups, TaskType::Dynamic)
+    }
+
+    /// dedicated event runtime for CPU/IO bound event
+    async fn compact_task_dedicated_event_handler(
         hummock_manager: Arc<HummockManager>,
         mut rx: UnboundedReceiver<(u32, subscribe_compaction_event_request::Event)>,
         shutdown_rx_shared: Shared<OneShotReceiver<()>>,
@@ -219,7 +590,7 @@ impl HummockManager {
                     let mut skip_times = 0;
                     match event {
                         RequestEvent::PullTask(PullTask { pull_task_count }) => {
-                            hummock_manager.handle_pull_task_event(context_id, pull_task_count as usize, &mut compaction_selectors).await;
+                            hummock_manager.handle_pull_task_event(context_id, pull_task_count as usize, &mut compaction_selectors, hummock_manager.env.opts.max_get_task_probe_times).await;
                         }
 
                         RequestEvent::ReportTask(task) => {
@@ -231,7 +602,7 @@ impl HummockManager {
                     while let Ok((context_id, event)) = rx.try_recv() {
                         match event {
                             RequestEvent::PullTask(PullTask { pull_task_count }) => {
-                                hummock_manager.handle_pull_task_event(context_id, pull_task_count as usize, &mut compaction_selectors).await;
+                                hummock_manager.handle_pull_task_event(context_id, pull_task_count as usize, &mut compaction_selectors, hummock_manager.env.opts.max_get_task_probe_times).await;
                                 if !report_events.is_empty() {
                                     if skip_times > MAX_SKIP_TIMES {
                                         break;
@@ -261,6 +632,980 @@ impl HummockManager {
     }
 }
 
+impl HummockManager {
+    pub async fn get_compact_tasks_impl(
+        &self,
+        compaction_groups: Vec<CompactionGroupId>,
+        max_select_count: usize,
+        selector: &mut Box<dyn CompactionSelector>,
+    ) -> Result<(Vec<CompactTask>, Vec<CompactionGroupId>)> {
+        // TODO: `get_all_table_options` will hold catalog_manager async lock, to avoid holding the
+        // lock in compaction_guard, take out all table_options in advance there may be a
+        // waste of resources here, need to add a more efficient filter in catalog_manager
+        let deterministic_mode = self.env.opts.compaction_deterministic_test;
+        let all_table_id_to_option = self
+            .metadata_manager
+            .get_all_table_options()
+            .await
+            .map_err(|err| Error::MetaStore(err.into()))?;
+
+        let mut compaction_guard = self.compaction.write().await;
+        let compaction: &mut Compaction = &mut compaction_guard;
+        let mut versioning_guard = self.versioning.write().await;
+        let versioning: &mut Versioning = &mut versioning_guard;
+
+        let _timer = start_measure_real_process_timer!(self, "get_compact_tasks_impl");
+
+        let start_time = Instant::now();
+        let max_committed_epoch = versioning.current_version.max_committed_epoch;
+        let watermark = self
+            .context_info
+            .read()
+            .await
+            .pinned_snapshots
+            .values()
+            .map(|v| v.minimal_pinned_snapshot)
+            .fold(max_committed_epoch, std::cmp::min);
+
+        let mut compaction_statuses = BTreeMapTransaction::new(&mut compaction.compaction_statuses);
+
+        let mut compact_task_assignment =
+            BTreeMapTransaction::new(&mut compaction.compact_task_assignment);
+
+        let mut version = HummockVersionTransaction::new(
+            &mut versioning.current_version,
+            &mut versioning.hummock_version_deltas,
+            self.env.notification_manager(),
+            &self.metrics,
+        );
+
+        if deterministic_mode {
+            version.disable_apply_to_txn();
+        }
+
+        let mut unschedule_groups = vec![];
+        let mut trivial_tasks = vec![];
+        let mut pick_tasks = vec![];
+        let developer_config = Arc::new(CompactionDeveloperConfig::new_from_meta_opts(
+            &self.env.opts,
+        ));
+        'outside: for compaction_group_id in compaction_groups {
+            if pick_tasks.len() >= max_select_count {
+                break;
+            }
+
+            if !version
+                .latest_version()
+                .levels
+                .contains_key(&compaction_group_id)
+            {
+                continue;
+            }
+
+            // When the last table of a compaction group is deleted, the compaction group (and its
+            // config) is destroyed as well. Then a compaction task for this group may come later and
+            // cannot find its config.
+            let group_config = {
+                let config_manager = self.compaction_group_manager.read().await;
+
+                match config_manager.try_get_compaction_group_config(compaction_group_id) {
+                    Some(config) => config,
+                    None => continue,
+                }
+            };
+
+            // StoredIdGenerator already implements ids pre-allocation by ID_PREALLOCATE_INTERVAL.
+            let task_id = next_compaction_task_id(&self.env).await?;
+
+            if !compaction_statuses.contains_key(&compaction_group_id) {
+                // lazy initialize.
+                compaction_statuses.insert(
+                    compaction_group_id,
+                    CompactStatus::new(
+                        compaction_group_id,
+                        group_config.compaction_config.max_level,
+                    ),
+                );
+            }
+            let mut compact_status = compaction_statuses.get_mut(compaction_group_id).unwrap();
+
+            let can_trivial_move = matches!(selector.task_type(), TaskType::Dynamic)
+                || matches!(selector.task_type(), TaskType::Emergency);
+
+            let mut stats = LocalSelectorStatistic::default();
+            let member_table_ids: Vec<_> = version
+                .latest_version()
+                .state_table_info
+                .compaction_group_member_table_ids(compaction_group_id)
+                .iter()
+                .map(|table_id| table_id.table_id)
+                .collect();
+
+            let mut table_id_to_option: HashMap<u32, _> = HashMap::default();
+
+            for table_id in &member_table_ids {
+                if let Some(opts) = all_table_id_to_option.get(table_id) {
+                    table_id_to_option.insert(*table_id, *opts);
+                }
+            }
+
+            while let Some(compact_task) = compact_status.get_compact_task(
+                version
+                    .latest_version()
+                    .get_compaction_group_levels(compaction_group_id),
+                version
+                    .latest_version()
+                    .state_table_info
+                    .compaction_group_member_table_ids(compaction_group_id),
+                task_id as HummockCompactionTaskId,
+                &group_config,
+                &mut stats,
+                selector,
+                table_id_to_option.clone(),
+                developer_config.clone(),
+            ) {
+                let target_level_id = compact_task.input.target_level as u32;
+
+                let compression_algorithm = match compact_task.compression_algorithm.as_str() {
+                    "Lz4" => 1,
+                    "Zstd" => 2,
+                    _ => 0,
+                };
+                let vnode_partition_count = compact_task.input.vnode_partition_count;
+                use risingwave_hummock_sdk::prost_key_range::KeyRangeExt;
+
+                let mut compact_task = CompactTask {
+                    input_ssts: compact_task.input.input_levels,
+                    splits: vec![risingwave_pb::hummock::KeyRange::inf()],
+                    watermark,
+                    sorted_output_ssts: vec![],
+                    task_id,
+                    target_level: target_level_id,
+                    // only gc delete keys in last level because there may be older version in more bottom
+                    // level.
+                    gc_delete_keys: version
+                        .latest_version()
+                        .get_compaction_group_levels(compaction_group_id)
+                        .is_last_level(target_level_id),
+                    base_level: compact_task.base_level as u32,
+                    task_status: TaskStatus::Pending as i32,
+                    compaction_group_id: group_config.group_id,
+                    existing_table_ids: member_table_ids.clone(),
+                    compression_algorithm,
+                    target_file_size: compact_task.target_file_size,
+                    table_options: table_id_to_option
+                        .iter()
+                        .map(|(table_id, table_option)| {
+                            (*table_id, TableOption::from(table_option))
+                        })
+                        .collect(),
+                    current_epoch_time: Epoch::now().0,
+                    compaction_filter_mask: group_config.compaction_config.compaction_filter_mask,
+                    target_sub_level_id: compact_task.input.target_sub_level_id,
+                    task_type: compact_task.compaction_task_type as i32,
+                    split_weight_by_vnode: vnode_partition_count,
+                    max_sub_compaction: group_config.compaction_config.max_sub_compaction,
+                    ..Default::default()
+                };
+
+                let is_trivial_reclaim = CompactStatus::is_trivial_reclaim(&compact_task);
+                let is_trivial_move = CompactStatus::is_trivial_move_task(&compact_task);
+                if is_trivial_reclaim || (is_trivial_move && can_trivial_move) {
+                    let log_label = if is_trivial_reclaim {
+                        "TrivialReclaim"
+                    } else {
+                        "TrivialMove"
+                    };
+                    let label = if is_trivial_reclaim {
+                        "trivial-space-reclaim"
+                    } else {
+                        "trivial-move"
+                    };
+
+                    tracing::debug!(
+                        "{} for compaction group {}: input: {:?}, cost time: {:?}",
+                        log_label,
+                        compact_task.compaction_group_id,
+                        compact_task.input_ssts,
+                        start_time.elapsed()
+                    );
+                    compact_task.set_task_status(TaskStatus::Success);
+                    compact_status.report_compact_task(&compact_task);
+                    if !is_trivial_reclaim {
+                        compact_task
+                            .sorted_output_ssts
+                            .clone_from(&compact_task.input_ssts[0].table_infos);
+                    }
+                    self.metrics
+                        .compact_frequency
+                        .with_label_values(&[
+                            label,
+                            &compact_task.compaction_group_id.to_string(),
+                            selector.task_type().as_str_name(),
+                            "SUCCESS",
+                        ])
+                        .inc();
+
+                    version.apply_compact_task(&compact_task);
+                    trivial_tasks.push(compact_task);
+                    if trivial_tasks.len() >= self.env.opts.max_trivial_move_task_count_per_loop {
+                        break 'outside;
+                    }
+                } else {
+                    self.calculate_vnode_partition(
+                        &mut compact_task,
+                        group_config.compaction_config.as_ref(),
+                    )
+                    .await;
+                    compact_task.table_watermarks = version
+                        .latest_version()
+                        .safe_epoch_table_watermarks(&compact_task.existing_table_ids);
+
+                    if self.env.opts.enable_dropped_column_reclaim {
+                        // TODO: get all table schemas for all tables in once call to avoid acquiring lock and await.
+                        compact_task.table_schemas = match self.metadata_manager() {
+                            MetadataManager::V1(mgr) => mgr
+                                .catalog_manager
+                                .get_versioned_table_schemas(&compact_task.existing_table_ids)
+                                .await
+                                .into_iter()
+                                .map(|(table_id, column_ids)| {
+                                    (table_id, TableSchema { column_ids })
+                                })
+                                .collect(),
+                            MetadataManager::V2(_) => {
+                                // TODO #13952: support V2
+                                BTreeMap::default()
+                            }
+                        };
+                    }
+
+                    compact_task_assignment.insert(
+                        compact_task.task_id,
+                        CompactTaskAssignment {
+                            compact_task: Some(compact_task.clone()),
+                            context_id: META_NODE_ID, // deprecated
+                        },
+                    );
+
+                    pick_tasks.push(compact_task);
+                    break;
+                }
+
+                stats.report_to_metrics(compaction_group_id, self.metrics.as_ref());
+                stats = LocalSelectorStatistic::default();
+            }
+            if pick_tasks
+                .last()
+                .map(|task| task.compaction_group_id != compaction_group_id)
+                .unwrap_or(true)
+            {
+                unschedule_groups.push(compaction_group_id);
+            }
+            stats.report_to_metrics(compaction_group_id, self.metrics.as_ref());
+        }
+
+        if !trivial_tasks.is_empty() {
+            commit_multi_var!(
+                self.meta_store_ref(),
+                compaction_statuses,
+                compact_task_assignment,
+                version
+            )?;
+            self.metrics
+                .compact_task_batch_count
+                .with_label_values(&["batch_trivial_move"])
+                .observe(trivial_tasks.len() as f64);
+            drop(versioning_guard);
+        } else {
+            // We are using a single transaction to ensure that each task has progress when it is
+            // created.
+            drop(versioning_guard);
+            commit_multi_var!(
+                self.meta_store_ref(),
+                compaction_statuses,
+                compact_task_assignment
+            )?;
+        }
+        drop(compaction_guard);
+        if !pick_tasks.is_empty() {
+            self.metrics
+                .compact_task_batch_count
+                .with_label_values(&["batch_get_compact_task"])
+                .observe(pick_tasks.len() as f64);
+        }
+
+        for compact_task in &mut pick_tasks {
+            let compaction_group_id = compact_task.compaction_group_id;
+
+            // Initiate heartbeat for the task to track its progress.
+            self.compactor_manager
+                .initiate_task_heartbeat(compact_task.clone());
+
+            // this task has been finished.
+            compact_task.set_task_status(TaskStatus::Pending);
+            let compact_task_statistics = statistics_compact_task(compact_task);
+
+            let level_type_label = build_compact_task_level_type_metrics_label(
+                compact_task.input_ssts[0].level_idx as usize,
+                compact_task.input_ssts.last().unwrap().level_idx as usize,
+            );
+
+            let level_count = compact_task.input_ssts.len();
+            if compact_task.input_ssts[0].level_idx == 0 {
+                self.metrics
+                    .l0_compact_level_count
+                    .with_label_values(&[&compaction_group_id.to_string(), &level_type_label])
+                    .observe(level_count as _);
+            }
+
+            self.metrics
+                .compact_task_size
+                .with_label_values(&[&compaction_group_id.to_string(), &level_type_label])
+                .observe(compact_task_statistics.total_file_size as _);
+
+            self.metrics
+                .compact_task_size
+                .with_label_values(&[
+                    &compaction_group_id.to_string(),
+                    &format!("{} uncompressed", level_type_label),
+                ])
+                .observe(compact_task_statistics.total_uncompressed_file_size as _);
+
+            self.metrics
+                .compact_task_file_count
+                .with_label_values(&[&compaction_group_id.to_string(), &level_type_label])
+                .observe(compact_task_statistics.total_file_count as _);
+
+            tracing::trace!(
+                    "For compaction group {}: pick up {} {} sub_level in level {} to compact to target {}. cost time: {:?} compact_task_statistics {:?}",
+                    compaction_group_id,
+                    level_count,
+                    compact_task.input_ssts[0].level_type().as_str_name(),
+                    compact_task.input_ssts[0].level_idx,
+                    compact_task.target_level,
+                    start_time.elapsed(),
+                    compact_task_statistics
+                );
+        }
+
+        #[cfg(test)]
+        {
+            self.check_state_consistency().await;
+        }
+        pick_tasks.extend(trivial_tasks);
+        Ok((pick_tasks, unschedule_groups))
+    }
+
+    /// Cancels a compaction task no matter it's assigned or unassigned.
+    pub async fn cancel_compact_task(&self, task_id: u64, task_status: TaskStatus) -> Result<bool> {
+        fail_point!("fp_cancel_compact_task", |_| Err(Error::MetaStore(
+            anyhow::anyhow!("failpoint metastore err")
+        )));
+        let ret = self
+            .cancel_compact_task_impl(vec![task_id], task_status)
+            .await?;
+        Ok(ret[0])
+    }
+
+    pub async fn cancel_compact_tasks(
+        &self,
+        tasks: Vec<u64>,
+        task_status: TaskStatus,
+    ) -> Result<Vec<bool>> {
+        self.cancel_compact_task_impl(tasks, task_status).await
+    }
+
+    async fn cancel_compact_task_impl(
+        &self,
+        task_ids: Vec<u64>,
+        task_status: TaskStatus,
+    ) -> Result<Vec<bool>> {
+        assert!(CANCEL_STATUS_SET.contains(&task_status));
+        let tasks = task_ids
+            .into_iter()
+            .map(|task_id| ReportTask {
+                task_id,
+                task_status: task_status as i32,
+                sorted_output_ssts: vec![],
+                table_stats_change: HashMap::default(),
+            })
+            .collect_vec();
+        let rets = self.report_compact_tasks(tasks).await?;
+        #[cfg(test)]
+        {
+            self.check_state_consistency().await;
+        }
+        Ok(rets)
+    }
+
+    async fn get_compact_tasks(
+        &self,
+        mut compaction_groups: Vec<CompactionGroupId>,
+        max_select_count: usize,
+        selector: &mut Box<dyn CompactionSelector>,
+    ) -> Result<(Vec<CompactTask>, Vec<CompactionGroupId>)> {
+        fail_point!("fp_get_compact_task", |_| Err(Error::MetaStore(
+            anyhow::anyhow!("failpoint metastore error")
+        )));
+        compaction_groups.shuffle(&mut thread_rng());
+        let (mut tasks, groups) = self
+            .get_compact_tasks_impl(compaction_groups, max_select_count, selector)
+            .await?;
+        tasks.retain(|task| {
+            if task.task_status() == TaskStatus::Success {
+                debug_assert!(
+                    CompactStatus::is_trivial_reclaim(task)
+                        || CompactStatus::is_trivial_move_task(task)
+                );
+                false
+            } else {
+                true
+            }
+        });
+        Ok((tasks, groups))
+    }
+
+    pub async fn get_compact_task(
+        &self,
+        compaction_group_id: CompactionGroupId,
+        selector: &mut Box<dyn CompactionSelector>,
+    ) -> Result<Option<CompactTask>> {
+        fail_point!("fp_get_compact_task", |_| Err(Error::MetaStore(
+            anyhow::anyhow!("failpoint metastore error")
+        )));
+
+        let (normal_tasks, _) = self
+            .get_compact_tasks_impl(vec![compaction_group_id], 1, selector)
+            .await?;
+        for task in normal_tasks {
+            if task.task_status() != TaskStatus::Success {
+                return Ok(Some(task));
+            }
+            debug_assert!(
+                CompactStatus::is_trivial_reclaim(&task)
+                    || CompactStatus::is_trivial_move_task(&task)
+            );
+        }
+        Ok(None)
+    }
+
+    pub async fn manual_get_compact_task(
+        &self,
+        compaction_group_id: CompactionGroupId,
+        manual_compaction_option: ManualCompactionOption,
+    ) -> Result<Option<CompactTask>> {
+        let mut selector: Box<dyn CompactionSelector> =
+            Box::new(ManualCompactionSelector::new(manual_compaction_option));
+        self.get_compact_task(compaction_group_id, &mut selector)
+            .await
+    }
+
+    pub(super) fn is_compact_task_expired(
+        compact_task: &CompactTask,
+        hummock_version: &HummockVersion,
+    ) -> bool {
+        if let Some(group) = hummock_version
+            .levels
+            .get(&compact_task.compaction_group_id)
+        {
+            for input_level in compact_task.get_input_ssts() {
+                let input_level: &InputLevel = input_level;
+                let mut sst_ids: HashSet<_> = input_level
+                    .table_infos
+                    .iter()
+                    .map(|sst| sst.sst_id)
+                    .collect();
+                fn filter_ssts(levels: &Level, sst_ids: &mut HashSet<u64>) {
+                    for sst in &levels.table_infos {
+                        sst_ids.remove(&sst.sst_id);
+                    }
+                }
+                if input_level.level_idx == 0 {
+                    for level in &group.get_level0().sub_levels {
+                        filter_ssts(level, &mut sst_ids);
+                    }
+                } else {
+                    filter_ssts(group.get_level(input_level.level_idx as _), &mut sst_ids);
+                }
+                if !sst_ids.is_empty() {
+                    warn!(stale_sst_id = ?sst_ids, ?compact_task, "compact task expired");
+                    return true;
+                }
+            }
+        }
+        false
+    }
+
+    pub async fn report_compact_task(
+        &self,
+        task_id: u64,
+        task_status: TaskStatus,
+        sorted_output_ssts: Vec<SstableInfo>,
+        table_stats_change: Option<PbTableStatsMap>,
+    ) -> Result<bool> {
+        let rets = self
+            .report_compact_tasks(vec![ReportTask {
+                task_id,
+                task_status: task_status as i32,
+                sorted_output_ssts,
+                table_stats_change: table_stats_change.unwrap_or_default(),
+            }])
+            .await?;
+        Ok(rets[0])
+    }
+
+    /// Finishes or cancels a compaction task, according to `task_status`.
+    ///
+    /// If `context_id` is not None, its validity will be checked when writing meta store.
+    /// Its ownership of the task is checked as well.
+    ///
+    /// Return Ok(false) indicates either the task is not found,
+    /// or the task is not owned by `context_id` when `context_id` is not None.
+
+    pub async fn report_compact_tasks(&self, report_tasks: Vec<ReportTask>) -> Result<Vec<bool>> {
+        let mut guard = self.compaction.write().await;
+        let deterministic_mode = self.env.opts.compaction_deterministic_test;
+        let compaction: &mut Compaction = &mut guard;
+        let start_time = Instant::now();
+        let original_keys = compaction.compaction_statuses.keys().cloned().collect_vec();
+        let mut compact_statuses = BTreeMapTransaction::new(&mut compaction.compaction_statuses);
+        let mut rets = vec![false; report_tasks.len()];
+        let mut compact_task_assignment =
+            BTreeMapTransaction::new(&mut compaction.compact_task_assignment);
+        // The compaction task is finished.
+        let mut versioning_guard = self.versioning.write().await;
+        let versioning: &mut Versioning = &mut versioning_guard;
+        let _timer = start_measure_real_process_timer!(self, "report_compact_tasks");
+
+        // purge stale compact_status
+        for group_id in original_keys {
+            if !versioning.current_version.levels.contains_key(&group_id) {
+                compact_statuses.remove(group_id);
+            }
+        }
+        let mut tasks = vec![];
+
+        let mut version = HummockVersionTransaction::new(
+            &mut versioning.current_version,
+            &mut versioning.hummock_version_deltas,
+            self.env.notification_manager(),
+            &self.metrics,
+        );
+
+        if deterministic_mode {
+            version.disable_apply_to_txn();
+        }
+
+        let mut version_stats = HummockVersionStatsTransaction::new(
+            &mut versioning.version_stats,
+            self.env.notification_manager(),
+        );
+        let mut success_count = 0;
+        for (idx, task) in report_tasks.into_iter().enumerate() {
+            rets[idx] = true;
+            let mut compact_task = match compact_task_assignment.remove(task.task_id) {
+                Some(compact_task) => compact_task.compact_task.unwrap(),
+                None => {
+                    tracing::warn!("{}", format!("compact task {} not found", task.task_id));
+                    rets[idx] = false;
+                    continue;
+                }
+            };
+
+            {
+                // apply result
+                compact_task.task_status = task.task_status;
+                compact_task.sorted_output_ssts = task.sorted_output_ssts;
+            }
+
+            match compact_statuses.get_mut(compact_task.compaction_group_id) {
+                Some(mut compact_status) => {
+                    compact_status.report_compact_task(&compact_task);
+                }
+                None => {
+                    compact_task.set_task_status(TaskStatus::InvalidGroupCanceled);
+                }
+            }
+
+            let input_sst_ids: HashSet<u64> = compact_task
+                .input_ssts
+                .iter()
+                .flat_map(|level| level.table_infos.iter().map(|sst| sst.sst_id))
+                .collect();
+            let input_level_ids: Vec<u32> = compact_task
+                .input_ssts
+                .iter()
+                .map(|level| level.level_idx)
+                .collect();
+            let is_success = if let TaskStatus::Success = compact_task.task_status() {
+                // if member_table_ids changes, the data of sstable may stale.
+                let is_expired =
+                    Self::is_compact_task_expired(&compact_task, version.latest_version());
+                if is_expired {
+                    compact_task.set_task_status(TaskStatus::InputOutdatedCanceled);
+                    false
+                } else {
+                    let group = version
+                        .latest_version()
+                        .levels
+                        .get(&compact_task.compaction_group_id)
+                        .unwrap();
+                    let input_exist =
+                        group.check_deleted_sst_exist(&input_level_ids, input_sst_ids);
+                    if !input_exist {
+                        compact_task.set_task_status(TaskStatus::InvalidGroupCanceled);
+                        warn!(
+                            "The task may be expired because of group split, task:\n {:?}",
+                            compact_task_to_string(&compact_task)
+                        );
+                    }
+                    input_exist
+                }
+            } else {
+                false
+            };
+            if is_success {
+                success_count += 1;
+                version.apply_compact_task(&compact_task);
+                if purge_prost_table_stats(&mut version_stats.table_stats, version.latest_version())
+                {
+                    self.metrics.version_stats.reset();
+                    versioning.local_metrics.clear();
+                }
+                add_prost_table_stats_map(&mut version_stats.table_stats, &task.table_stats_change);
+                trigger_local_table_stat(
+                    &self.metrics,
+                    &mut versioning.local_metrics,
+                    &version_stats,
+                    &task.table_stats_change,
+                );
+            }
+            tasks.push(compact_task);
+        }
+        if success_count > 0 {
+            commit_multi_var!(
+                self.meta_store_ref(),
+                compact_statuses,
+                compact_task_assignment,
+                version,
+                version_stats
+            )?;
+
+            self.metrics
+                .compact_task_batch_count
+                .with_label_values(&["batch_report_task"])
+                .observe(success_count as f64);
+        } else {
+            // The compaction task is cancelled or failed.
+            commit_multi_var!(
+                self.meta_store_ref(),
+                compact_statuses,
+                compact_task_assignment
+            )?;
+        }
+        let mut success_groups = vec![];
+        for compact_task in tasks {
+            let task_status = compact_task.task_status();
+            let task_status_label = task_status.as_str_name();
+            let task_type_label = compact_task.task_type().as_str_name();
+
+            self.compactor_manager
+                .remove_task_heartbeat(compact_task.task_id);
+
+            self.metrics
+                .compact_frequency
+                .with_label_values(&[
+                    "normal",
+                    &compact_task.compaction_group_id.to_string(),
+                    task_type_label,
+                    task_status_label,
+                ])
+                .inc();
+
+            tracing::trace!(
+                "Reported compaction task. {}. cost time: {:?}",
+                compact_task_to_string(&compact_task),
+                start_time.elapsed(),
+            );
+
+            trigger_sst_stat(
+                &self.metrics,
+                compaction
+                    .compaction_statuses
+                    .get(&compact_task.compaction_group_id),
+                &versioning_guard.current_version,
+                compact_task.compaction_group_id,
+            );
+
+            if !deterministic_mode
+                && (matches!(compact_task.task_type(), compact_task::TaskType::Dynamic)
+                    || matches!(compact_task.task_type(), compact_task::TaskType::Emergency))
+            {
+                // only try send Dynamic compaction
+                self.try_send_compaction_request(
+                    compact_task.compaction_group_id,
+                    compact_task::TaskType::Dynamic,
+                );
+            }
+
+            if task_status == TaskStatus::Success {
+                success_groups.push(compact_task.compaction_group_id);
+            }
+        }
+        drop(versioning_guard);
+        if !success_groups.is_empty() {
+            self.try_update_write_limits(&success_groups).await;
+        }
+        Ok(rets)
+    }
+
+    /// Triggers compacitons to specified compaction groups.
+    /// Don't wait for compaction finish
+    pub async fn trigger_compaction_deterministic(
+        &self,
+        _base_version_id: HummockVersionId,
+        compaction_groups: Vec<CompactionGroupId>,
+    ) -> Result<()> {
+        let old_version = self.get_current_version().await;
+        tracing::info!(
+            "Trigger compaction for version {}, epoch {}, groups {:?}",
+            old_version.id,
+            old_version.max_committed_epoch,
+            compaction_groups
+        );
+
+        if compaction_groups.is_empty() {
+            return Ok(());
+        }
+        for compaction_group in compaction_groups {
+            self.try_send_compaction_request(compaction_group, compact_task::TaskType::Dynamic);
+        }
+        Ok(())
+    }
+
+    pub async fn trigger_manual_compaction(
+        &self,
+        compaction_group: CompactionGroupId,
+        manual_compaction_option: ManualCompactionOption,
+    ) -> Result<()> {
+        let start_time = Instant::now();
+
+        // 1. Get idle compactor.
+        let compactor = match self.compactor_manager.next_compactor() {
+            Some(compactor) => compactor,
+            None => {
+                tracing::warn!("trigger_manual_compaction No compactor is available.");
+                return Err(anyhow::anyhow!(
+                    "trigger_manual_compaction No compactor is available. compaction_group {}",
+                    compaction_group
+                )
+                .into());
+            }
+        };
+
+        // 2. Get manual compaction task.
+        let compact_task = self
+            .manual_get_compact_task(compaction_group, manual_compaction_option)
+            .await;
+        let compact_task = match compact_task {
+            Ok(Some(compact_task)) => compact_task,
+            Ok(None) => {
+                // No compaction task available.
+                return Err(anyhow::anyhow!(
+                    "trigger_manual_compaction No compaction_task is available. compaction_group {}",
+                    compaction_group
+                )
+                    .into());
+            }
+            Err(err) => {
+                tracing::warn!(error = %err.as_report(), "Failed to get compaction task");
+
+                return Err(anyhow::anyhow!(err)
+                    .context(format!(
+                        "Failed to get compaction task for compaction_group {}",
+                        compaction_group,
+                    ))
+                    .into());
+            }
+        };
+
+        // 3. send task to compactor
+        let compact_task_string = compact_task_to_string(&compact_task);
+        // TODO: shall we need to cancel on meta ?
+        compactor
+            .send_event(ResponseEvent::CompactTask(compact_task))
+            .with_context(|| {
+                format!(
+                    "Failed to trigger compaction task for compaction_group {}",
+                    compaction_group,
+                )
+            })?;
+
+        tracing::info!(
+            "Trigger manual compaction task. {}. cost time: {:?}",
+            &compact_task_string,
+            start_time.elapsed(),
+        );
+
+        Ok(())
+    }
+
+    /// Sends a compaction request.
+    pub fn try_send_compaction_request(
+        &self,
+        compaction_group: CompactionGroupId,
+        task_type: compact_task::TaskType,
+    ) -> bool {
+        match self
+            .compaction_state
+            .try_sched_compaction(compaction_group, task_type)
+        {
+            Ok(_) => true,
+            Err(e) => {
+                tracing::error!(
+                    error = %e.as_report(),
+                    "failed to send compaction request for compaction group {}",
+                    compaction_group,
+                );
+                false
+            }
+        }
+    }
+
+    pub(crate) async fn calculate_vnode_partition(
+        &self,
+        compact_task: &mut CompactTask,
+        compaction_config: &CompactionConfig,
+    ) {
+        // do not split sst by vnode partition when target_level > base_level
+        // The purpose of data alignment is mainly to improve the parallelism of base level compaction and reduce write amplification.
+        // However, at high level, the size of the sst file is often larger and only contains the data of a single table_id, so there is no need to cut it.
+        if compact_task.target_level > compact_task.base_level {
+            return;
+        }
+        if compaction_config.split_weight_by_vnode > 0 {
+            for table_id in &compact_task.existing_table_ids {
+                compact_task
+                    .table_vnode_partition
+                    .insert(*table_id, compact_task.split_weight_by_vnode);
+            }
+        } else {
+            let mut table_size_info: HashMap<u32, u64> = HashMap::default();
+            let mut existing_table_ids: HashSet<u32> = HashSet::default();
+            for input_ssts in &compact_task.input_ssts {
+                for sst in &input_ssts.table_infos {
+                    existing_table_ids.extend(sst.table_ids.iter());
+                    for table_id in &sst.table_ids {
+                        *table_size_info.entry(*table_id).or_default() +=
+                            sst.file_size / (sst.table_ids.len() as u64);
+                    }
+                }
+            }
+            compact_task
+                .existing_table_ids
+                .retain(|table_id| existing_table_ids.contains(table_id));
+
+            let hybrid_vnode_count = self.env.opts.hybrid_partition_node_count;
+            let default_partition_count = self.env.opts.partition_vnode_count;
+            // We must ensure the partition threshold large enough to avoid too many small files.
+            let compact_task_table_size_partition_threshold_low = self
+                .env
+                .opts
+                .compact_task_table_size_partition_threshold_low;
+            let compact_task_table_size_partition_threshold_high = self
+                .env
+                .opts
+                .compact_task_table_size_partition_threshold_high;
+            use risingwave_common::system_param::reader::SystemParamsRead;
+            let params = self.env.system_params_reader().await;
+            let barrier_interval_ms = params.barrier_interval_ms() as u64;
+            let checkpoint_secs = std::cmp::max(
+                1,
+                params.checkpoint_frequency() * barrier_interval_ms / 1000,
+            );
+            // check latest write throughput
+            let history_table_throughput = self.history_table_throughput.read();
+            for (table_id, compact_table_size) in table_size_info {
+                let write_throughput = history_table_throughput
+                    .get(&table_id)
+                    .map(|que| que.back().cloned().unwrap_or(0))
+                    .unwrap_or(0)
+                    / checkpoint_secs;
+                if compact_table_size > compact_task_table_size_partition_threshold_high
+                    && default_partition_count > 0
+                {
+                    compact_task
+                        .table_vnode_partition
+                        .insert(table_id, default_partition_count);
+                } else if (compact_table_size > compact_task_table_size_partition_threshold_low
+                    || (write_throughput > self.env.opts.table_write_throughput_threshold
+                        && compact_table_size > compaction_config.target_file_size_base))
+                    && hybrid_vnode_count > 0
+                {
+                    // partition for large write throughput table. But we also need to make sure that it can not be too small.
+                    compact_task
+                        .table_vnode_partition
+                        .insert(table_id, hybrid_vnode_count);
+                } else if compact_table_size > compaction_config.target_file_size_base {
+                    // partition for small table
+                    compact_task.table_vnode_partition.insert(table_id, 1);
+                }
+            }
+            compact_task
+                .table_vnode_partition
+                .retain(|table_id, _| compact_task.existing_table_ids.contains(table_id));
+        }
+    }
+}
+
+#[cfg(any(test, feature = "test"))]
+impl HummockManager {
+    pub fn compactor_manager_ref_for_test(&self) -> crate::hummock::CompactorManagerRef {
+        self.compactor_manager.clone()
+    }
+
+    pub async fn compaction_task_from_assignment_for_test(
+        &self,
+        task_id: u64,
+    ) -> Option<CompactTaskAssignment> {
+        let compaction_guard = self.compaction.read().await;
+        let assignment_ref = &compaction_guard.compact_task_assignment;
+        assignment_ref.get(&task_id).cloned()
+    }
+
+    pub async fn report_compact_task_for_test(
+        &self,
+        task_id: u64,
+        compact_task: Option<CompactTask>,
+        task_status: TaskStatus,
+        sorted_output_ssts: Vec<SstableInfo>,
+        table_stats_change: Option<PbTableStatsMap>,
+    ) -> Result<()> {
+        if let Some(task) = compact_task {
+            let mut guard = self.compaction.write().await;
+            guard.compact_task_assignment.insert(
+                task_id,
+                CompactTaskAssignment {
+                    compact_task: Some(task),
+                    context_id: 0,
+                },
+            );
+        }
+
+        // In the test, the contents of the compact task may have been modified directly, while the contents of compact_task_assignment were not modified.
+        // So we pass the modified compact_task directly into the `report_compact_task_impl`
+        self.report_compact_tasks(vec![ReportTask {
+            task_id,
+            task_status: task_status as i32,
+            sorted_output_ssts,
+            table_stats_change: table_stats_change.unwrap_or_default(),
+        }])
+        .await?;
+        Ok(())
+    }
+}
+
 pub fn check_cg_write_limit(
     levels: &Levels,
     compaction_config: &CompactionConfig,
@@ -298,3 +1643,54 @@ impl WriteLimitType {
         matches!(self, Self::WriteStop(_, _))
     }
 }
+
+#[derive(Debug, Default)]
+pub struct CompactionState {
+    scheduled: Mutex<HashSet<(CompactionGroupId, compact_task::TaskType)>>,
+}
+
+impl CompactionState {
+    pub fn new() -> Self {
+        Self {
+            scheduled: Default::default(),
+        }
+    }
+
+    /// Enqueues only if the target is not yet in queue.
+    pub fn try_sched_compaction(
+        &self,
+        compaction_group: CompactionGroupId,
+        task_type: TaskType,
+    ) -> std::result::Result<bool, SendError<CompactionRequestChannelItem>> {
+        let mut guard = self.scheduled.lock();
+        let key = (compaction_group, task_type);
+        if guard.contains(&key) {
+            return Ok(false);
+        }
+        guard.insert(key);
+        Ok(true)
+    }
+
+    pub fn unschedule(
+        &self,
+        compaction_group: CompactionGroupId,
+        task_type: compact_task::TaskType,
+    ) {
+        self.scheduled.lock().remove(&(compaction_group, task_type));
+    }
+
+    pub fn auto_pick_type(&self, group: CompactionGroupId) -> Option<TaskType> {
+        let guard = self.scheduled.lock();
+        if guard.contains(&(group, compact_task::TaskType::Dynamic)) {
+            Some(compact_task::TaskType::Dynamic)
+        } else if guard.contains(&(group, compact_task::TaskType::SpaceReclaim)) {
+            Some(compact_task::TaskType::SpaceReclaim)
+        } else if guard.contains(&(group, compact_task::TaskType::Ttl)) {
+            Some(compact_task::TaskType::Ttl)
+        } else if guard.contains(&(group, compact_task::TaskType::Tombstone)) {
+            Some(compact_task::TaskType::Tombstone)
+        } else {
+            None
+        }
+    }
+}
diff --git a/src/meta/src/hummock/manager/compaction_group_manager.rs b/src/meta/src/hummock/manager/compaction_group_manager.rs
index 18bf9b263e734..79596c8a3e774 100644
--- a/src/meta/src/hummock/manager/compaction_group_manager.rs
+++ b/src/meta/src/hummock/manager/compaction_group_manager.rs
@@ -16,12 +16,10 @@ use std::collections::{BTreeMap, HashMap, HashSet};
 use std::ops::DerefMut;
 use std::sync::Arc;
 
-use function_name::named;
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
 use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
-    build_version_delta_after_version, get_compaction_group_ids, get_compaction_group_ssts,
-    get_member_table_ids, try_get_compaction_group_id_by_table_id, TableGroupInfo,
+    get_compaction_group_ids, get_member_table_ids, TableGroupInfo,
 };
 use risingwave_hummock_sdk::compaction_group::{StateTableId, StaticCompactionGroupId};
 use risingwave_hummock_sdk::CompactionGroupId;
@@ -31,60 +29,82 @@ use risingwave_pb::hummock::group_delta::DeltaType;
 use risingwave_pb::hummock::hummock_version_delta::GroupDeltas;
 use risingwave_pb::hummock::rise_ctl_update_compaction_config_request::mutable_config::MutableConfig;
 use risingwave_pb::hummock::subscribe_compaction_event_request::ReportTask;
+use risingwave_pb::hummock::write_limits::WriteLimit;
 use risingwave_pb::hummock::{
     compact_task, CompactionConfig, CompactionGroupInfo, CompatibilityVersion, GroupConstruct,
-    GroupDelta, GroupDestroy, GroupMetaChange, GroupTableChange,
+    GroupDelta, GroupDestroy, StateTableInfoDelta,
 };
-use thiserror_ext::AsReport;
-use tokio::sync::{OnceCell, RwLock};
+use tokio::sync::OnceCell;
 
-use super::write_lock;
 use crate::hummock::compaction::compaction_config::{
     validate_compaction_config, CompactionConfigBuilder,
 };
 use crate::hummock::error::{Error, Result};
-use crate::hummock::manager::{
-    commit_multi_var, create_trx_wrapper, drop_sst, read_lock, HummockManager,
-};
+use crate::hummock::manager::transaction::HummockVersionTransaction;
+use crate::hummock::manager::versioning::Versioning;
+use crate::hummock::manager::{commit_multi_var, HummockManager};
 use crate::hummock::metrics_utils::remove_compaction_group_in_sst_stat;
 use crate::hummock::model::CompactionGroup;
 use crate::hummock::sequence::{next_compaction_group_id, next_sstable_object_id};
 use crate::manager::{MetaSrvEnv, MetaStoreImpl};
-use crate::model::{
-    BTreeMapEntryTransaction, BTreeMapEntryTransactionWrapper, BTreeMapTransaction,
-    BTreeMapTransactionWrapper, MetadataModel, MetadataModelError, ValTransaction,
-};
-use crate::storage::MetaStore;
+use crate::model::{BTreeMapTransaction, MetadataModel, MetadataModelError};
 
-impl HummockManager {
-    pub(super) async fn build_compaction_group_manager(
-        env: &MetaSrvEnv,
-    ) -> Result<RwLock<CompactionGroupManager>> {
+type CompactionGroupTransaction<'a> = BTreeMapTransaction<'a, CompactionGroupId, CompactionGroup>;
+
+impl CompactionGroupManager {
+    pub(super) async fn new(env: &MetaSrvEnv) -> Result<CompactionGroupManager> {
         let default_config = match env.opts.compaction_config.as_ref() {
             None => CompactionConfigBuilder::new().build(),
             Some(opt) => CompactionConfigBuilder::with_opt(opt).build(),
         };
-        Self::build_compaction_group_manager_with_config(env, default_config).await
+        Self::new_with_config(env, default_config).await
     }
 
-    pub(super) async fn build_compaction_group_manager_with_config(
+    pub(super) async fn new_with_config(
         env: &MetaSrvEnv,
         default_config: CompactionConfig,
-    ) -> Result<RwLock<CompactionGroupManager>> {
-        let compaction_group_manager = RwLock::new(CompactionGroupManager {
+    ) -> Result<CompactionGroupManager> {
+        let mut compaction_group_manager = CompactionGroupManager {
             compaction_groups: BTreeMap::new(),
-            default_config,
-            meta_store_impl: env.meta_store_ref(),
-        });
-        compaction_group_manager.write().await.init().await?;
+            default_config: Arc::new(default_config),
+            write_limit: Default::default(),
+        };
+
+        let loaded_compaction_groups: BTreeMap<CompactionGroupId, CompactionGroup> =
+            match env.meta_store_ref() {
+                MetaStoreImpl::Kv(meta_store) => CompactionGroup::list(meta_store)
+                    .await?
+                    .into_iter()
+                    .map(|cg| (cg.group_id(), cg))
+                    .collect(),
+                MetaStoreImpl::Sql(sql_meta_store) => {
+                    use sea_orm::EntityTrait;
+                    compaction_config::Entity::find()
+                        .all(&sql_meta_store.conn)
+                        .await
+                        .map_err(MetadataModelError::from)?
+                        .into_iter()
+                        .map(|m| (m.compaction_group_id as CompactionGroupId, m.into()))
+                        .collect()
+                }
+            };
+
+        compaction_group_manager.init(loaded_compaction_groups);
         Ok(compaction_group_manager)
     }
 
+    fn init(&mut self, loaded_compaction_groups: BTreeMap<CompactionGroupId, CompactionGroup>) {
+        if !loaded_compaction_groups.is_empty() {
+            self.compaction_groups = loaded_compaction_groups;
+        }
+    }
+}
+
+impl HummockManager {
     /// Should not be called inside [`HummockManager`], because it requests locks internally.
     /// The implementation acquires `versioning` lock.
-    #[named]
     pub async fn compaction_group_ids(&self) -> Vec<CompactionGroupId> {
-        get_compaction_group_ids(&read_lock!(self, versioning).await.current_version).collect_vec()
+        get_compaction_group_ids(&self.versioning.read().await.current_version).collect_vec()
     }
 
     /// The implementation acquires `compaction_group_manager` lock.
@@ -121,7 +141,7 @@ impl HummockManager {
                 CompactionGroupId::from(StaticCompactionGroupId::StateDefault),
             ));
         }
-        self.register_table_ids(&pairs).await?;
+        self.register_table_ids_for_test(&pairs).await?;
         Ok(pairs.iter().map(|(table_id, ..)| *table_id).collect_vec())
     }
 
@@ -131,22 +151,22 @@ impl HummockManager {
         &self,
         table_fragments: &[crate::model::TableFragments],
     ) {
-        self.unregister_table_ids_fail_fast(
+        self.unregister_table_ids(
             &table_fragments
                 .iter()
                 .flat_map(|t| t.all_table_ids())
                 .collect_vec(),
         )
-        .await;
+        .await
+        .unwrap();
     }
 
     /// Unregisters stale members and groups
     /// The caller should ensure `table_fragments_list` remain unchanged during `purge`.
     /// Currently `purge` is only called during meta service start ups.
-    #[named]
     pub async fn purge(&self, valid_ids: &[u32]) -> Result<()> {
         let registered_members =
-            get_member_table_ids(&read_lock!(self, versioning).await.current_version);
+            get_member_table_ids(&self.versioning.read().await.current_version);
         let to_unregister = registered_members
             .into_iter()
             .filter(|table_id| !valid_ids.contains(table_id))
@@ -157,39 +177,45 @@ impl HummockManager {
     }
 
     /// The implementation acquires `versioning` lock.
-    #[named]
-    pub async fn register_table_ids(
+    ///
+    /// The method name is temporarily added with a `_for_test` prefix to mark
+    /// that it's currently only used in test.
+    pub async fn register_table_ids_for_test(
         &self,
         pairs: &[(StateTableId, CompactionGroupId)],
     ) -> Result<()> {
         if pairs.is_empty() {
             return Ok(());
         }
-        let mut versioning_guard = write_lock!(self, versioning).await;
+        let mut versioning_guard = self.versioning.write().await;
         let versioning = versioning_guard.deref_mut();
+        let mut compaction_group_manager = self.compaction_group_manager.write().await;
         let current_version = &versioning.current_version;
+        let default_config = compaction_group_manager.default_compaction_config();
+        let mut compaction_groups_txn = compaction_group_manager.start_compaction_groups_txn();
 
         for (table_id, _) in pairs {
-            if let Some(old_group) =
-                try_get_compaction_group_id_by_table_id(current_version, *table_id)
+            if let Some(info) = current_version
+                .state_table_info
+                .info()
+                .get(&TableId::new(*table_id))
             {
                 return Err(Error::CompactionGroup(format!(
-                    "table {} already in group {}",
-                    *table_id, old_group
+                    "table {} already {:?}",
+                    *table_id, info
                 )));
             }
         }
         // All NewCompactionGroup pairs are mapped to one new compaction group.
         let new_compaction_group_id: OnceCell<CompactionGroupId> = OnceCell::new();
-        let mut new_version_delta = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapEntryTransactionWrapper,
-            BTreeMapEntryTransaction::new_insert(
-                &mut versioning.hummock_version_deltas,
-                current_version.id + 1,
-                build_version_delta_after_version(current_version),
-            )
+        let mut version = HummockVersionTransaction::new(
+            &mut versioning.current_version,
+            &mut versioning.hummock_version_deltas,
+            self.env.notification_manager(),
+            &self.metrics,
         );
+        let mut new_version_delta = version.new_delta();
+        let epoch = new_version_delta.latest_version().max_committed_epoch;
 
         for (table_id, raw_group_id) in pairs {
             let mut group_id = *raw_group_id;
@@ -211,15 +237,17 @@ impl HummockManager {
                         .entry(group_id)
                         .or_default()
                         .group_deltas;
-                    let config = self
-                        .compaction_group_manager
-                        .write()
-                        .await
-                        .get_or_insert_compaction_group_config(group_id)
-                        .await?
-                        .compaction_config
-                        .as_ref()
-                        .clone();
+
+                    let config =
+                        match compaction_groups_txn.try_get_compaction_group_config(group_id) {
+                            Some(config) => config.compaction_config.as_ref().clone(),
+                            None => {
+                                compaction_groups_txn
+                                    .create_compaction_groups(group_id, default_config.clone());
+                                default_config.as_ref().clone()
+                            }
+                        };
+
                     group_deltas.push(GroupDelta {
                         delta_type: Some(DeltaType::GroupConstruct(GroupConstruct {
                             group_config: Some(config),
@@ -229,104 +257,85 @@ impl HummockManager {
                     });
                 }
             }
-            let group_deltas = &mut new_version_delta
-                .group_deltas
-                .entry(group_id)
-                .or_default()
-                .group_deltas;
-            group_deltas.push(GroupDelta {
-                delta_type: Some(DeltaType::GroupMetaChange(GroupMetaChange {
-                    table_ids_add: vec![*table_id],
-                    ..Default::default()
-                })),
-            });
+            assert!(new_version_delta
+                .state_table_info_delta
+                .insert(
+                    TableId::new(*table_id),
+                    StateTableInfoDelta {
+                        committed_epoch: epoch,
+                        safe_epoch: epoch,
+                        compaction_group_id: *raw_group_id,
+                    }
+                )
+                .is_none());
         }
-        let mut current_version = versioning.current_version.clone();
-        let sst_split_info = current_version.apply_version_delta(&new_version_delta);
-        assert!(sst_split_info.is_empty());
-        commit_multi_var!(self.meta_store_ref(), new_version_delta)?;
-        versioning.current_version = current_version;
-
-        self.notify_last_version_delta(versioning);
+        new_version_delta.pre_apply();
+        commit_multi_var!(self.meta_store_ref(), version, compaction_groups_txn)?;
 
         Ok(())
     }
 
-    #[named]
     pub async fn unregister_table_ids(&self, table_ids: &[StateTableId]) -> Result<()> {
         if table_ids.is_empty() {
             return Ok(());
         }
-        let mut versioning_guard = write_lock!(self, versioning).await;
+        let mut versioning_guard = self.versioning.write().await;
         let versioning = versioning_guard.deref_mut();
-        let current_version = &versioning.current_version;
-        let mut new_version_delta = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapEntryTransactionWrapper,
-            BTreeMapEntryTransaction::new_insert(
-                &mut versioning.hummock_version_deltas,
-                current_version.id + 1,
-                build_version_delta_after_version(current_version),
-            )
+        let mut version = HummockVersionTransaction::new(
+            &mut versioning.current_version,
+            &mut versioning.hummock_version_deltas,
+            self.env.notification_manager(),
+            &self.metrics,
         );
+        let mut new_version_delta = version.new_delta();
         let mut modified_groups: HashMap<CompactionGroupId, /* #member table */ u64> =
             HashMap::new();
         // Remove member tables
         for table_id in table_ids.iter().unique() {
-            let group_id = match try_get_compaction_group_id_by_table_id(current_version, *table_id)
-            {
-                Some(group_id) => group_id,
-                None => continue,
+            let version = new_version_delta.latest_version();
+            let Some(info) = version
+                .state_table_info
+                .info()
+                .get(&TableId::new(*table_id))
+            else {
+                continue;
             };
-            let group_deltas = &mut new_version_delta
-                .group_deltas
-                .entry(group_id)
-                .or_default()
-                .group_deltas;
-            group_deltas.push(GroupDelta {
-                delta_type: Some(DeltaType::GroupMetaChange(GroupMetaChange {
-                    table_ids_remove: vec![*table_id],
-                    ..Default::default()
-                })),
-            });
+
             modified_groups
-                .entry(group_id)
+                .entry(info.compaction_group_id)
                 .and_modify(|count| *count -= 1)
                 .or_insert(
-                    current_version
-                        .get_compaction_group_levels(group_id)
-                        .member_table_ids
+                    version
+                        .state_table_info
+                        .compaction_group_member_tables()
+                        .get(&info.compaction_group_id)
+                        .expect("should exist")
                         .len() as u64
                         - 1,
                 );
             new_version_delta
                 .removed_table_ids
-                .push(TableId::new(*table_id));
+                .insert(TableId::new(*table_id));
         }
 
-        // Remove empty group, GC SSTs and remove metric.
-        let mut branched_ssts = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning.branched_ssts)
-        );
         let groups_to_remove = modified_groups
             .into_iter()
             .filter_map(|(group_id, member_count)| {
                 if member_count == 0 && group_id > StaticCompactionGroupId::End as CompactionGroupId
                 {
-                    return Some(group_id);
+                    return Some((
+                        group_id,
+                        new_version_delta
+                            .latest_version()
+                            .get_compaction_group_levels(group_id)
+                            .get_levels()
+                            .len(),
+                    ));
                 }
                 None
             })
             .collect_vec();
-        for group_id in &groups_to_remove {
-            // We don't bother to add IntraLevelDelta to remove SSTs from group, because the entire
-            // group is to be removed.
-            // However, we need to take care of SST GC for the removed group.
-            for (object_id, sst_id) in get_compaction_group_ssts(current_version, *group_id) {
-                drop_sst(&mut branched_ssts, *group_id, object_id, sst_id);
-            }
+        for (group_id, _) in &groups_to_remove {
             let group_deltas = &mut new_version_delta
                 .group_deltas
                 .entry(*group_id)
@@ -336,91 +345,81 @@ impl HummockManager {
                 delta_type: Some(DeltaType::GroupDestroy(GroupDestroy {})),
             });
         }
-        let mut current_version = versioning.current_version.clone();
-        let sst_split_info = current_version.apply_version_delta(&new_version_delta);
-        assert!(sst_split_info.is_empty());
-        commit_multi_var!(self.meta_store_ref(), new_version_delta)?;
 
-        for group_id in &groups_to_remove {
-            let max_level = versioning
-                .current_version
-                .get_compaction_group_levels(*group_id)
-                .get_levels()
-                .len();
-            remove_compaction_group_in_sst_stat(&self.metrics, *group_id, max_level);
+        for (group_id, max_level) in groups_to_remove {
+            remove_compaction_group_in_sst_stat(&self.metrics, group_id, max_level);
         }
-        versioning.current_version = current_version;
-        branched_ssts.commit_memory();
 
-        self.notify_last_version_delta(versioning);
+        new_version_delta.pre_apply();
 
         // Purge may cause write to meta store. If it hurts performance while holding versioning
         // lock, consider to make it in batch.
-        self.compaction_group_manager
-            .write()
-            .await
-            .purge(HashSet::from_iter(get_compaction_group_ids(
-                &versioning.current_version,
-            )))
-            .await?;
-        Ok(())
-    }
+        let mut compaction_group_manager = self.compaction_group_manager.write().await;
+        let mut compaction_groups_txn = compaction_group_manager.start_compaction_groups_txn();
 
-    /// The implementation acquires `versioning` lock and `compaction_group_manager` lock.
-    pub async fn unregister_table_ids_fail_fast(&self, table_ids: &[StateTableId]) {
-        self.unregister_table_ids(table_ids)
-            .await
-            .unwrap_or_else(|e| {
-                panic!("unregister table ids fail: {table_ids:?} {}", e.as_report())
-            });
+        compaction_groups_txn.purge(HashSet::from_iter(get_compaction_group_ids(
+            version.latest_version(),
+        )));
+        commit_multi_var!(self.meta_store_ref(), version, compaction_groups_txn)?;
+
+        Ok(())
     }
 
     pub async fn update_compaction_config(
         &self,
         compaction_group_ids: &[CompactionGroupId],
         config_to_update: &[MutableConfig],
-    ) -> Result<Vec<CompactionGroup>> {
-        let result = self
-            .compaction_group_manager
-            .write()
-            .await
-            .update_compaction_config(compaction_group_ids, config_to_update)
-            .await?;
+    ) -> Result<()> {
+        {
+            // Avoid lock conflicts with `try_update_write_limits``
+            let mut compaction_group_manager = self.compaction_group_manager.write().await;
+            let mut compaction_groups_txn = compaction_group_manager.start_compaction_groups_txn();
+            compaction_groups_txn
+                .update_compaction_config(compaction_group_ids, config_to_update)?;
+            commit_multi_var!(self.meta_store_ref(), compaction_groups_txn)?;
+        }
+
         if config_to_update
             .iter()
             .any(|c| matches!(c, MutableConfig::Level0StopWriteThresholdSubLevelNumber(_)))
         {
+            // Update write limits with lock
             self.try_update_write_limits(compaction_group_ids).await;
         }
 
-        Ok(result)
+        Ok(())
     }
 
     /// Gets complete compaction group info.
     /// It is the aggregate of `HummockVersion` and `CompactionGroupConfig`
-    #[named]
     pub async fn list_compaction_group(&self) -> Vec<CompactionGroupInfo> {
-        let mut versioning_guard = write_lock!(self, versioning).await;
+        let mut versioning_guard = self.versioning.write().await;
         let versioning = versioning_guard.deref_mut();
         let current_version = &versioning.current_version;
-        let mut compaction_groups = vec![];
+        let mut results = vec![];
+        let compaction_group_manager = self.compaction_group_manager.read().await;
+
         for levels in current_version.levels.values() {
-            let config = self
-                .compaction_group_manager
-                .read()
-                .await
+            let compaction_config = compaction_group_manager
                 .try_get_compaction_group_config(levels.group_id)
                 .unwrap()
-                .compaction_config;
+                .compaction_config
+                .as_ref()
+                .clone();
             let group = CompactionGroupInfo {
                 id: levels.group_id,
                 parent_id: levels.parent_group_id,
-                member_table_ids: levels.member_table_ids.clone(),
-                compaction_config: Some(config.as_ref().clone()),
+                member_table_ids: current_version
+                    .state_table_info
+                    .compaction_group_member_table_ids(levels.group_id)
+                    .iter()
+                    .map(|table_id| table_id.table_id)
+                    .collect_vec(),
+                compaction_config: Some(compaction_config),
             };
-            compaction_groups.push(group);
+            results.push(group);
         }
-        compaction_groups
+        results
     }
 
     /// Splits a compaction group into two. The new one will contain `table_ids`.
@@ -434,44 +433,47 @@ impl HummockManager {
             .move_state_table_to_compaction_group(
                 parent_group_id,
                 table_ids,
-                None,
                 self.env.opts.partition_vnode_count,
             )
             .await?;
-        self.group_to_table_vnode_partition
-            .write()
-            .insert(result.0, result.1);
 
-        Ok(result.0)
+        Ok(result)
     }
 
     /// move some table to another compaction-group. Create a new compaction group if it does not
     /// exist.
-    /// TODO: Move table_to_partition in result to compaction group
-    #[named]
     pub async fn move_state_table_to_compaction_group(
         &self,
         parent_group_id: CompactionGroupId,
         table_ids: &[StateTableId],
-        target_group_id: Option<CompactionGroupId>,
         partition_vnode_count: u32,
-    ) -> Result<(CompactionGroupId, BTreeMap<StateTableId, u32>)> {
-        let mut table_to_partition = BTreeMap::default();
+    ) -> Result<CompactionGroupId> {
         if table_ids.is_empty() {
-            return Ok((parent_group_id, table_to_partition));
+            return Ok(parent_group_id);
         }
         let table_ids = table_ids.iter().cloned().unique().collect_vec();
-        let compaction_guard = write_lock!(self, compaction).await;
-        let mut versioning_guard = write_lock!(self, versioning).await;
+        let compaction_guard = self.compaction.write().await;
+        let mut versioning_guard = self.versioning.write().await;
         let versioning = versioning_guard.deref_mut();
-        let current_version = &versioning.current_version;
         // Validate parameters.
-        let parent_group = current_version
+        if !versioning
+            .current_version
             .levels
-            .get(&parent_group_id)
-            .ok_or_else(|| Error::CompactionGroup(format!("invalid group {}", parent_group_id)))?;
+            .contains_key(&parent_group_id)
+        {
+            return Err(Error::CompactionGroup(format!(
+                "invalid group {}",
+                parent_group_id
+            )));
+        }
+
         for table_id in &table_ids {
-            if !parent_group.member_table_ids.contains(table_id) {
+            if !versioning
+                .current_version
+                .state_table_info
+                .compaction_group_member_table_ids(parent_group_id)
+                .contains(&TableId::new(*table_id))
+            {
                 return Err(Error::CompactionGroup(format!(
                     "table {} doesn't in group {}",
                     table_id, parent_group_id
@@ -479,79 +481,38 @@ impl HummockManager {
             }
         }
 
-        if table_ids.len() == parent_group.member_table_ids.len() {
+        if table_ids.len()
+            == versioning
+                .current_version
+                .state_table_info
+                .compaction_group_member_table_ids(parent_group_id)
+                .len()
+        {
             return Err(Error::CompactionGroup(format!(
                 "invalid split attempt for group {}: all member tables are moved",
                 parent_group_id
             )));
         }
-        if let Some(compaction_group_id) = target_group_id {
-            if !versioning.check_branched_sst_in_target_group(
-                &table_ids,
-                &parent_group_id,
-                &compaction_group_id,
-            ) {
-                return Err(Error::CompactionGroup(format!(
-                    "invalid split attempt for group {}: we shall wait some time for parent group and target group could compact stale sst files",
-                    parent_group_id
-                )));
-            }
-        }
-        let mut new_version_delta = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapEntryTransactionWrapper,
-            BTreeMapEntryTransaction::new_insert(
-                &mut versioning.hummock_version_deltas,
-                current_version.id + 1,
-                build_version_delta_after_version(current_version),
-            )
+        let mut version = HummockVersionTransaction::new(
+            &mut versioning.current_version,
+            &mut versioning.hummock_version_deltas,
+            self.env.notification_manager(),
+            &self.metrics,
         );
+        let mut new_version_delta = version.new_delta();
+
         let new_sst_start_id = next_sstable_object_id(
             &self.env,
-            current_version.count_new_ssts_in_group_split(
-                parent_group_id,
-                HashSet::from_iter(table_ids.clone()),
-            ),
+            new_version_delta
+                .latest_version()
+                .count_new_ssts_in_group_split(
+                    parent_group_id,
+                    HashSet::from_iter(table_ids.clone()),
+                ),
         )
         .await?;
-        let mut new_group = None;
-        let target_compaction_group_id = match target_group_id {
-            Some(compaction_group_id) => {
-                match current_version.levels.get(&compaction_group_id) {
-                    Some(group) => {
-                        for table_id in &table_ids {
-                            if group.member_table_ids.contains(table_id) {
-                                return Err(Error::CompactionGroup(format!(
-                                    "table {} already exist in group {}",
-                                    *table_id, compaction_group_id,
-                                )));
-                            }
-                        }
-                    }
-                    None => {
-                        return Err(Error::CompactionGroup(format!(
-                            "target group {} does not exist",
-                            compaction_group_id,
-                        )));
-                    }
-                }
-                let group_deltas = &mut new_version_delta
-                    .group_deltas
-                    .entry(compaction_group_id)
-                    .or_default()
-                    .group_deltas;
-                group_deltas.push(GroupDelta {
-                    delta_type: Some(DeltaType::GroupTableChange(GroupTableChange {
-                        table_ids: table_ids.to_vec(),
-                        origin_group_id: parent_group_id,
-                        target_group_id: compaction_group_id,
-                        new_sst_start_id,
-                        version: CompatibilityVersion::NoTrivialSplit as i32,
-                    })),
-                });
-                compaction_group_id
-            }
-            None => {
+        let (new_group, target_compaction_group_id) = {
+            {
                 // All NewCompactionGroup pairs are mapped to one new compaction group.
                 let new_compaction_group_id = next_compaction_group_id(&self.env).await?;
                 // The new config will be persisted later.
@@ -559,9 +520,13 @@ impl HummockManager {
                     .compaction_group_manager
                     .read()
                     .await
-                    .default_compaction_config();
+                    .default_compaction_config()
+                    .as_ref()
+                    .clone();
                 config.split_weight_by_vnode = partition_vnode_count;
 
+                #[expect(deprecated)]
+                // fill the deprecated field with default value
                 new_version_delta.group_deltas.insert(
                     new_compaction_group_id,
                     GroupDeltas {
@@ -571,92 +536,53 @@ impl HummockManager {
                                 group_id: new_compaction_group_id,
                                 parent_group_id,
                                 new_sst_start_id,
-                                table_ids: table_ids.to_vec(),
-                                version: CompatibilityVersion::NoTrivialSplit as i32,
+                                table_ids: vec![],
+                                version: CompatibilityVersion::NoMemberTableIds as i32,
                             })),
                         }],
                     },
                 );
-
-                new_group = Some((new_compaction_group_id, config));
-                new_version_delta.group_deltas.insert(
-                    parent_group_id,
-                    GroupDeltas {
-                        group_deltas: vec![GroupDelta {
-                            delta_type: Some(DeltaType::GroupMetaChange(GroupMetaChange {
-                                table_ids_remove: table_ids.to_vec(),
-                                ..Default::default()
-                            })),
-                        }],
-                    },
-                );
-                new_compaction_group_id
+                ((new_compaction_group_id, config), new_compaction_group_id)
             }
         };
 
-        let mut current_version = versioning.current_version.clone();
-        let sst_split_info = current_version.apply_version_delta(&new_version_delta);
-
-        // `branched_ssts` only commit in memory, so `TXN` make no difference.
-        let mut branched_ssts = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning.branched_ssts)
-        );
-        if let Some((new_compaction_group_id, config)) = new_group {
-            let mut compaction_group_manager = self.compaction_group_manager.write().await;
-            let insert = create_trx_wrapper!(
-                self.meta_store_ref(),
-                BTreeMapEntryTransactionWrapper,
-                BTreeMapEntryTransaction::new_insert(
-                    &mut compaction_group_manager.compaction_groups,
-                    new_compaction_group_id,
-                    CompactionGroup {
-                        group_id: new_compaction_group_id,
-                        compaction_config: Arc::new(config),
-                    },
-                )
-            );
-            commit_multi_var!(self.meta_store_ref(), new_version_delta, insert)?;
-            // Currently, only splitting out a single table_id is supported.
-            for table_id in table_ids {
-                table_to_partition.insert(table_id, partition_vnode_count);
-            }
-        } else {
-            commit_multi_var!(self.meta_store_ref(), new_version_delta)?;
-        }
-        versioning.current_version = current_version;
-        // Updates SST split info
-        let mut changed_sst_ids: HashSet<u64> = HashSet::default();
-        for (object_id, sst_id, parent_old_sst_id, parent_new_sst_id) in sst_split_info {
-            changed_sst_ids.insert(parent_old_sst_id);
-            match branched_ssts.get_mut(object_id) {
-                Some(mut entry) => {
-                    entry.insert(parent_group_id, parent_new_sst_id);
-                    entry.insert(target_compaction_group_id, sst_id);
-                }
-                None => {
-                    let mut groups = HashMap::from_iter([(target_compaction_group_id, sst_id)]);
-                    groups.insert(parent_group_id, parent_new_sst_id);
-                    branched_ssts.insert(object_id, groups);
-                }
+        let (new_compaction_group_id, config) = new_group;
+        new_version_delta.with_latest_version(|version, new_version_delta| {
+            for table_id in &table_ids {
+                let table_id = TableId::new(*table_id);
+                let info = version
+                    .state_table_info
+                    .info()
+                    .get(&table_id)
+                    .expect("have check exist previously");
+                assert!(new_version_delta
+                    .state_table_info_delta
+                    .insert(
+                        table_id,
+                        StateTableInfoDelta {
+                            committed_epoch: info.committed_epoch,
+                            safe_epoch: info.safe_epoch,
+                            compaction_group_id: new_compaction_group_id,
+                        }
+                    )
+                    .is_none());
             }
+        });
+        {
+            let mut compaction_group_manager = self.compaction_group_manager.write().await;
+            let mut compaction_groups_txn = compaction_group_manager.start_compaction_groups_txn();
+            compaction_groups_txn
+                .create_compaction_groups(new_compaction_group_id, Arc::new(config));
+
+            new_version_delta.pre_apply();
+            commit_multi_var!(self.meta_store_ref(), version, compaction_groups_txn)?;
         }
-        branched_ssts.commit_memory();
-        self.notify_last_version_delta(versioning);
-        drop(versioning_guard);
+
         let mut canceled_tasks = vec![];
         for task_assignment in compaction_guard.compact_task_assignment.values() {
-            let mut need_cancel = false;
             if let Some(task) = task_assignment.compact_task.as_ref() {
-                for input_level in &task.input_ssts {
-                    for sst in &input_level.table_infos {
-                        if changed_sst_ids.contains(&sst.sst_id) {
-                            need_cancel = true;
-                            break;
-                        }
-                    }
-                }
+                let need_cancel =
+                    HummockManager::is_compact_task_expired(task, &versioning.current_version);
                 if need_cancel {
                     canceled_tasks.push(ReportTask {
                         task_id: task.task_id,
@@ -668,6 +594,7 @@ impl HummockManager {
             }
         }
 
+        drop(versioning_guard);
         drop(compaction_guard);
         self.report_compact_tasks(canceled_tasks).await?;
 
@@ -686,30 +613,34 @@ impl HummockManager {
             .with_label_values(&[&parent_group_id.to_string()])
             .inc();
 
-        Ok((target_compaction_group_id, table_to_partition))
+        Ok(target_compaction_group_id)
     }
 
-    #[named]
     pub async fn calculate_compaction_group_statistic(&self) -> Vec<TableGroupInfo> {
         let mut infos = vec![];
         {
-            let versioning_guard = read_lock!(self, versioning).await;
+            let versioning_guard = self.versioning.read().await;
             let version = &versioning_guard.current_version;
-            for (group_id, group) in &version.levels {
+            for group_id in version.levels.keys() {
                 let mut group_info = TableGroupInfo {
                     group_id: *group_id,
                     ..Default::default()
                 };
-                for table_id in &group.member_table_ids {
+                for table_id in version
+                    .state_table_info
+                    .compaction_group_member_table_ids(*group_id)
+                {
                     let stats_size = versioning_guard
                         .version_stats
                         .table_stats
-                        .get(table_id)
+                        .get(&table_id.table_id)
                         .map(|stats| stats.total_key_size + stats.total_value_size)
                         .unwrap_or(0);
                     let table_size = std::cmp::max(stats_size, 0) as u64;
                     group_info.group_size += table_size;
-                    group_info.table_statistic.insert(*table_id, table_size);
+                    group_info
+                        .table_statistic
+                        .insert(table_id.table_id, table_size);
                 }
                 infos.push(group_info);
             }
@@ -722,6 +653,22 @@ impl HummockManager {
         }
         infos
     }
+
+    pub(super) async fn initial_compaction_group_config_after_load(
+        &self,
+        versioning_guard: &Versioning,
+        compaction_group_manager: &mut CompactionGroupManager,
+    ) -> Result<()> {
+        // 1. Due to version compatibility, we fix some of the configuration of older versions after hummock starts.
+        let current_version = &versioning_guard.current_version;
+        let all_group_ids = get_compaction_group_ids(current_version).collect_vec();
+        let default_config = compaction_group_manager.default_compaction_config();
+        let mut compaction_groups_txn = compaction_group_manager.start_compaction_groups_txn();
+        compaction_groups_txn.try_create_compaction_groups(&all_group_ids, default_config);
+        commit_multi_var!(self.meta_store_ref(), compaction_groups_txn)?;
+
+        Ok(())
+    }
 }
 
 /// We muse ensure there is an entry exists in [`CompactionGroupManager`] for any
@@ -733,71 +680,15 @@ impl HummockManager {
 /// 3. move existent table to new compaction group.
 pub(super) struct CompactionGroupManager {
     compaction_groups: BTreeMap<CompactionGroupId, CompactionGroup>,
-    default_config: CompactionConfig,
-    meta_store_impl: MetaStoreImpl,
+    default_config: Arc<CompactionConfig>,
+    /// Tables that write limit is trigger for.
+    pub write_limit: HashMap<CompactionGroupId, WriteLimit>,
 }
 
 impl CompactionGroupManager {
-    async fn init(&mut self) -> Result<()> {
-        let loaded_compaction_groups: BTreeMap<CompactionGroupId, CompactionGroup> =
-            match &self.meta_store_impl {
-                MetaStoreImpl::Kv(meta_store) => CompactionGroup::list(meta_store)
-                    .await?
-                    .into_iter()
-                    .map(|cg| (cg.group_id(), cg))
-                    .collect(),
-                MetaStoreImpl::Sql(sql_meta_store) => {
-                    use sea_orm::EntityTrait;
-                    compaction_config::Entity::find()
-                        .all(&sql_meta_store.conn)
-                        .await
-                        .map_err(MetadataModelError::from)?
-                        .into_iter()
-                        .map(|m| (m.compaction_group_id as CompactionGroupId, m.into()))
-                        .collect()
-                }
-            };
-        if !loaded_compaction_groups.is_empty() {
-            self.compaction_groups = loaded_compaction_groups;
-        }
-        Ok(())
-    }
-
-    /// Gets compaction group config for `compaction_group_id`, inserts default one if missing.
-    pub(super) async fn get_or_insert_compaction_group_config(
-        &mut self,
-        compaction_group_id: CompactionGroupId,
-    ) -> Result<CompactionGroup> {
-        let r = self
-            .get_or_insert_compaction_group_configs(&[compaction_group_id])
-            .await?;
-        Ok(r.into_values().next().unwrap())
-    }
-
-    /// Gets compaction group configs for `compaction_group_ids`, inserts default one if missing.
-    pub(super) async fn get_or_insert_compaction_group_configs(
-        &mut self,
-        compaction_group_ids: &[CompactionGroupId],
-    ) -> Result<HashMap<CompactionGroupId, CompactionGroup>> {
-        let mut compaction_groups = create_trx_wrapper!(
-            self.meta_store_impl,
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut self.compaction_groups,)
-        );
-        for id in compaction_group_ids {
-            if compaction_groups.contains_key(id) {
-                continue;
-            }
-            let new_entry = CompactionGroup::new(*id, self.default_config.clone());
-            compaction_groups.insert(*id, new_entry);
-        }
-        commit_multi_var!(self.meta_store_impl, compaction_groups)?;
-
-        let r = compaction_group_ids
-            .iter()
-            .map(|id| (*id, self.compaction_groups[id].clone()))
-            .collect();
-        Ok(r)
+    /// Starts a transaction to update compaction group configs.
+    pub fn start_compaction_groups_txn(&mut self) -> CompactionGroupTransaction<'_> {
+        CompactionGroupTransaction::new(&mut self.compaction_groups)
     }
 
     /// Tries to get compaction group config for `compaction_group_id`.
@@ -808,84 +699,10 @@ impl CompactionGroupManager {
         self.compaction_groups.get(&compaction_group_id).cloned()
     }
 
-    pub(super) fn default_compaction_config(&self) -> CompactionConfig {
+    /// Tries to get compaction group config for `compaction_group_id`.
+    pub(super) fn default_compaction_config(&self) -> Arc<CompactionConfig> {
         self.default_config.clone()
     }
-
-    pub async fn update_compaction_config(
-        &mut self,
-        compaction_group_ids: &[CompactionGroupId],
-        config_to_update: &[MutableConfig],
-    ) -> Result<Vec<CompactionGroup>> {
-        let mut compaction_groups = create_trx_wrapper!(
-            self.meta_store_impl,
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut self.compaction_groups,)
-        );
-        let mut result = Vec::with_capacity(compaction_group_ids.len());
-        for compaction_group_id in compaction_group_ids.iter().unique() {
-            let group = compaction_groups.get(compaction_group_id).ok_or_else(|| {
-                Error::CompactionGroup(format!("invalid group {}", *compaction_group_id))
-            })?;
-            let mut config = group.compaction_config.as_ref().clone();
-            update_compaction_config(&mut config, config_to_update);
-            if let Err(reason) = validate_compaction_config(&config) {
-                return Err(Error::CompactionGroup(reason));
-            }
-            let mut new_group = group.clone();
-            new_group.compaction_config = Arc::new(config);
-            compaction_groups.insert(*compaction_group_id, new_group.clone());
-            result.push(new_group);
-        }
-        commit_multi_var!(self.meta_store_impl, compaction_groups)?;
-        Ok(result)
-    }
-
-    /// Initializes the config for a group.
-    /// Should only be used by compaction test.
-    pub async fn init_compaction_config_for_replay(
-        &mut self,
-        group_id: CompactionGroupId,
-        config: CompactionConfig,
-    ) -> Result<()> {
-        let insert = create_trx_wrapper!(
-            self.meta_store_impl,
-            BTreeMapEntryTransactionWrapper,
-            BTreeMapEntryTransaction::new_insert(
-                &mut self.compaction_groups,
-                group_id,
-                CompactionGroup {
-                    group_id,
-                    compaction_config: Arc::new(config),
-                },
-            )
-        );
-        commit_multi_var!(self.meta_store_impl, insert)?;
-        Ok(())
-    }
-
-    /// Removes stale group configs.
-    async fn purge(&mut self, existing_groups: HashSet<CompactionGroupId>) -> Result<()> {
-        let mut compaction_groups = create_trx_wrapper!(
-            self.meta_store_impl,
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut self.compaction_groups,)
-        );
-        let stale_group = compaction_groups
-            .tree_ref()
-            .keys()
-            .cloned()
-            .filter(|k| !existing_groups.contains(k))
-            .collect_vec();
-        if stale_group.is_empty() {
-            return Ok(());
-        }
-        for group in stale_group {
-            compaction_groups.remove(group);
-        }
-        commit_multi_var!(self.meta_store_impl, compaction_groups)?;
-        Ok(())
-    }
 }
 
 fn update_compaction_config(target: &mut CompactionConfig, items: &[MutableConfig]) {
@@ -936,7 +753,92 @@ fn update_compaction_config(target: &mut CompactionConfig, items: &[MutableConfi
             MutableConfig::TombstoneReclaimRatio(c) => {
                 target.tombstone_reclaim_ratio = *c;
             }
+            MutableConfig::CompressionAlgorithm(c) => {
+                target.compression_algorithm[c.get_level() as usize]
+                    .clone_from(&c.compression_algorithm);
+            }
+            MutableConfig::MaxL0CompactLevelCount(c) => {
+                target.max_l0_compact_level_count = *c;
+            }
+        }
+    }
+}
+
+impl<'a> CompactionGroupTransaction<'a> {
+    /// Inserts compaction group configs if they do not exist.
+    pub fn try_create_compaction_groups(
+        &mut self,
+        compaction_group_ids: &[CompactionGroupId],
+        config: Arc<CompactionConfig>,
+    ) -> bool {
+        let mut trivial = true;
+        for id in compaction_group_ids {
+            if self.contains_key(id) {
+                continue;
+            }
+            let new_entry = CompactionGroup::new(*id, config.as_ref().clone());
+            self.insert(*id, new_entry);
+
+            trivial = false;
+        }
+
+        !trivial
+    }
+
+    pub fn create_compaction_groups(
+        &mut self,
+        compaction_group_id: CompactionGroupId,
+        config: Arc<CompactionConfig>,
+    ) {
+        self.try_create_compaction_groups(&[compaction_group_id], config);
+    }
+
+    /// Tries to get compaction group config for `compaction_group_id`.
+    pub(super) fn try_get_compaction_group_config(
+        &self,
+        compaction_group_id: CompactionGroupId,
+    ) -> Option<&CompactionGroup> {
+        self.get(&compaction_group_id)
+    }
+
+    /// Removes stale group configs.
+    fn purge(&mut self, existing_groups: HashSet<CompactionGroupId>) {
+        let stale_group = self
+            .tree_ref()
+            .keys()
+            .cloned()
+            .filter(|k| !existing_groups.contains(k))
+            .collect_vec();
+        if stale_group.is_empty() {
+            return;
+        }
+        for group in stale_group {
+            self.remove(group);
+        }
+    }
+
+    pub(super) fn update_compaction_config(
+        &mut self,
+        compaction_group_ids: &[CompactionGroupId],
+        config_to_update: &[MutableConfig],
+    ) -> Result<HashMap<CompactionGroupId, CompactionGroup>> {
+        let mut results = HashMap::default();
+        for compaction_group_id in compaction_group_ids.iter().unique() {
+            let group = self.get(compaction_group_id).ok_or_else(|| {
+                Error::CompactionGroup(format!("invalid group {}", *compaction_group_id))
+            })?;
+            let mut config = group.compaction_config.as_ref().clone();
+            update_compaction_config(&mut config, config_to_update);
+            if let Err(reason) = validate_compaction_config(&config) {
+                return Err(Error::CompactionGroup(reason));
+            }
+            let mut new_group = group.clone();
+            new_group.compaction_config = Arc::new(config);
+            self.insert(*compaction_group_id, new_group.clone());
+            results.insert(new_group.group_id(), new_group);
         }
+
+        Ok(results)
     }
 }
 
@@ -949,45 +851,61 @@ mod tests {
     use risingwave_pb::hummock::rise_ctl_update_compaction_config_request::mutable_config::MutableConfig;
     use risingwave_pb::meta::table_fragments::Fragment;
 
+    use crate::hummock::commit_multi_var;
+    use crate::hummock::error::Result;
+    use crate::hummock::manager::compaction_group_manager::CompactionGroupManager;
     use crate::hummock::test_utils::setup_compute_env;
-    use crate::hummock::HummockManager;
+    use crate::manager::MetaStoreImpl;
     use crate::model::TableFragments;
 
     #[tokio::test]
     async fn test_inner() {
         let (env, ..) = setup_compute_env(8080).await;
-        let inner = HummockManager::build_compaction_group_manager(&env)
-            .await
-            .unwrap();
-        assert_eq!(inner.read().await.compaction_groups.len(), 2);
-        inner
-            .write()
-            .await
-            .update_compaction_config(&[100, 200], &[])
+        let mut inner = CompactionGroupManager::new(&env).await.unwrap();
+        assert_eq!(inner.compaction_groups.len(), 2);
+
+        async fn update_compaction_config(
+            meta: &MetaStoreImpl,
+            inner: &mut CompactionGroupManager,
+            cg_ids: &[u64],
+            config_to_update: &[MutableConfig],
+        ) -> Result<()> {
+            let mut compaction_groups_txn = inner.start_compaction_groups_txn();
+            compaction_groups_txn.update_compaction_config(cg_ids, config_to_update)?;
+            commit_multi_var!(meta, compaction_groups_txn)
+        }
+
+        async fn insert_compaction_group_configs(
+            meta: &MetaStoreImpl,
+            inner: &mut CompactionGroupManager,
+            cg_ids: &[u64],
+        ) {
+            let default_config = inner.default_compaction_config();
+            let mut compaction_groups_txn = inner.start_compaction_groups_txn();
+            if compaction_groups_txn.try_create_compaction_groups(cg_ids, default_config) {
+                commit_multi_var!(meta, compaction_groups_txn).unwrap();
+            }
+        }
+
+        update_compaction_config(env.meta_store_ref(), &mut inner, &[100, 200], &[])
             .await
             .unwrap_err();
-        inner
-            .write()
-            .await
-            .get_or_insert_compaction_group_configs(&[100, 200])
-            .await
-            .unwrap();
-        assert_eq!(inner.read().await.compaction_groups.len(), 4);
-        let inner = HummockManager::build_compaction_group_manager(&env)
-            .await
-            .unwrap();
-        assert_eq!(inner.read().await.compaction_groups.len(), 4);
-        inner
-            .write()
-            .await
-            .update_compaction_config(&[100, 200], &[MutableConfig::MaxSubCompaction(123)])
-            .await
-            .unwrap();
-        assert_eq!(inner.read().await.compaction_groups.len(), 4);
+        insert_compaction_group_configs(env.meta_store_ref(), &mut inner, &[100, 200]).await;
+        assert_eq!(inner.compaction_groups.len(), 4);
+        let mut inner = CompactionGroupManager::new(&env).await.unwrap();
+        assert_eq!(inner.compaction_groups.len(), 4);
+
+        update_compaction_config(
+            env.meta_store_ref(),
+            &mut inner,
+            &[100, 200],
+            &[MutableConfig::MaxSubCompaction(123)],
+        )
+        .await
+        .unwrap();
+        assert_eq!(inner.compaction_groups.len(), 4);
         assert_eq!(
             inner
-                .read()
-                .await
                 .try_get_compaction_group_config(100)
                 .unwrap()
                 .compaction_config
@@ -996,8 +914,6 @@ mod tests {
         );
         assert_eq!(
             inner
-                .read()
-                .await
                 .try_get_compaction_group_config(200)
                 .unwrap()
                 .compaction_config
diff --git a/src/meta/src/hummock/manager/context.rs b/src/meta/src/hummock/manager/context.rs
index 21aeb8402838b..982a94fd5f9db 100644
--- a/src/meta/src/hummock/manager/context.rs
+++ b/src/meta/src/hummock/manager/context.rs
@@ -12,35 +12,66 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{HashMap, HashSet};
-use std::ops::DerefMut;
+use std::collections::{BTreeMap, HashMap, HashSet};
 
 use fail::fail_point;
-use function_name::named;
 use itertools::Itertools;
+use risingwave_common::util::epoch::INVALID_EPOCH;
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::{
-    ExtendedSstableInfo, HummockContextId, HummockEpoch, HummockSstableObjectId,
+    ExtendedSstableInfo, HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId,
+    INVALID_VERSION_ID,
+};
+use risingwave_pb::hummock::{
+    HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot, ValidationTask,
 };
-use risingwave_pb::hummock::ValidationTask;
 
 use crate::hummock::error::{Error, Result};
-use crate::hummock::manager::{
-    commit_multi_var, create_trx_wrapper, read_lock, start_measure_real_process_timer, write_lock,
+use crate::hummock::manager::worker::{HummockManagerEvent, HummockManagerEventSender};
+use crate::hummock::manager::{commit_multi_var, start_measure_real_process_timer};
+use crate::hummock::metrics_utils::{
+    trigger_pin_unpin_snapshot_state, trigger_pin_unpin_version_state,
 };
 use crate::hummock::HummockManager;
-use crate::manager::META_NODE_ID;
-use crate::model::{BTreeMapTransaction, BTreeMapTransactionWrapper, ValTransaction};
-use crate::storage::MetaStore;
+use crate::manager::{MetaStoreImpl, MetadataManager, META_NODE_ID};
+use crate::model::BTreeMapTransaction;
+use crate::rpc::metrics::MetaMetrics;
 
-impl HummockManager {
+/// `HummockVersionSafePoint` prevents hummock versions GE than it from being GC.
+/// It's used by meta node itself to temporarily pin versions.
+pub struct HummockVersionSafePoint {
+    pub id: HummockVersionId,
+    event_sender: HummockManagerEventSender,
+}
+
+impl Drop for HummockVersionSafePoint {
+    fn drop(&mut self) {
+        if self
+            .event_sender
+            .send(HummockManagerEvent::DropSafePoint(self.id))
+            .is_err()
+        {
+            tracing::debug!("failed to drop hummock version safe point {}", self.id);
+        }
+    }
+}
+
+#[derive(Default)]
+pub(super) struct ContextInfo {
+    pub pinned_versions: BTreeMap<HummockContextId, HummockPinnedVersion>,
+    pub pinned_snapshots: BTreeMap<HummockContextId, HummockPinnedSnapshot>,
+    /// `version_safe_points` is similar to `pinned_versions` expect for being a transient state.
+    pub version_safe_points: Vec<HummockVersionId>,
+}
+
+impl ContextInfo {
     /// Release resources pinned by these contexts, including:
     /// - Version
     /// - Snapshot
-    #[named]
-    pub async fn release_contexts(
-        &self,
+    async fn release_contexts(
+        &mut self,
         context_ids: impl AsRef<[HummockContextId]>,
+        meta_store_ref: MetaStoreImpl,
     ) -> Result<()> {
         fail_point!("release_contexts_metastore_err", |_| Err(Error::MetaStore(
             anyhow::anyhow!("failpoint metastore error")
@@ -49,50 +80,91 @@ impl HummockManager {
             anyhow::anyhow!("failpoint internal error")
         )));
 
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        let versioning = versioning_guard.deref_mut();
-        let mut pinned_versions = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning.pinned_versions,)
-        );
-        let mut pinned_snapshots = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning.pinned_snapshots,)
-        );
+        let mut pinned_versions = BTreeMapTransaction::new(&mut self.pinned_versions);
+        let mut pinned_snapshots = BTreeMapTransaction::new(&mut self.pinned_snapshots);
         for context_id in context_ids.as_ref() {
             pinned_versions.remove(*context_id);
             pinned_snapshots.remove(*context_id);
         }
-        commit_multi_var!(self.meta_store_ref(), pinned_versions, pinned_snapshots)?;
+        commit_multi_var!(meta_store_ref, pinned_versions, pinned_snapshots)?;
 
+        Ok(())
+    }
+}
+
+impl HummockManager {
+    pub async fn release_contexts(
+        &self,
+        context_ids: impl AsRef<[HummockContextId]>,
+    ) -> Result<()> {
+        let mut context_info = self.context_info.write().await;
+        context_info
+            .release_contexts(context_ids, self.env.meta_store())
+            .await?;
         #[cfg(test)]
         {
-            drop(versioning_guard);
+            drop(context_info);
             self.check_state_consistency().await;
         }
-
         Ok(())
     }
 
     /// Checks whether `context_id` is valid.
     pub async fn check_context(&self, context_id: HummockContextId) -> Result<bool> {
-        Ok(self
-            .metadata_manager()
+        self.context_info
+            .read()
+            .await
+            .check_context(context_id, &self.metadata_manager)
+            .await
+    }
+
+    async fn check_context_with_meta_node(
+        &self,
+        context_id: HummockContextId,
+        context_info: &ContextInfo,
+    ) -> Result<()> {
+        if context_id == META_NODE_ID {
+            // Using the preserved meta id is allowed.
+        } else if !context_info
+            .check_context(context_id, &self.metadata_manager)
+            .await?
+        {
+            // The worker is not found in cluster.
+            return Err(Error::InvalidContext(context_id));
+        }
+        Ok(())
+    }
+
+    #[cfg(any(test, feature = "test"))]
+    pub async fn get_min_pinned_version_id(&self) -> HummockVersionId {
+        self.context_info.read().await.min_pinned_version_id()
+    }
+}
+
+impl ContextInfo {
+    /// Checks whether `context_id` is valid.
+    ///
+    /// Need `&self` to sync with `release_context`
+    pub(super) async fn check_context(
+        &self,
+        context_id: HummockContextId,
+        metadata_manager: &MetadataManager,
+    ) -> Result<bool> {
+        Ok(metadata_manager
             .get_worker_by_id(context_id)
             .await
             .map_err(|err| Error::MetaStore(err.into()))?
             .is_some())
     }
+}
 
+impl HummockManager {
     /// Release invalid contexts, aka worker node ids which are no longer valid in `ClusterManager`.
-    #[named]
     pub(super) async fn release_invalid_contexts(&self) -> Result<Vec<HummockContextId>> {
-        let active_context_ids = {
-            let compaction_guard = read_lock!(self, compaction).await;
-            let versioning_guard = read_lock!(self, versioning).await;
-            let _timer = start_measure_real_process_timer!(self);
+        let (active_context_ids, mut context_info) = {
+            let compaction_guard = self.compaction.read().await;
+            let context_info = self.context_info.write().await;
+            let _timer = start_measure_real_process_timer!(self, "release_invalid_contexts");
             let mut active_context_ids = HashSet::new();
             active_context_ids.extend(
                 compaction_guard
@@ -100,19 +172,24 @@ impl HummockManager {
                     .values()
                     .map(|c| c.context_id),
             );
-            active_context_ids.extend(versioning_guard.pinned_versions.keys());
-            active_context_ids.extend(versioning_guard.pinned_snapshots.keys());
-            active_context_ids
+            active_context_ids.extend(context_info.pinned_versions.keys());
+            active_context_ids.extend(context_info.pinned_snapshots.keys());
+            (active_context_ids, context_info)
         };
 
         let mut invalid_context_ids = vec![];
         for active_context_id in &active_context_ids {
-            if !self.check_context(*active_context_id).await? {
+            if !context_info
+                .check_context(*active_context_id, &self.metadata_manager)
+                .await?
+            {
                 invalid_context_ids.push(*active_context_id);
             }
         }
 
-        self.release_contexts(&invalid_context_ids).await?;
+        context_info
+            .release_contexts(&invalid_context_ids, self.env.meta_store())
+            .await?;
 
         Ok(invalid_context_ids)
     }
@@ -133,7 +210,13 @@ impl HummockManager {
                     continue;
                 }
             }
-            if !self.check_context(*context_id).await? {
+            if !self
+                .context_info
+                .read()
+                .await
+                .check_context(*context_id, &self.metadata_manager)
+                .await?
+            {
                 return Err(Error::InvalidSst(*sst_id));
             }
         }
@@ -184,3 +267,235 @@ impl HummockManager {
         self.release_contexts([META_NODE_ID]).await
     }
 }
+
+// pin and unpin method
+impl HummockManager {
+    /// Pin the current greatest hummock version. The pin belongs to `context_id`
+    /// and will be unpinned when `context_id` is invalidated.
+    pub async fn pin_version(&self, context_id: HummockContextId) -> Result<HummockVersion> {
+        let versioning = self.versioning.read().await;
+        let mut context_info = self.context_info.write().await;
+        self.check_context_with_meta_node(context_id, &context_info)
+            .await?;
+        let _timer = start_measure_real_process_timer!(self, "pin_version");
+        let mut pinned_versions = BTreeMapTransaction::new(&mut context_info.pinned_versions);
+        let mut context_pinned_version = pinned_versions.new_entry_txn_or_default(
+            context_id,
+            HummockPinnedVersion {
+                context_id,
+                min_pinned_id: INVALID_VERSION_ID,
+            },
+        );
+        let version_id = versioning.current_version.id;
+        let ret = versioning.current_version.clone();
+        if context_pinned_version.min_pinned_id == INVALID_VERSION_ID
+            || context_pinned_version.min_pinned_id > version_id
+        {
+            context_pinned_version.min_pinned_id = version_id;
+            commit_multi_var!(self.meta_store_ref(), context_pinned_version)?;
+            trigger_pin_unpin_version_state(&self.metrics, &context_info.pinned_versions);
+        }
+
+        #[cfg(test)]
+        {
+            drop(context_info);
+            drop(versioning);
+            self.check_state_consistency().await;
+        }
+
+        Ok(ret)
+    }
+
+    /// Unpin all pins which belongs to `context_id` and has an id which is older than
+    /// `unpin_before`. All versions >= `unpin_before` will be treated as if they are all pinned by
+    /// this `context_id` so they will not be vacuumed.
+    pub async fn unpin_version_before(
+        &self,
+        context_id: HummockContextId,
+        unpin_before: HummockVersionId,
+    ) -> Result<()> {
+        let mut context_info = self.context_info.write().await;
+        self.check_context_with_meta_node(context_id, &context_info)
+            .await?;
+        let _timer = start_measure_real_process_timer!(self, "unpin_version_before");
+        let mut pinned_versions = BTreeMapTransaction::new(&mut context_info.pinned_versions);
+        let mut context_pinned_version = pinned_versions.new_entry_txn_or_default(
+            context_id,
+            HummockPinnedVersion {
+                context_id,
+                min_pinned_id: 0,
+            },
+        );
+        assert!(
+            context_pinned_version.min_pinned_id <= unpin_before,
+            "val must be monotonically non-decreasing. old = {}, new = {}.",
+            context_pinned_version.min_pinned_id,
+            unpin_before
+        );
+        context_pinned_version.min_pinned_id = unpin_before;
+        commit_multi_var!(self.meta_store_ref(), context_pinned_version)?;
+        trigger_pin_unpin_version_state(&self.metrics, &context_info.pinned_versions);
+
+        #[cfg(test)]
+        {
+            drop(context_info);
+            self.check_state_consistency().await;
+        }
+
+        Ok(())
+    }
+
+    pub async fn pin_specific_snapshot(
+        &self,
+        context_id: HummockContextId,
+        epoch: HummockEpoch,
+    ) -> Result<HummockSnapshot> {
+        let snapshot = self.latest_snapshot.load();
+        let mut guard = self.context_info.write().await;
+        self.check_context_with_meta_node(context_id, &guard)
+            .await?;
+        let mut pinned_snapshots = BTreeMapTransaction::new(&mut guard.pinned_snapshots);
+        let mut context_pinned_snapshot = pinned_snapshots.new_entry_txn_or_default(
+            context_id,
+            HummockPinnedSnapshot {
+                context_id,
+                minimal_pinned_snapshot: INVALID_EPOCH,
+            },
+        );
+        let epoch_to_pin = std::cmp::min(epoch, snapshot.committed_epoch);
+        if context_pinned_snapshot.minimal_pinned_snapshot == INVALID_EPOCH {
+            context_pinned_snapshot.minimal_pinned_snapshot = epoch_to_pin;
+            commit_multi_var!(self.meta_store_ref(), context_pinned_snapshot)?;
+        }
+        Ok(HummockSnapshot::clone(&snapshot))
+    }
+
+    /// Make sure `max_committed_epoch` is pinned and return it.
+    pub async fn pin_snapshot(&self, context_id: HummockContextId) -> Result<HummockSnapshot> {
+        let snapshot = self.latest_snapshot.load();
+        let mut guard = self.context_info.write().await;
+        self.check_context_with_meta_node(context_id, &guard)
+            .await?;
+        let _timer = start_measure_real_process_timer!(self, "pin_snapshot");
+        let mut pinned_snapshots = BTreeMapTransaction::new(&mut guard.pinned_snapshots);
+        let mut context_pinned_snapshot = pinned_snapshots.new_entry_txn_or_default(
+            context_id,
+            HummockPinnedSnapshot {
+                context_id,
+                minimal_pinned_snapshot: INVALID_EPOCH,
+            },
+        );
+        if context_pinned_snapshot.minimal_pinned_snapshot == INVALID_EPOCH {
+            context_pinned_snapshot.minimal_pinned_snapshot = snapshot.committed_epoch;
+            commit_multi_var!(self.meta_store_ref(), context_pinned_snapshot)?;
+            trigger_pin_unpin_snapshot_state(&self.metrics, &guard.pinned_snapshots);
+        }
+        Ok(HummockSnapshot::clone(&snapshot))
+    }
+
+    pub async fn unpin_snapshot(&self, context_id: HummockContextId) -> Result<()> {
+        let mut context_info = self.context_info.write().await;
+        self.check_context_with_meta_node(context_id, &context_info)
+            .await?;
+        let _timer = start_measure_real_process_timer!(self, "unpin_snapshot");
+        let mut pinned_snapshots = BTreeMapTransaction::new(&mut context_info.pinned_snapshots);
+        let release_snapshot = pinned_snapshots.remove(context_id);
+        if release_snapshot.is_some() {
+            commit_multi_var!(self.meta_store_ref(), pinned_snapshots)?;
+            trigger_pin_unpin_snapshot_state(&self.metrics, &context_info.pinned_snapshots);
+        }
+
+        #[cfg(test)]
+        {
+            drop(context_info);
+            self.check_state_consistency().await;
+        }
+
+        Ok(())
+    }
+
+    /// Unpin all snapshots smaller than specified epoch for current context.
+    pub async fn unpin_snapshot_before(
+        &self,
+        context_id: HummockContextId,
+        hummock_snapshot: HummockSnapshot,
+    ) -> Result<()> {
+        let versioning = self.versioning.read().await;
+        let mut context_info = self.context_info.write().await;
+        self.check_context_with_meta_node(context_id, &context_info)
+            .await?;
+        let _timer = start_measure_real_process_timer!(self, "unpin_snapshot_before");
+        // Use the max_committed_epoch in storage as the snapshot ts so only committed changes are
+        // visible in the snapshot.
+        let max_committed_epoch = versioning.current_version.max_committed_epoch;
+        // Ensure the unpin will not clean the latest one.
+        let snapshot_committed_epoch = hummock_snapshot.committed_epoch;
+        #[cfg(not(test))]
+        {
+            assert!(snapshot_committed_epoch <= max_committed_epoch);
+        }
+        let last_read_epoch = std::cmp::min(snapshot_committed_epoch, max_committed_epoch);
+
+        let mut pinned_snapshots = BTreeMapTransaction::new(&mut context_info.pinned_snapshots);
+        let mut context_pinned_snapshot = pinned_snapshots.new_entry_txn_or_default(
+            context_id,
+            HummockPinnedSnapshot {
+                context_id,
+                minimal_pinned_snapshot: INVALID_EPOCH,
+            },
+        );
+
+        // Unpin the snapshots pinned by meta but frontend doesn't know. Also equal to unpin all
+        // epochs below specific watermark.
+        if context_pinned_snapshot.minimal_pinned_snapshot < last_read_epoch
+            || context_pinned_snapshot.minimal_pinned_snapshot == INVALID_EPOCH
+        {
+            context_pinned_snapshot.minimal_pinned_snapshot = last_read_epoch;
+            commit_multi_var!(self.meta_store_ref(), context_pinned_snapshot)?;
+            trigger_pin_unpin_snapshot_state(&self.metrics, &context_info.pinned_snapshots);
+        }
+
+        #[cfg(test)]
+        {
+            drop(context_info);
+            drop(versioning);
+            self.check_state_consistency().await;
+        }
+
+        Ok(())
+    }
+}
+
+// safe point
+impl HummockManager {
+    pub async fn register_safe_point(&self) -> HummockVersionSafePoint {
+        let versioning = self.versioning.read().await;
+        let mut wl = self.context_info.write().await;
+        let safe_point = HummockVersionSafePoint {
+            id: versioning.current_version.id,
+            event_sender: self.event_sender.clone(),
+        };
+        wl.version_safe_points.push(safe_point.id);
+        trigger_safepoint_stat(&self.metrics, &wl.version_safe_points);
+        safe_point
+    }
+
+    pub async fn unregister_safe_point(&self, safe_point: HummockVersionId) {
+        let mut wl = self.context_info.write().await;
+        let version_safe_points = &mut wl.version_safe_points;
+        if let Some(pos) = version_safe_points.iter().position(|sp| *sp == safe_point) {
+            version_safe_points.remove(pos);
+        }
+        trigger_safepoint_stat(&self.metrics, &wl.version_safe_points);
+    }
+}
+
+fn trigger_safepoint_stat(metrics: &MetaMetrics, safepoints: &[HummockVersionId]) {
+    if let Some(sp) = safepoints.iter().min() {
+        metrics.min_safepoint_version_id.set(*sp as _);
+    } else {
+        metrics
+            .min_safepoint_version_id
+            .set(HummockVersionId::MAX as _);
+    }
+}
diff --git a/src/meta/src/hummock/manager/gc.rs b/src/meta/src/hummock/manager/gc.rs
index dafd3231afcce..5f5150b7777cb 100644
--- a/src/meta/src/hummock/manager/gc.rs
+++ b/src/meta/src/hummock/manager/gc.rs
@@ -19,9 +19,9 @@ use std::ops::DerefMut;
 use std::time::Duration;
 
 use anyhow::Context;
-use function_name::named;
 use futures::{stream, StreamExt};
 use itertools::Itertools;
+use parking_lot::Mutex;
 use risingwave_hummock_sdk::HummockSstableObjectId;
 use risingwave_pb::common::worker_node::State::Running;
 use risingwave_pb::common::WorkerType;
@@ -29,32 +29,53 @@ use risingwave_pb::hummock::subscribe_compaction_event_response::Event as Respon
 use risingwave_pb::hummock::FullScanTask;
 
 use crate::hummock::error::{Error, Result};
-use crate::hummock::manager::versioning::Versioning;
-use crate::hummock::manager::{commit_multi_var, create_trx_wrapper, read_lock, write_lock};
+use crate::hummock::manager::commit_multi_var;
 use crate::hummock::HummockManager;
 use crate::manager::MetadataManager;
-use crate::model::{BTreeMapTransaction, BTreeMapTransactionWrapper, ValTransaction};
-use crate::storage::MetaStore;
+use crate::model::BTreeMapTransaction;
+
+#[derive(Default)]
+pub(super) struct DeleteObjectTracker {
+    /// Objects that waits to be deleted from object store. It comes from either compaction, or
+    /// full GC (listing object store).
+    objects_to_delete: Mutex<HashSet<HummockSstableObjectId>>,
+}
+
+impl DeleteObjectTracker {
+    pub(super) fn add(&self, objects: impl Iterator<Item = HummockSstableObjectId>) {
+        self.objects_to_delete.lock().extend(objects)
+    }
+
+    pub(super) fn current(&self) -> HashSet<HummockSstableObjectId> {
+        self.objects_to_delete.lock().clone()
+    }
+
+    pub(super) fn clear(&self) {
+        self.objects_to_delete.lock().clear();
+    }
+
+    pub(super) fn ack<'a>(&self, objects: impl Iterator<Item = &'a HummockSstableObjectId>) {
+        let mut lock = self.objects_to_delete.lock();
+        for object in objects {
+            lock.remove(object);
+        }
+    }
+}
 
 impl HummockManager {
     /// Gets SST objects that is safe to be deleted from object store.
-    #[named]
-    pub async fn get_objects_to_delete(&self) -> Vec<HummockSstableObjectId> {
-        read_lock!(self, versioning)
-            .await
-            .objects_to_delete
+    pub fn get_objects_to_delete(&self) -> Vec<HummockSstableObjectId> {
+        self.delete_object_tracker
+            .current()
             .iter()
             .cloned()
             .collect_vec()
     }
 
     /// Acknowledges SSTs have been deleted from object store.
-    #[named]
     pub async fn ack_deleted_objects(&self, object_ids: &[HummockSstableObjectId]) -> Result<()> {
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        for object_id in object_ids {
-            versioning_guard.objects_to_delete.remove(object_id);
-        }
+        self.delete_object_tracker.ack(object_ids.iter());
+        let mut versioning_guard = self.versioning.write().await;
         for stale_objects in versioning_guard.checkpoint.stale_objects.values_mut() {
             stale_objects.id.retain(|id| !object_ids.contains(id));
         }
@@ -69,10 +90,10 @@ impl HummockManager {
     /// Deletes at most `batch_size` deltas.
     ///
     /// Returns (number of deleted deltas, number of remain `deltas_to_delete`).
-    #[named]
     pub async fn delete_version_deltas(&self, batch_size: usize) -> Result<(usize, usize)> {
-        let mut versioning_guard = write_lock!(self, versioning).await;
+        let mut versioning_guard = self.versioning.write().await;
         let versioning = versioning_guard.deref_mut();
+        let context_info = self.context_info.read().await;
         let deltas_to_delete = versioning
             .hummock_version_deltas
             .range(..=versioning.checkpoint.version.id)
@@ -80,14 +101,11 @@ impl HummockManager {
             .collect_vec();
         // If there is any safe point, skip this to ensure meta backup has required delta logs to
         // replay version.
-        if !versioning.version_safe_points.is_empty() {
+        if !context_info.version_safe_points.is_empty() {
             return Ok((0, deltas_to_delete.len()));
         }
-        let mut hummock_version_deltas = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning.hummock_version_deltas,)
-        );
+        let mut hummock_version_deltas =
+            BTreeMapTransaction::new(&mut versioning.hummock_version_deltas);
         let batch = deltas_to_delete
             .iter()
             .take(batch_size)
@@ -102,6 +120,7 @@ impl HummockManager {
         commit_multi_var!(self.meta_store_ref(), hummock_version_deltas)?;
         #[cfg(test)]
         {
+            drop(context_info);
             drop(versioning_guard);
             self.check_state_consistency().await;
         }
@@ -110,16 +129,15 @@ impl HummockManager {
 
     /// Extends `objects_to_delete` according to object store full scan result.
     /// Caller should ensure `object_ids` doesn't include any SST objects belong to a on-going
-    /// version write. That's to say, these object_ids won't appear in either `commit_epoch` or
+    /// version write. That's to say, these `object_ids` won't appear in either `commit_epoch` or
     /// `report_compact_task`.
-    #[named]
     pub async fn extend_objects_to_delete_from_scan(
         &self,
         object_ids: &[HummockSstableObjectId],
     ) -> usize {
         let tracked_object_ids: HashSet<HummockSstableObjectId> = {
-            let versioning_guard = read_lock!(self, versioning).await;
-            let versioning: &Versioning = &versioning_guard;
+            let versioning = self.versioning.read().await;
+            let context_info = self.context_info.read().await;
 
             // object ids in checkpoint version
             let mut tracked_object_ids = versioning.checkpoint.version.get_object_ids();
@@ -131,7 +149,7 @@ impl HummockManager {
                 tracked_object_ids.extend(delta.newly_added_object_ids());
             }
             // add stale object ids before the checkpoint version
-            let min_pinned_version_id = versioning.min_pinned_version_id();
+            let min_pinned_version_id = context_info.min_pinned_version_id();
             tracked_object_ids.extend(
                 versioning
                     .checkpoint
@@ -147,16 +165,15 @@ impl HummockManager {
             .iter()
             .filter(|object_id| !tracked_object_ids.contains(object_id))
             .collect_vec();
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        versioning_guard.objects_to_delete.extend(to_delete.clone());
-        drop(versioning_guard);
+        self.delete_object_tracker
+            .add(to_delete.iter().map(|id| **id));
         to_delete.len()
     }
 
     /// Starts a full GC.
     /// 1. Meta node sends a `FullScanTask` to a compactor in this method.
     /// 2. The compactor returns scan result of object store to meta node. See
-    /// `HummockManager::full_scan_inner` in storage crate.
+    ///    `HummockManager::full_scan_inner` in storage crate.
     /// 3. Meta node decides which SSTs to delete. See `HummockManager::complete_full_gc`.
     ///
     /// Returns Ok(false) if there is no worker available.
@@ -300,7 +317,7 @@ mod tests {
     use itertools::Itertools;
     use risingwave_hummock_sdk::HummockSstableObjectId;
 
-    use crate::hummock::manager::ResponseEvent;
+    use super::ResponseEvent;
     use crate::hummock::test_utils::{add_test_tables, setup_compute_env};
     use crate::MetaOpts;
 
diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs
index 0dc940e006584..8a49d91a55fc3 100644
--- a/src/meta/src/hummock/manager/mod.rs
+++ b/src/meta/src/hummock/manager/mod.rs
@@ -12,107 +12,61 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::borrow::BorrowMut;
-use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
+use std::collections::{BTreeMap, HashMap, VecDeque};
 use std::ops::{Deref, DerefMut};
 use std::sync::atomic::AtomicBool;
-use std::sync::{Arc, LazyLock};
-use std::time::{Duration, Instant, SystemTime};
+use std::sync::Arc;
 
-use anyhow::Context;
 use arc_swap::ArcSwap;
 use bytes::Bytes;
-use fail::fail_point;
-use function_name::named;
-use futures::future::Either;
-use futures::stream::{BoxStream, FuturesUnordered};
-use futures::{FutureExt, StreamExt};
 use itertools::Itertools;
-use parking_lot::Mutex;
-use rand::prelude::SliceRandom;
-use rand::thread_rng;
-use risingwave_common::catalog::TableId;
-use risingwave_common::config::default::compaction_config;
 use risingwave_common::monitor::rwlock::MonitoredRwLock;
 use risingwave_common::system_param::reader::SystemParamsRead;
-use risingwave_common::util::epoch::{Epoch, INVALID_EPOCH};
-use risingwave_hummock_sdk::compact::{compact_task_to_string, statistics_compact_task};
-use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
-    build_version_delta_after_version, get_compaction_group_ids,
-    get_table_compaction_group_id_mapping, BranchedSstInfo, HummockLevelsExt,
-};
-use risingwave_hummock_sdk::version::HummockVersionDelta;
+use risingwave_common::util::epoch::INVALID_EPOCH;
+use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
 use risingwave_hummock_sdk::{
-    version_archive_dir, version_checkpoint_path, CompactionGroupId, ExtendedSstableInfo,
-    HummockCompactionTaskId, HummockContextId, HummockEpoch, HummockSstableId,
-    HummockSstableObjectId, HummockVersionId, SstObjectIdRange, INVALID_VERSION_ID,
+    version_archive_dir, version_checkpoint_path, CompactionGroupId, HummockCompactionTaskId,
+    HummockContextId, HummockVersionId,
 };
 use risingwave_meta_model_v2::{
     compaction_status, compaction_task, hummock_pinned_snapshot, hummock_pinned_version,
     hummock_version_delta, hummock_version_stats,
 };
-use risingwave_pb::hummock::compact_task::{self, TaskStatus, TaskType};
-use risingwave_pb::hummock::group_delta::DeltaType;
-use risingwave_pb::hummock::rise_ctl_update_compaction_config_request::mutable_config;
-use risingwave_pb::hummock::subscribe_compaction_event_request::{
-    Event as RequestEvent, HeartBeat, ReportTask,
-};
-use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent;
 use risingwave_pb::hummock::{
-    CompactTask, CompactTaskAssignment, GroupDelta, GroupMetaChange, HummockPinnedSnapshot,
-    HummockPinnedVersion, HummockSnapshot, HummockVersionStats, IntraLevelDelta,
-    PbCompactionGroupInfo, SstableInfo, SubscribeCompactionEventRequest, TableOption, TableSchema,
+    CompactTaskAssignment, HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot,
+    HummockVersionStats, PbCompactionGroupInfo, SubscribeCompactionEventRequest,
 };
-use risingwave_pb::meta::subscribe_response::{Info, Operation};
-use rw_futures_util::{pending_on_none, select_all};
-use thiserror_ext::AsReport;
-use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender};
-use tokio::sync::oneshot::Sender;
-use tokio::sync::RwLockWriteGuard;
-use tokio::task::JoinHandle;
-use tokio_stream::wrappers::IntervalStream;
+use risingwave_pb::meta::subscribe_response::Operation;
+use tokio::sync::mpsc::UnboundedSender;
 use tonic::Streaming;
-use tracing::warn;
 
-use crate::hummock::compaction::selector::{
-    DynamicLevelSelector, LocalSelectorStatistic, ManualCompactionOption, ManualCompactionSelector,
-    SpaceReclaimCompactionSelector, TombstoneCompactionSelector, TtlCompactionSelector,
-};
-use crate::hummock::compaction::{CompactStatus, CompactionDeveloperConfig};
-use crate::hummock::error::{Error, Result};
-use crate::hummock::metrics_utils::{
-    build_compact_task_level_type_metrics_label, get_or_create_local_table_stat,
-    trigger_delta_log_stats, trigger_local_table_stat, trigger_lsm_stat, trigger_mv_stat,
-    trigger_pin_unpin_snapshot_state, trigger_pin_unpin_version_state, trigger_split_stat,
-    trigger_sst_stat, trigger_version_stat, trigger_write_stop_stats,
-};
-use crate::hummock::sequence::next_compaction_task_id;
-use crate::hummock::{CompactorManagerRef, TASK_NORMAL};
-#[cfg(any(test, feature = "test"))]
-use crate::manager::{ClusterManagerRef, FragmentManagerRef};
-use crate::manager::{MetaSrvEnv, MetaStoreImpl, MetadataManager, META_NODE_ID};
-use crate::model::{
-    BTreeMapEntryTransaction, BTreeMapEntryTransactionWrapper, BTreeMapTransaction,
-    BTreeMapTransactionWrapper, ClusterId, MetadataModel, MetadataModelError, ValTransaction,
-    VarTransaction, VarTransactionWrapper,
-};
+use crate::hummock::compaction::CompactStatus;
+use crate::hummock::error::Result;
+use crate::hummock::manager::checkpoint::HummockVersionCheckpoint;
+use crate::hummock::manager::context::ContextInfo;
+use crate::hummock::manager::gc::DeleteObjectTracker;
+use crate::hummock::CompactorManagerRef;
+use crate::manager::{MetaSrvEnv, MetaStoreImpl, MetadataManager};
+use crate::model::{ClusterId, MetadataModel, MetadataModelError};
 use crate::rpc::metrics::MetaMetrics;
-use crate::storage::MetaStore;
 
 mod compaction_group_manager;
 mod context;
 mod gc;
-#[cfg(test)]
 mod tests;
 mod versioning;
-pub use versioning::HummockVersionSafePoint;
+pub use context::HummockVersionSafePoint;
 use versioning::*;
 pub(crate) mod checkpoint;
+mod commit_epoch;
 mod compaction;
 pub mod sequence;
+mod timer_task;
+mod transaction;
 mod utils;
 mod worker;
 
+pub(crate) use commit_epoch::*;
 use compaction::*;
 pub use compaction::{check_cg_write_limit, WriteLimitType};
 pub(crate) use utils::*;
@@ -129,13 +83,14 @@ pub struct HummockManager {
     pub env: MetaSrvEnv,
 
     metadata_manager: MetadataManager,
-    /// Lock order: compaction, versioning, `compaction_group_manager`.
-    /// - Lock compaction first, then versioning, and finally `compaction_group_manager`.
+    /// Lock order: `compaction`, `versioning`, `compaction_group_manager`, `context_info`
+    /// - Lock `compaction` first, then `versioning`, then `compaction_group_manager` and finally `context_info`.
     /// - This order should be strictly followed to prevent deadlock.
     compaction: MonitoredRwLock<Compaction>,
     versioning: MonitoredRwLock<Versioning>,
     /// `CompactionGroupManager` manages compaction configs for compaction groups.
-    compaction_group_manager: tokio::sync::RwLock<CompactionGroupManager>,
+    compaction_group_manager: MonitoredRwLock<CompactionGroupManager>,
+    context_info: MonitoredRwLock<ContextInfo>,
     latest_snapshot: Snapshot,
 
     pub metrics: Arc<MetaMetrics>,
@@ -143,6 +98,8 @@ pub struct HummockManager {
     pub compactor_manager: CompactorManagerRef,
     event_sender: HummockManagerEventSender,
 
+    delete_object_tracker: DeleteObjectTracker,
+
     object_store: ObjectStoreRef,
     version_checkpoint_path: String,
     version_archive_dir: String,
@@ -157,69 +114,21 @@ pub struct HummockManager {
     // `compaction_state` will record the types of compact tasks that can be triggered in `hummock`
     // and suggest types with a certain priority.
     pub compaction_state: CompactionState,
-
-    // Record the partition corresponding to the table in each group (accepting delays)
-    // The compactor will refer to this structure to determine how to cut the boundaries of sst.
-    // Currently, we update it in a couple of scenarios
-    // 1. throughput and size are checked periodically and calculated according to the rules
-    // 2. A new group is created (split)
-    // 3. split_weight_by_vnode is modified for an existing group. (not supported yet)
-    // Tips:
-    // 1. When table_id does not exist in the current structure, compactor will not cut the boundary
-    // 2. When partition count <=1, compactor will still use table_id as the cutting boundary of sst
-    // 3. Modify the special configuration item hybrid_vnode_count = 0 to remove the table_id in hybrid cg and no longer perform alignment cutting.
-    group_to_table_vnode_partition:
-        parking_lot::RwLock<HashMap<CompactionGroupId, BTreeMap<StateTableId, u32>>>,
 }
 
 pub type HummockManagerRef = Arc<HummockManager>;
 
-/// Acquire read lock of the lock with `lock_name`.
-/// The macro will use macro `function_name` to get the name of the function of method that calls
-/// the lock, and therefore, anyone to call this macro should ensured that the caller method has the
-/// macro #[named]
-macro_rules! read_lock {
-    ($hummock_mgr:expr, $lock_name:ident) => {
-        async {
-            $hummock_mgr
-                .$lock_name
-                .read(&[function_name!(), stringify!($lock_name), "read"])
-                .await
-        }
-    };
-}
-pub(crate) use read_lock;
 use risingwave_hummock_sdk::compaction_group::{StateTableId, StaticCompactionGroupId};
-use risingwave_hummock_sdk::table_stats::{
-    add_prost_table_stats_map, purge_prost_table_stats, PbTableStatsMap,
-};
 use risingwave_object_store::object::{build_remote_object_store, ObjectError, ObjectStoreRef};
 use risingwave_pb::catalog::Table;
-use risingwave_pb::hummock::level_handler::RunningCompactTask;
 use risingwave_pb::meta::relation::RelationInfo;
 
-/// Acquire write lock of the lock with `lock_name`.
-/// The macro will use macro `function_name` to get the name of the function of method that calls
-/// the lock, and therefore, anyone to call this macro should ensured that the caller method has the
-/// macro #[named]
-macro_rules! write_lock {
-    ($hummock_mgr:expr, $lock_name:ident) => {
-        async {
-            $hummock_mgr
-                .$lock_name
-                .write(&[function_name!(), stringify!($lock_name), "write"])
-                .await
-        }
-    };
-}
-pub(crate) use write_lock;
-
 macro_rules! start_measure_real_process_timer {
-    ($hummock_mgr:expr) => {
+    ($hummock_mgr:expr, $func_name:literal) => {
         $hummock_mgr
             .metrics
             .hummock_manager_real_process_time
-            .with_label_values(&[function_name!()])
+            .with_label_values(&[$func_name])
             .start_timer()
     };
 }
@@ -228,72 +137,6 @@ pub(crate) use start_measure_real_process_timer;
 use crate::hummock::manager::compaction_group_manager::CompactionGroupManager;
 use crate::hummock::manager::worker::HummockManagerEventSender;
 
-pub static CANCEL_STATUS_SET: LazyLock<HashSet<TaskStatus>> = LazyLock::new(|| {
-    [
-        TaskStatus::ManualCanceled,
-        TaskStatus::SendFailCanceled,
-        TaskStatus::AssignFailCanceled,
-        TaskStatus::HeartbeatCanceled,
-        TaskStatus::InvalidGroupCanceled,
-        TaskStatus::NoAvailMemoryResourceCanceled,
-        TaskStatus::NoAvailCpuResourceCanceled,
-    ]
-    .into_iter()
-    .collect()
-});
-
-#[derive(Debug, Clone)]
-pub struct NewTableFragmentInfo {
-    pub table_id: TableId,
-    pub mv_table_id: Option<TableId>,
-    pub internal_table_ids: Vec<TableId>,
-}
-
-impl NewTableFragmentInfo {
-    pub fn state_table_ids(&self) -> impl Iterator<Item = TableId> + '_ {
-        self.mv_table_id
-            .iter()
-            .chain(self.internal_table_ids.iter())
-            .cloned()
-    }
-}
-
-pub struct CommitEpochInfo {
-    pub sstables: Vec<ExtendedSstableInfo>,
-    pub new_table_watermarks: HashMap<TableId, TableWatermarks>,
-    pub sst_to_context: HashMap<HummockSstableObjectId, HummockContextId>,
-    pub new_table_fragment_info: Option<NewTableFragmentInfo>,
-}
-
-impl CommitEpochInfo {
-    pub fn new(
-        sstables: Vec<ExtendedSstableInfo>,
-        new_table_watermarks: HashMap<TableId, TableWatermarks>,
-        sst_to_context: HashMap<HummockSstableObjectId, HummockContextId>,
-        new_table_fragment_info: Option<NewTableFragmentInfo>,
-    ) -> Self {
-        Self {
-            sstables,
-            new_table_watermarks,
-            sst_to_context,
-            new_table_fragment_info,
-        }
-    }
-
-    #[cfg(any(test, feature = "test"))]
-    pub(crate) fn for_test(
-        sstables: Vec<impl Into<ExtendedSstableInfo>>,
-        sst_to_context: HashMap<HummockSstableObjectId, HummockContextId>,
-    ) -> Self {
-        Self::new(
-            sstables.into_iter().map(Into::into).collect(),
-            HashMap::new(),
-            sst_to_context,
-            None,
-        )
-    }
-}
-
 impl HummockManager {
     pub async fn new(
         env: MetaSrvEnv,
@@ -305,7 +148,7 @@ impl HummockManager {
             Streaming<SubscribeCompactionEventRequest>,
         )>,
     ) -> Result<HummockManagerRef> {
-        let compaction_group_manager = Self::build_compaction_group_manager(&env).await?;
+        let compaction_group_manager = CompactionGroupManager::new(&env).await?;
         Self::new_impl(
             env,
             metadata_manager,
@@ -320,8 +163,8 @@ impl HummockManager {
     #[cfg(any(test, feature = "test"))]
     pub(super) async fn with_config(
         env: MetaSrvEnv,
-        cluster_manager: ClusterManagerRef,
-        fragment_manager: FragmentManagerRef,
+        cluster_manager: crate::manager::ClusterManagerRef,
+        fragment_manager: crate::manager::FragmentManagerRef,
         metrics: Arc<MetaMetrics>,
         compactor_manager: CompactorManagerRef,
         config: risingwave_pb::hummock::CompactionConfig,
@@ -331,10 +174,9 @@ impl HummockManager {
         )>,
     ) -> HummockManagerRef {
         use crate::manager::CatalogManager;
-        let compaction_group_manager =
-            Self::build_compaction_group_manager_with_config(&env, config)
-                .await
-                .unwrap();
+        let compaction_group_manager = CompactionGroupManager::new_with_config(&env, config)
+            .await
+            .unwrap();
         let catalog_manager = Arc::new(CatalogManager::new(env.clone()).await.unwrap());
         let metadata_manager =
             MetadataManager::new_v1(cluster_manager, catalog_manager, fragment_manager);
@@ -355,7 +197,7 @@ impl HummockManager {
         metadata_manager: MetadataManager,
         metrics: Arc<MetaMetrics>,
         compactor_manager: CompactorManagerRef,
-        compaction_group_manager: tokio::sync::RwLock<CompactionGroupManager>,
+        compaction_group_manager: CompactionGroupManager,
         compactor_streams_change_tx: UnboundedSender<(
             u32,
             Streaming<SubscribeCompactionEventRequest>,
@@ -374,7 +216,7 @@ impl HummockManager {
                 state_store_url.strip_prefix("hummock+").unwrap_or("memory"),
                 metrics.object_store_metric.clone(),
                 "Version Checkpoint",
-                object_store_config,
+                Arc::new(object_store_config),
             )
             .await,
         );
@@ -411,14 +253,25 @@ impl HummockManager {
             versioning: MonitoredRwLock::new(
                 metrics.hummock_manager_lock_time.clone(),
                 Default::default(),
+                "hummock_manager::versioning",
             ),
             compaction: MonitoredRwLock::new(
                 metrics.hummock_manager_lock_time.clone(),
                 Default::default(),
+                "hummock_manager::compaction",
+            ),
+            compaction_group_manager: MonitoredRwLock::new(
+                metrics.hummock_manager_lock_time.clone(),
+                compaction_group_manager,
+                "hummock_manager::compaction_group_manager",
+            ),
+            context_info: MonitoredRwLock::new(
+                metrics.hummock_manager_lock_time.clone(),
+                Default::default(),
+                "hummock_manager::context_info",
             ),
             metrics,
             metadata_manager,
-            compaction_group_manager,
             // compaction_request_channel: parking_lot::RwLock::new(None),
             compactor_manager,
             latest_snapshot: ArcSwap::from_pointee(HummockSnapshot {
@@ -426,6 +279,7 @@ impl HummockManager {
                 current_epoch: INVALID_EPOCH,
             }),
             event_sender: tx,
+            delete_object_tracker: Default::default(),
             object_store,
             version_checkpoint_path,
             version_archive_dir,
@@ -433,7 +287,6 @@ impl HummockManager {
             history_table_throughput: parking_lot::RwLock::new(HashMap::default()),
             compactor_streams_change_tx,
             compaction_state: CompactionState::new(),
-            group_to_table_vnode_partition: parking_lot::RwLock::new(HashMap::default()),
         };
         let instance = Arc::new(instance);
         instance.start_worker(rx).await;
@@ -444,18 +297,19 @@ impl HummockManager {
         Ok(instance)
     }
 
-    fn meta_store_ref(&self) -> MetaStoreImpl {
+    fn meta_store_ref(&self) -> &MetaStoreImpl {
         self.env.meta_store_ref()
     }
 
     /// Load state from meta store.
-    #[named]
     async fn load_meta_store_state(&self) -> Result<()> {
-        let mut compaction_guard = write_lock!(self, compaction).await;
-        let mut versioning_guard = write_lock!(self, versioning).await;
+        let mut compaction_guard = self.compaction.write().await;
+        let mut versioning_guard = self.versioning.write().await;
+        let mut context_info_guard = self.context_info.write().await;
         self.load_meta_store_state_impl(
-            compaction_guard.borrow_mut(),
-            versioning_guard.borrow_mut(),
+            &mut compaction_guard,
+            &mut versioning_guard,
+            &mut context_info_guard,
         )
         .await
     }
@@ -463,8 +317,9 @@ impl HummockManager {
     /// Load state from meta store.
     async fn load_meta_store_state_impl(
         &self,
-        compaction_guard: &mut RwLockWriteGuard<'_, Compaction>,
-        versioning_guard: &mut RwLockWriteGuard<'_, Versioning>,
+        compaction_guard: &mut Compaction,
+        versioning_guard: &mut Versioning,
+        context_info: &mut ContextInfo,
     ) -> Result<()> {
         use sea_orm::EntityTrait;
         let meta_store = self.meta_store_ref();
@@ -537,7 +392,7 @@ impl HummockManager {
                 .read()
                 .await
                 .default_compaction_config();
-            let checkpoint_version = create_init_version(default_compaction_config);
+            let checkpoint_version = HummockVersion::create_init_version(default_compaction_config);
             tracing::info!("init hummock version checkpoint");
             versioning_guard.checkpoint = HummockVersionCheckpoint {
                 version: checkpoint_version.clone(),
@@ -576,10 +431,9 @@ impl HummockManager {
             .into(),
         );
         versioning_guard.current_version = redo_state;
-        versioning_guard.branched_ssts = versioning_guard.current_version.build_branched_sst_info();
         versioning_guard.hummock_version_deltas = hummock_version_deltas;
 
-        versioning_guard.pinned_versions = match &meta_store {
+        context_info.pinned_versions = match &meta_store {
             MetaStoreImpl::Kv(meta_store) => HummockPinnedVersion::list(meta_store)
                 .await?
                 .into_iter()
@@ -594,7 +448,7 @@ impl HummockManager {
                 .collect(),
         };
 
-        versioning_guard.pinned_snapshots = match &meta_store {
+        context_info.pinned_snapshots = match &meta_store {
             MetaStoreImpl::Kv(meta_store) => HummockPinnedSnapshot::list(meta_store)
                 .await?
                 .into_iter()
@@ -609,2925 +463,134 @@ impl HummockManager {
                 .collect(),
         };
 
-        versioning_guard.objects_to_delete.clear();
+        self.delete_object_tracker.clear();
         // Not delete stale objects when archive is enabled
         if !self.env.opts.enable_hummock_data_archive {
-            versioning_guard.mark_objects_for_deletion();
+            versioning_guard.mark_objects_for_deletion(context_info, &self.delete_object_tracker);
         }
 
-        self.initial_compaction_group_config_after_load(versioning_guard)
-            .await?;
-
-        Ok(())
-    }
+        self.initial_compaction_group_config_after_load(
+            versioning_guard,
+            self.compaction_group_manager.write().await.deref_mut(),
+        )
+        .await?;
 
-    /// Caller should hold `versioning` lock, to sync with `HummockManager::release_contexts`.
-    async fn check_context_with_meta_node(&self, context_id: HummockContextId) -> Result<()> {
-        if context_id == META_NODE_ID {
-            // Using the preserved meta id is allowed.
-        } else if !self.check_context(context_id).await? {
-            // The worker is not found in cluster.
-            return Err(Error::InvalidContext(context_id));
-        }
         Ok(())
     }
 
-    /// Pin the current greatest hummock version. The pin belongs to `context_id`
-    /// and will be unpinned when `context_id` is invalidated.
-    #[named]
-    pub async fn pin_version(&self, context_id: HummockContextId) -> Result<HummockVersion> {
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        self.check_context_with_meta_node(context_id).await?;
-        let _timer = start_measure_real_process_timer!(self);
-        let versioning = versioning_guard.deref_mut();
-        let mut pinned_versions = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning.pinned_versions,)
-        );
-        let mut context_pinned_version = pinned_versions.new_entry_txn_or_default(
-            context_id,
-            HummockPinnedVersion {
-                context_id,
-                min_pinned_id: INVALID_VERSION_ID,
-            },
-        );
-        let version_id = versioning.current_version.id;
-        let ret = versioning.current_version.clone();
-        if context_pinned_version.min_pinned_id == INVALID_VERSION_ID
-            || context_pinned_version.min_pinned_id > version_id
-        {
-            context_pinned_version.min_pinned_id = version_id;
-            commit_multi_var!(self.meta_store_ref(), context_pinned_version)?;
-            trigger_pin_unpin_version_state(&self.metrics, &versioning.pinned_versions);
-        }
-
-        #[cfg(test)]
-        {
-            drop(versioning_guard);
-            self.check_state_consistency().await;
-        }
-
-        Ok(ret)
-    }
-
-    /// Unpin all pins which belongs to `context_id` and has an id which is older than
-    /// `unpin_before`. All versions >= `unpin_before` will be treated as if they are all pinned by
-    /// this `context_id` so they will not be vacuumed.
-    #[named]
-    pub async fn unpin_version_before(
+    pub async fn init_metadata_for_version_replay(
         &self,
-        context_id: HummockContextId,
-        unpin_before: HummockVersionId,
+        table_catalogs: Vec<Table>,
+        compaction_groups: Vec<PbCompactionGroupInfo>,
     ) -> Result<()> {
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        self.check_context_with_meta_node(context_id).await?;
-        let _timer = start_measure_real_process_timer!(self);
-        let versioning = versioning_guard.deref_mut();
-        let mut pinned_versions = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning.pinned_versions,)
-        );
-        let mut context_pinned_version = pinned_versions.new_entry_txn_or_default(
-            context_id,
-            HummockPinnedVersion {
-                context_id,
-                min_pinned_id: 0,
-            },
-        );
-        assert!(
-            context_pinned_version.min_pinned_id <= unpin_before,
-            "val must be monotonically non-decreasing. old = {}, new = {}.",
-            context_pinned_version.min_pinned_id,
-            unpin_before
-        );
-        context_pinned_version.min_pinned_id = unpin_before;
-        commit_multi_var!(self.meta_store_ref(), context_pinned_version)?;
-        trigger_pin_unpin_version_state(&self.metrics, &versioning.pinned_versions);
-
-        #[cfg(test)]
-        {
-            drop(versioning_guard);
-            self.check_state_consistency().await;
-        }
-
-        Ok(())
-    }
-
-    #[named]
-    pub async fn pin_specific_snapshot(
-        &self,
-        context_id: HummockContextId,
-        epoch: HummockEpoch,
-    ) -> Result<HummockSnapshot> {
-        let snapshot = self.latest_snapshot.load();
-        let mut guard = write_lock!(self, versioning).await;
-        self.check_context_with_meta_node(context_id).await?;
-        let mut pinned_snapshots = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut guard.pinned_snapshots,)
-        );
-        let mut context_pinned_snapshot = pinned_snapshots.new_entry_txn_or_default(
-            context_id,
-            HummockPinnedSnapshot {
-                context_id,
-                minimal_pinned_snapshot: INVALID_EPOCH,
-            },
-        );
-        let epoch_to_pin = std::cmp::min(epoch, snapshot.committed_epoch);
-        if context_pinned_snapshot.minimal_pinned_snapshot == INVALID_EPOCH {
-            context_pinned_snapshot.minimal_pinned_snapshot = epoch_to_pin;
-            commit_multi_var!(self.meta_store_ref(), context_pinned_snapshot)?;
-        }
-        Ok(HummockSnapshot::clone(&snapshot))
-    }
-
-    /// Make sure `max_committed_epoch` is pinned and return it.
-    #[named]
-    pub async fn pin_snapshot(&self, context_id: HummockContextId) -> Result<HummockSnapshot> {
-        let snapshot = self.latest_snapshot.load();
-        let mut guard = write_lock!(self, versioning).await;
-        self.check_context_with_meta_node(context_id).await?;
-        let _timer = start_measure_real_process_timer!(self);
-        let mut pinned_snapshots = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut guard.pinned_snapshots,)
-        );
-        let mut context_pinned_snapshot = pinned_snapshots.new_entry_txn_or_default(
-            context_id,
-            HummockPinnedSnapshot {
-                context_id,
-                minimal_pinned_snapshot: INVALID_EPOCH,
-            },
-        );
-        if context_pinned_snapshot.minimal_pinned_snapshot == INVALID_EPOCH {
-            context_pinned_snapshot.minimal_pinned_snapshot = snapshot.committed_epoch;
-            commit_multi_var!(self.meta_store_ref(), context_pinned_snapshot)?;
-            trigger_pin_unpin_snapshot_state(&self.metrics, &guard.pinned_snapshots);
-        }
-        Ok(HummockSnapshot::clone(&snapshot))
-    }
-
-    pub fn latest_snapshot(&self) -> HummockSnapshot {
-        let snapshot = self.latest_snapshot.load();
-        HummockSnapshot::clone(&snapshot)
-    }
-
-    #[named]
-    pub async fn unpin_snapshot(&self, context_id: HummockContextId) -> Result<()> {
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        self.check_context_with_meta_node(context_id).await?;
-        let _timer = start_measure_real_process_timer!(self);
-        let mut pinned_snapshots = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning_guard.pinned_snapshots,)
-        );
-        let release_snapshot = pinned_snapshots.remove(context_id);
-        if release_snapshot.is_some() {
-            commit_multi_var!(self.meta_store_ref(), pinned_snapshots)?;
-            trigger_pin_unpin_snapshot_state(&self.metrics, &versioning_guard.pinned_snapshots);
+        for table in &table_catalogs {
+            table.insert(self.env.meta_store().as_kv()).await?;
         }
-
-        #[cfg(test)]
-        {
-            drop(versioning_guard);
-            self.check_state_consistency().await;
+        for group in &compaction_groups {
+            assert!(
+                group.id == StaticCompactionGroupId::NewCompactionGroup as u64
+                    || (group.id >= StaticCompactionGroupId::StateDefault as u64
+                    && group.id <= StaticCompactionGroupId::MaterializedView as u64),
+                "compaction group id should be either NewCompactionGroup to create new one, or predefined static ones."
+            );
         }
 
-        Ok(())
-    }
+        let mut compaction_group_manager = self.compaction_group_manager.write().await;
+        let mut compaction_groups_txn = compaction_group_manager.start_compaction_groups_txn();
+        for group in &compaction_groups {
+            let mut pairs = vec![];
+            for table_id in group.member_table_ids.clone() {
+                pairs.push((table_id as StateTableId, group.id));
+            }
+            let group_config = group.compaction_config.clone().unwrap();
+            compaction_groups_txn.create_compaction_groups(group.id, Arc::new(group_config));
 
-    /// Unpin all snapshots smaller than specified epoch for current context.
-    #[named]
-    pub async fn unpin_snapshot_before(
-        &self,
-        context_id: HummockContextId,
-        hummock_snapshot: HummockSnapshot,
-    ) -> Result<()> {
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        self.check_context_with_meta_node(context_id).await?;
-        let _timer = start_measure_real_process_timer!(self);
-        // Use the max_committed_epoch in storage as the snapshot ts so only committed changes are
-        // visible in the snapshot.
-        let max_committed_epoch = versioning_guard.current_version.max_committed_epoch;
-        // Ensure the unpin will not clean the latest one.
-        let snapshot_committed_epoch = hummock_snapshot.committed_epoch;
-        #[cfg(not(test))]
-        {
-            assert!(snapshot_committed_epoch <= max_committed_epoch);
+            self.register_table_ids_for_test(&pairs).await?;
+            tracing::info!("Registered table ids {:?}", pairs);
         }
-        let last_read_epoch = std::cmp::min(snapshot_committed_epoch, max_committed_epoch);
 
-        let mut pinned_snapshots = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning_guard.pinned_snapshots,)
-        );
-        let mut context_pinned_snapshot = pinned_snapshots.new_entry_txn_or_default(
-            context_id,
-            HummockPinnedSnapshot {
-                context_id,
-                minimal_pinned_snapshot: INVALID_EPOCH,
-            },
-        );
+        commit_multi_var!(self.meta_store_ref(), compaction_groups_txn)?;
 
-        // Unpin the snapshots pinned by meta but frontend doesn't know. Also equal to unpin all
-        // epochs below specific watermark.
-        if context_pinned_snapshot.minimal_pinned_snapshot < last_read_epoch
-            || context_pinned_snapshot.minimal_pinned_snapshot == INVALID_EPOCH
-        {
-            context_pinned_snapshot.minimal_pinned_snapshot = last_read_epoch;
-            commit_multi_var!(self.meta_store_ref(), context_pinned_snapshot)?;
-            trigger_pin_unpin_snapshot_state(&self.metrics, &versioning_guard.pinned_snapshots);
+        // Notify that tables have created
+        for table in table_catalogs {
+            self.env
+                .notification_manager()
+                .notify_hummock_relation_info(Operation::Add, RelationInfo::Table(table.clone()))
+                .await;
+            self.env
+                .notification_manager()
+                .notify_compactor_relation_info(Operation::Add, RelationInfo::Table(table))
+                .await;
         }
 
-        #[cfg(test)]
-        {
-            drop(versioning_guard);
-            self.check_state_consistency().await;
+        tracing::info!("Inited compaction groups:");
+        for group in compaction_groups {
+            tracing::info!("{:?}", group);
         }
-
         Ok(())
     }
 
-    #[named]
-    pub async fn get_compact_tasks_impl(
+    /// Replay a version delta to current hummock version.
+    /// Returns the `version_id`, `max_committed_epoch` of the new version and the modified
+    /// compaction groups
+    pub async fn replay_version_delta(
         &self,
-        compaction_groups: Vec<CompactionGroupId>,
-        max_select_count: usize,
-        selector: &mut Box<dyn CompactionSelector>,
-    ) -> Result<(Vec<CompactTask>, Vec<CompactionGroupId>)> {
-        // TODO: `get_all_table_options` will hold catalog_manager async lock, to avoid holding the
-        // lock in compaction_guard, take out all table_options in advance there may be a
-        // waste of resources here, need to add a more efficient filter in catalog_manager
-        let deterministic_mode = self.env.opts.compaction_deterministic_test;
-        let all_table_id_to_option = self
-            .metadata_manager
-            .get_all_table_options()
-            .await
-            .map_err(|err| Error::MetaStore(err.into()))?;
-
-        let mut compaction_guard = write_lock!(self, compaction).await;
-        let compaction = compaction_guard.deref_mut();
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        let versioning = versioning_guard.deref_mut();
-
-        let _timer = start_measure_real_process_timer!(self);
-
-        let mut current_version = versioning.current_version.clone();
-        let start_time = Instant::now();
-        let max_committed_epoch = current_version.max_committed_epoch;
-        let watermark = versioning
-            .pinned_snapshots
-            .values()
-            .map(|v| v.minimal_pinned_snapshot)
-            .fold(max_committed_epoch, std::cmp::min);
-        let last_apply_version_id = current_version.id;
-
-        let mut compaction_statuses = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut compaction.compaction_statuses)
-        );
-
-        let mut compact_task_assignment = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut compaction.compact_task_assignment)
-        );
-
-        let mut hummock_version_deltas = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning.hummock_version_deltas)
-        );
-        let mut branched_ssts = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning.branched_ssts)
-        );
-
-        let mut unschedule_groups = vec![];
-        let mut trivial_tasks = vec![];
-        let mut pick_tasks = vec![];
-        let developer_config = Arc::new(CompactionDeveloperConfig::new_from_meta_opts(
-            &self.env.opts,
-        ));
-        const MAX_TRIVIAL_MOVE_TASK_COUNT: usize = 256;
-        'outside: for compaction_group_id in compaction_groups {
-            if pick_tasks.len() >= max_select_count {
-                break;
-            }
-
-            if current_version.levels.get(&compaction_group_id).is_none() {
-                continue;
-            }
-
-            // When the last table of a compaction group is deleted, the compaction group (and its
-            // config) is destroyed as well. Then a compaction task for this group may come later and
-            // cannot find its config.
-            let group_config = match self
-                .compaction_group_manager
-                .read()
-                .await
-                .try_get_compaction_group_config(compaction_group_id)
-            {
-                Some(config) => config,
-                None => continue,
-            };
-            // StoredIdGenerator already implements ids pre-allocation by ID_PREALLOCATE_INTERVAL.
-            let task_id = next_compaction_task_id(&self.env).await?;
-
-            if !compaction_statuses.contains_key(&compaction_group_id) {
-                // lazy initialize.
-                compaction_statuses.insert(
-                    compaction_group_id,
-                    CompactStatus::new(
-                        compaction_group_id,
-                        group_config.compaction_config.max_level,
-                    ),
-                );
-            }
-            let mut compact_status = compaction_statuses.get_mut(compaction_group_id).unwrap();
-
-            let can_trivial_move = matches!(selector.task_type(), TaskType::Dynamic)
-                || matches!(selector.task_type(), TaskType::Emergency);
-
-            let mut stats = LocalSelectorStatistic::default();
-            let member_table_ids = current_version
-                .get_compaction_group_levels(compaction_group_id)
-                .member_table_ids
-                .clone();
-
-            let mut table_id_to_option: HashMap<u32, _> = HashMap::default();
-
-            for table_id in &member_table_ids {
-                if let Some(opts) = all_table_id_to_option.get(table_id) {
-                    table_id_to_option.insert(*table_id, *opts);
-                }
-            }
-
-            let table_to_vnode_partition = match self
-                .group_to_table_vnode_partition
-                .read()
-                .get(&compaction_group_id)
-            {
-                Some(table_to_vnode_partition) => table_to_vnode_partition.clone(),
-                None => BTreeMap::default(),
-            };
-
-            while let Some(compact_task) = compact_status.get_compact_task(
-                current_version.get_compaction_group_levels(compaction_group_id),
-                task_id as HummockCompactionTaskId,
-                &group_config,
-                &mut stats,
-                selector,
-                table_id_to_option.clone(),
-                developer_config.clone(),
-            ) {
-                let target_level_id = compact_task.input.target_level as u32;
-
-                let compression_algorithm = match compact_task.compression_algorithm.as_str() {
-                    "Lz4" => 1,
-                    "Zstd" => 2,
-                    _ => 0,
-                };
-                let vnode_partition_count = compact_task.input.vnode_partition_count;
-                use risingwave_hummock_sdk::prost_key_range::KeyRangeExt;
-
-                let mut compact_task = CompactTask {
-                    input_ssts: compact_task.input.input_levels,
-                    splits: vec![risingwave_pb::hummock::KeyRange::inf()],
-                    watermark,
-                    sorted_output_ssts: vec![],
-                    task_id,
-                    target_level: target_level_id,
-                    // only gc delete keys in last level because there may be older version in more bottom
-                    // level.
-                    gc_delete_keys: current_version
-                        .get_compaction_group_levels(compaction_group_id)
-                        .is_last_level(target_level_id),
-                    base_level: compact_task.base_level as u32,
-                    task_status: TaskStatus::Pending as i32,
-                    compaction_group_id: group_config.group_id,
-                    existing_table_ids: member_table_ids.clone(),
-                    compression_algorithm,
-                    target_file_size: compact_task.target_file_size,
-                    table_options: table_id_to_option
-                        .iter()
-                        .map(|(table_id, table_option)| {
-                            (*table_id, TableOption::from(table_option))
-                        })
-                        .collect(),
-                    current_epoch_time: Epoch::now().0,
-                    compaction_filter_mask: group_config.compaction_config.compaction_filter_mask,
-                    target_sub_level_id: compact_task.input.target_sub_level_id,
-                    task_type: compact_task.compaction_task_type as i32,
-                    split_weight_by_vnode: vnode_partition_count,
-                    ..Default::default()
-                };
-
-                let is_trivial_reclaim = CompactStatus::is_trivial_reclaim(&compact_task);
-                let is_trivial_move = CompactStatus::is_trivial_move_task(&compact_task);
-                if is_trivial_reclaim || (is_trivial_move && can_trivial_move) {
-                    let log_label = if is_trivial_reclaim {
-                        "TrivialReclaim"
-                    } else {
-                        "TrivialMove"
-                    };
-                    let label = if is_trivial_reclaim {
-                        "trivial-space-reclaim"
-                    } else {
-                        "trivial-move"
-                    };
-
-                    tracing::debug!(
-                        "{} for compaction group {}: input: {:?}, cost time: {:?}",
-                        log_label,
-                        compact_task.compaction_group_id,
-                        compact_task.input_ssts,
-                        start_time.elapsed()
-                    );
-                    compact_task.set_task_status(TaskStatus::Success);
-                    compact_status.report_compact_task(&compact_task);
-                    if !is_trivial_reclaim {
-                        compact_task.sorted_output_ssts =
-                            compact_task.input_ssts[0].table_infos.clone();
-                    }
-                    self.metrics
-                        .compact_frequency
-                        .with_label_values(&[
-                            label,
-                            &compact_task.compaction_group_id.to_string(),
-                            selector.task_type().as_str_name(),
-                            "SUCCESS",
-                        ])
-                        .inc();
-                    let version_delta = gen_version_delta(
-                        &mut hummock_version_deltas,
-                        &mut branched_ssts,
-                        &current_version,
-                        &compact_task,
-                        deterministic_mode,
-                    );
-                    current_version.apply_version_delta(&version_delta);
-                    trivial_tasks.push(compact_task);
-                    if trivial_tasks.len() >= MAX_TRIVIAL_MOVE_TASK_COUNT {
-                        break 'outside;
-                    }
-                } else {
-                    if group_config.compaction_config.split_weight_by_vnode > 0 {
-                        for table_id in &compact_task.existing_table_ids {
-                            compact_task
-                                .table_vnode_partition
-                                .insert(*table_id, vnode_partition_count);
-                        }
-                    } else {
-                        compact_task.table_vnode_partition = table_to_vnode_partition.clone();
-                    }
-                    compact_task
-                        .table_vnode_partition
-                        .retain(|table_id, _| compact_task.existing_table_ids.contains(table_id));
-                    compact_task.table_watermarks = current_version
-                        .safe_epoch_table_watermarks(&compact_task.existing_table_ids);
-
-                    if self.env.opts.enable_dropped_column_reclaim {
-                        // TODO: get all table schemas for all tables in once call to avoid acquiring lock and await.
-                        compact_task.table_schemas = match self.metadata_manager() {
-                            MetadataManager::V1(mgr) => mgr
-                                .catalog_manager
-                                .get_versioned_table_schemas(&compact_task.existing_table_ids)
-                                .await
-                                .into_iter()
-                                .map(|(table_id, column_ids)| {
-                                    (table_id, TableSchema { column_ids })
-                                })
-                                .collect(),
-                            MetadataManager::V2(_) => {
-                                // TODO #13952: support V2
-                                BTreeMap::default()
-                            }
-                        };
-                    }
-
-                    compact_task_assignment.insert(
-                        compact_task.task_id,
-                        CompactTaskAssignment {
-                            compact_task: Some(compact_task.clone()),
-                            context_id: META_NODE_ID, // deprecated
-                        },
-                    );
-
-                    pick_tasks.push(compact_task);
-                    break;
-                }
-
-                stats.report_to_metrics(compaction_group_id, self.metrics.as_ref());
-                stats = LocalSelectorStatistic::default();
-            }
-            if pick_tasks
-                .last()
-                .map(|task| task.compaction_group_id != compaction_group_id)
-                .unwrap_or(true)
-            {
-                unschedule_groups.push(compaction_group_id);
-            }
-
-            stats.report_to_metrics(compaction_group_id, self.metrics.as_ref());
-        }
-
-        if !trivial_tasks.is_empty() {
-            commit_multi_var!(
-                self.meta_store_ref(),
-                compaction_statuses,
-                compact_task_assignment,
-                hummock_version_deltas
-            )?;
-            branched_ssts.commit_memory();
-
-            trigger_version_stat(&self.metrics, &current_version);
-            versioning.current_version = current_version;
-            trigger_delta_log_stats(&self.metrics, versioning.hummock_version_deltas.len());
-            self.notify_stats(&versioning.version_stats);
-            self.notify_version_deltas(versioning, last_apply_version_id);
-            self.metrics
-                .compact_task_batch_count
-                .with_label_values(&["batch_trivial_move"])
-                .observe(trivial_tasks.len() as f64);
-            drop(versioning_guard);
-        } else {
-            // We are using a single transaction to ensure that each task has progress when it is
-            // created.
-            drop(versioning_guard);
-            commit_multi_var!(
-                self.meta_store_ref(),
-                compaction_statuses,
-                compact_task_assignment
-            )?;
-        }
-        drop(compaction_guard);
-        if !pick_tasks.is_empty() {
-            self.metrics
-                .compact_task_batch_count
-                .with_label_values(&["batch_get_compact_task"])
-                .observe(pick_tasks.len() as f64);
-        }
-
-        for compact_task in &mut pick_tasks {
-            let compaction_group_id = compact_task.compaction_group_id;
-
-            // Initiate heartbeat for the task to track its progress.
-            self.compactor_manager
-                .initiate_task_heartbeat(compact_task.clone());
-
-            // this task has been finished.
-            compact_task.set_task_status(TaskStatus::Pending);
-            let compact_task_statistics = statistics_compact_task(compact_task);
-
-            let level_type_label = build_compact_task_level_type_metrics_label(
-                compact_task.input_ssts[0].level_idx as usize,
-                compact_task.input_ssts.last().unwrap().level_idx as usize,
-            );
-
-            let level_count = compact_task.input_ssts.len();
-            if compact_task.input_ssts[0].level_idx == 0 {
-                self.metrics
-                    .l0_compact_level_count
-                    .with_label_values(&[&compaction_group_id.to_string(), &level_type_label])
-                    .observe(level_count as _);
-            }
-
-            self.metrics
-                .compact_task_size
-                .with_label_values(&[&compaction_group_id.to_string(), &level_type_label])
-                .observe(compact_task_statistics.total_file_size as _);
-
-            self.metrics
-                .compact_task_size
-                .with_label_values(&[
-                    &compaction_group_id.to_string(),
-                    &format!("{} uncompressed", level_type_label),
-                ])
-                .observe(compact_task_statistics.total_uncompressed_file_size as _);
-
-            self.metrics
-                .compact_task_file_count
-                .with_label_values(&[&compaction_group_id.to_string(), &level_type_label])
-                .observe(compact_task_statistics.total_file_count as _);
-
-            tracing::trace!(
-                    "For compaction group {}: pick up {} {} sub_level in level {} to compact to target {}. cost time: {:?} compact_task_statistics {:?}",
-                    compaction_group_id,
-                    level_count,
-                    compact_task.input_ssts[0].level_type().as_str_name(),
-                    compact_task.input_ssts[0].level_idx,
-                    compact_task.target_level,
-                    start_time.elapsed(),
-                    compact_task_statistics
-                );
-        }
-
-        #[cfg(test)]
-        {
-            self.check_state_consistency().await;
-        }
-        pick_tasks.extend(trivial_tasks);
-        Ok((pick_tasks, unschedule_groups))
-    }
+        mut version_delta: HummockVersionDelta,
+    ) -> Result<(HummockVersion, Vec<CompactionGroupId>)> {
+        let mut versioning_guard = self.versioning.write().await;
+        // ensure the version id is ascending after replay
+        version_delta.id = versioning_guard.current_version.next_version_id();
+        version_delta.prev_id = versioning_guard.current_version.id;
+        versioning_guard
+            .current_version
+            .apply_version_delta(&version_delta);
 
-    /// Cancels a compaction task no matter it's assigned or unassigned.
-    pub async fn cancel_compact_task(&self, task_id: u64, task_status: TaskStatus) -> Result<bool> {
-        fail_point!("fp_cancel_compact_task", |_| Err(Error::MetaStore(
-            anyhow::anyhow!("failpoint metastore err")
-        )));
-        let ret = self
-            .cancel_compact_task_impl(vec![task_id], task_status)
-            .await?;
-        Ok(ret[0])
+        let version_new = versioning_guard.current_version.clone();
+        let compaction_group_ids = version_delta.group_deltas.keys().cloned().collect_vec();
+        Ok((version_new, compaction_group_ids))
     }
 
-    pub async fn cancel_compact_tasks(
-        &self,
-        tasks: Vec<u64>,
-        task_status: TaskStatus,
-    ) -> Result<Vec<bool>> {
-        self.cancel_compact_task_impl(tasks, task_status).await
+    pub async fn disable_commit_epoch(&self) -> HummockVersion {
+        let mut versioning_guard = self.versioning.write().await;
+        versioning_guard.disable_commit_epochs = true;
+        versioning_guard.current_version.clone()
     }
 
-    pub async fn cancel_compact_task_impl(
-        &self,
-        task_ids: Vec<u64>,
-        task_status: TaskStatus,
-    ) -> Result<Vec<bool>> {
-        assert!(CANCEL_STATUS_SET.contains(&task_status));
-        let tasks = task_ids
-            .into_iter()
-            .map(|task_id| ReportTask {
-                task_id,
-                task_status: task_status as i32,
-                sorted_output_ssts: vec![],
-                table_stats_change: HashMap::default(),
-            })
-            .collect_vec();
-        let rets = self.report_compact_tasks(tasks).await?;
-        #[cfg(test)]
-        {
-            self.check_state_consistency().await;
-        }
-        Ok(rets)
+    pub fn metadata_manager(&self) -> &MetadataManager {
+        &self.metadata_manager
     }
+}
 
-    pub async fn get_compact_tasks(
-        &self,
-        mut compaction_groups: Vec<CompactionGroupId>,
-        max_select_count: usize,
-        selector: &mut Box<dyn CompactionSelector>,
-    ) -> Result<(Vec<CompactTask>, Vec<CompactionGroupId>)> {
-        fail_point!("fp_get_compact_task", |_| Err(Error::MetaStore(
-            anyhow::anyhow!("failpoint metastore error")
-        )));
-        compaction_groups.shuffle(&mut thread_rng());
-        let (mut tasks, groups) = self
-            .get_compact_tasks_impl(compaction_groups, max_select_count, selector)
-            .await?;
-        tasks.retain(|task| {
-            if task.task_status() == TaskStatus::Success {
-                debug_assert!(
-                    CompactStatus::is_trivial_reclaim(task)
-                        || CompactStatus::is_trivial_move_task(task)
-                );
-                false
-            } else {
-                true
+async fn write_exclusive_cluster_id(
+    state_store_dir: &str,
+    cluster_id: ClusterId,
+    object_store: ObjectStoreRef,
+) -> Result<()> {
+    const CLUSTER_ID_DIR: &str = "cluster_id";
+    const CLUSTER_ID_NAME: &str = "0";
+    let cluster_id_dir = format!("{}/{}/", state_store_dir, CLUSTER_ID_DIR);
+    let cluster_id_full_path = format!("{}{}", cluster_id_dir, CLUSTER_ID_NAME);
+    match object_store.read(&cluster_id_full_path, ..).await {
+        Ok(stored_cluster_id) => {
+            let stored_cluster_id = String::from_utf8(stored_cluster_id.to_vec()).unwrap();
+            if cluster_id.deref() == stored_cluster_id {
+                return Ok(());
             }
-        });
-        Ok((tasks, groups))
-    }
-
-    pub async fn get_compact_task(
-        &self,
-        compaction_group_id: CompactionGroupId,
-        selector: &mut Box<dyn CompactionSelector>,
-    ) -> Result<Option<CompactTask>> {
-        fail_point!("fp_get_compact_task", |_| Err(Error::MetaStore(
-            anyhow::anyhow!("failpoint metastore error")
-        )));
 
-        let (normal_tasks, _) = self
-            .get_compact_tasks_impl(vec![compaction_group_id], 1, selector)
-            .await?;
-        for task in normal_tasks {
-            if task.task_status() != TaskStatus::Success {
-                return Ok(Some(task));
-            }
-            debug_assert!(
-                CompactStatus::is_trivial_reclaim(&task)
-                    || CompactStatus::is_trivial_move_task(&task)
-            );
+            Err(ObjectError::internal(format!(
+                "Data directory is already used by another cluster with id {:?}, path {}.",
+                stored_cluster_id, cluster_id_full_path,
+            ))
+            .into())
         }
-        Ok(None)
-    }
-
-    pub async fn manual_get_compact_task(
-        &self,
-        compaction_group_id: CompactionGroupId,
-        manual_compaction_option: ManualCompactionOption,
-    ) -> Result<Option<CompactTask>> {
-        let mut selector: Box<dyn CompactionSelector> =
-            Box::new(ManualCompactionSelector::new(manual_compaction_option));
-        self.get_compact_task(compaction_group_id, &mut selector)
-            .await
-    }
-
-    fn is_compact_task_expired(
-        compact_task: &CompactTask,
-        branched_ssts: &BTreeMap<HummockSstableObjectId, BranchedSstInfo>,
-    ) -> bool {
-        for input_level in compact_task.get_input_ssts() {
-            for table_info in input_level.get_table_infos() {
-                if let Some(mp) = branched_ssts.get(&table_info.object_id) {
-                    if mp
-                        .get(&compact_task.compaction_group_id)
-                        .map_or(true, |sst_id| *sst_id != table_info.sst_id)
-                    {
-                        return true;
-                    }
-                }
-            }
-        }
-        false
-    }
-
-    pub async fn report_compact_task(
-        &self,
-        task_id: u64,
-        task_status: TaskStatus,
-        sorted_output_ssts: Vec<SstableInfo>,
-        table_stats_change: Option<PbTableStatsMap>,
-    ) -> Result<bool> {
-        let rets = self
-            .report_compact_tasks(vec![ReportTask {
-                task_id,
-                task_status: task_status as i32,
-                sorted_output_ssts,
-                table_stats_change: table_stats_change.unwrap_or_default(),
-            }])
-            .await?;
-        Ok(rets[0])
-    }
-
-    /// Finishes or cancels a compaction task, according to `task_status`.
-    ///
-    /// If `context_id` is not None, its validity will be checked when writing meta store.
-    /// Its ownership of the task is checked as well.
-    ///
-    /// Return Ok(false) indicates either the task is not found,
-    /// or the task is not owned by `context_id` when `context_id` is not None.
-    #[named]
-    pub async fn report_compact_tasks(&self, report_tasks: Vec<ReportTask>) -> Result<Vec<bool>> {
-        let mut guard = write_lock!(self, compaction).await;
-        let deterministic_mode = self.env.opts.compaction_deterministic_test;
-        let compaction = guard.deref_mut();
-        let start_time = Instant::now();
-        let original_keys = compaction.compaction_statuses.keys().cloned().collect_vec();
-        let mut compact_statuses = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut compaction.compaction_statuses,)
-        );
-        let mut rets = vec![false; report_tasks.len()];
-        let mut compact_task_assignment = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut compaction.compact_task_assignment,)
-        );
-        // The compaction task is finished.
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        let versioning = versioning_guard.deref_mut();
-        let _timer = start_measure_real_process_timer!(self);
-
-        let mut current_version = versioning.current_version.clone();
-        // purge stale compact_status
-        for group_id in original_keys {
-            if !current_version.levels.contains_key(&group_id) {
-                compact_statuses.remove(group_id);
-            }
-        }
-        let mut tasks = vec![];
-
-        let mut hummock_version_deltas = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning.hummock_version_deltas,)
-        );
-        let mut branched_ssts = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning.branched_ssts)
-        );
-
-        let mut version_stats = create_trx_wrapper!(
-            self.meta_store_ref(),
-            VarTransactionWrapper,
-            VarTransaction::new(&mut versioning.version_stats)
-        );
-        let mut success_count = 0;
-        let last_version_id = current_version.id;
-        for (idx, task) in report_tasks.into_iter().enumerate() {
-            rets[idx] = true;
-            let mut compact_task = match compact_task_assignment.remove(task.task_id) {
-                Some(compact_task) => compact_task.compact_task.unwrap(),
-                None => {
-                    tracing::warn!("{}", format!("compact task {} not found", task.task_id));
-                    rets[idx] = false;
-                    continue;
-                }
-            };
-
-            {
-                // apply result
-                compact_task.task_status = task.task_status;
-                compact_task.sorted_output_ssts = task.sorted_output_ssts;
-            }
-
-            match compact_statuses.get_mut(compact_task.compaction_group_id) {
-                Some(mut compact_status) => {
-                    compact_status.report_compact_task(&compact_task);
-                }
-                None => {
-                    compact_task.set_task_status(TaskStatus::InvalidGroupCanceled);
-                }
-            }
-
-            let input_sst_ids: HashSet<u64> = compact_task
-                .input_ssts
-                .iter()
-                .flat_map(|level| level.table_infos.iter().map(|sst| sst.sst_id))
-                .collect();
-            let input_level_ids: Vec<u32> = compact_task
-                .input_ssts
-                .iter()
-                .map(|level| level.level_idx)
-                .collect();
-            let is_success = if let TaskStatus::Success = compact_task.task_status() {
-                // if member_table_ids changes, the data of sstable may stale.
-                let is_expired =
-                    Self::is_compact_task_expired(&compact_task, branched_ssts.tree_ref());
-                if is_expired {
-                    compact_task.set_task_status(TaskStatus::InputOutdatedCanceled);
-                    false
-                } else {
-                    let group = current_version
-                        .levels
-                        .get(&compact_task.compaction_group_id)
-                        .unwrap();
-                    let input_exist =
-                        group.check_deleted_sst_exist(&input_level_ids, input_sst_ids);
-                    if !input_exist {
-                        compact_task.set_task_status(TaskStatus::InvalidGroupCanceled);
-                        warn!(
-                            "The task may be expired because of group split, task:\n {:?}",
-                            compact_task_to_string(&compact_task)
-                        );
-                    }
-                    input_exist
-                }
-            } else {
-                false
-            };
-            if is_success {
-                success_count += 1;
-                let version_delta = gen_version_delta(
-                    &mut hummock_version_deltas,
-                    &mut branched_ssts,
-                    &current_version,
-                    &compact_task,
-                    deterministic_mode,
-                );
-                // apply version delta before we persist this change. If it causes panic we can
-                // recover to a correct state after restarting meta-node.
-                current_version.apply_version_delta(&version_delta);
-                if purge_prost_table_stats(&mut version_stats.table_stats, &current_version) {
-                    self.metrics.version_stats.reset();
-                    versioning.local_metrics.clear();
-                }
-                add_prost_table_stats_map(&mut version_stats.table_stats, &task.table_stats_change);
-                trigger_local_table_stat(
-                    &self.metrics,
-                    &mut versioning.local_metrics,
-                    &version_stats,
-                    &task.table_stats_change,
-                );
-            }
-            tasks.push(compact_task);
-        }
-        if success_count > 0 {
-            commit_multi_var!(
-                self.meta_store_ref(),
-                compact_statuses,
-                compact_task_assignment,
-                hummock_version_deltas,
-                version_stats
-            )?;
-            branched_ssts.commit_memory();
-
-            trigger_version_stat(&self.metrics, &current_version);
-            trigger_delta_log_stats(&self.metrics, versioning.hummock_version_deltas.len());
-            self.notify_stats(&versioning.version_stats);
-            versioning.current_version = current_version;
-
-            if !deterministic_mode {
-                self.notify_version_deltas(versioning, last_version_id);
-            }
-
-            self.metrics
-                .compact_task_batch_count
-                .with_label_values(&["batch_report_task"])
-                .observe(success_count as f64);
-        } else {
-            // The compaction task is cancelled or failed.
-            commit_multi_var!(
-                self.meta_store_ref(),
-                compact_statuses,
-                compact_task_assignment
-            )?;
-        }
-        let mut success_groups = vec![];
-        for compact_task in tasks {
-            let task_status = compact_task.task_status();
-            let task_status_label = task_status.as_str_name();
-            let task_type_label = compact_task.task_type().as_str_name();
-
-            self.compactor_manager
-                .remove_task_heartbeat(compact_task.task_id);
-
-            self.metrics
-                .compact_frequency
-                .with_label_values(&[
-                    "normal",
-                    &compact_task.compaction_group_id.to_string(),
-                    task_type_label,
-                    task_status_label,
-                ])
-                .inc();
-
-            tracing::trace!(
-                "Reported compaction task. {}. cost time: {:?}",
-                compact_task_to_string(&compact_task),
-                start_time.elapsed(),
-            );
-
-            trigger_sst_stat(
-                &self.metrics,
-                compaction
-                    .compaction_statuses
-                    .get(&compact_task.compaction_group_id),
-                &versioning_guard.current_version,
-                compact_task.compaction_group_id,
-            );
-
-            if !deterministic_mode
-                && (matches!(compact_task.task_type(), compact_task::TaskType::Dynamic)
-                    || matches!(compact_task.task_type(), compact_task::TaskType::Emergency))
-            {
-                // only try send Dynamic compaction
-                self.try_send_compaction_request(
-                    compact_task.compaction_group_id,
-                    compact_task::TaskType::Dynamic,
-                );
-            }
-
-            if task_status == TaskStatus::Success {
-                success_groups.push(compact_task.compaction_group_id);
-            }
-        }
-        drop(versioning_guard);
-        if !success_groups.is_empty() {
-            self.try_update_write_limits(&success_groups).await;
-        }
-        Ok(rets)
-    }
-
-    /// Caller should ensure `epoch` > `max_committed_epoch`
-    #[named]
-    pub async fn commit_epoch(
-        &self,
-        epoch: HummockEpoch,
-        commit_info: CommitEpochInfo,
-    ) -> Result<Option<HummockSnapshot>> {
-        let CommitEpochInfo {
-            mut sstables,
-            new_table_watermarks,
-            sst_to_context,
-            new_table_fragment_info,
-        } = commit_info;
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        let _timer = start_measure_real_process_timer!(self);
-        // Prevent commit new epochs if this flag is set
-        if versioning_guard.disable_commit_epochs {
-            return Ok(None);
-        }
-
-        let versioning = versioning_guard.deref_mut();
-        self.commit_epoch_sanity_check(
-            epoch,
-            &sstables,
-            &sst_to_context,
-            &versioning.current_version,
-        )
-        .await?;
-
-        // Consume and aggregate table stats.
-        let mut table_stats_change = PbTableStatsMap::default();
-        for s in &mut sstables {
-            add_prost_table_stats_map(&mut table_stats_change, &std::mem::take(&mut s.table_stats));
-        }
-
-        let old_version: &HummockVersion = &versioning.current_version;
-        let mut new_version_delta = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapEntryTransactionWrapper,
-            BTreeMapEntryTransaction::new_insert(
-                &mut versioning.hummock_version_deltas,
-                old_version.id + 1,
-                build_version_delta_after_version(old_version),
-            )
-        );
-        new_version_delta.max_committed_epoch = epoch;
-        new_version_delta.new_table_watermarks = new_table_watermarks;
-
-        let mut table_compaction_group_mapping = old_version.build_compaction_group_info();
-
-        // Add new table
-        if let Some(new_fragment_table_info) = new_table_fragment_info {
-            if !new_fragment_table_info.internal_table_ids.is_empty() {
-                if let Some(levels) = old_version
-                    .levels
-                    .get(&(StaticCompactionGroupId::StateDefault as u64))
-                {
-                    for table_id in &new_fragment_table_info.internal_table_ids {
-                        if levels.member_table_ids.contains(&table_id.table_id) {
-                            return Err(Error::CompactionGroup(format!(
-                                "table {} already in group {}",
-                                table_id,
-                                StaticCompactionGroupId::StateDefault as u64
-                            )));
-                        }
-                    }
-                }
-
-                let group_deltas = &mut new_version_delta
-                    .group_deltas
-                    .entry(StaticCompactionGroupId::StateDefault as u64)
-                    .or_default()
-                    .group_deltas;
-                group_deltas.push(GroupDelta {
-                    delta_type: Some(DeltaType::GroupMetaChange(GroupMetaChange {
-                        table_ids_add: new_fragment_table_info
-                            .internal_table_ids
-                            .iter()
-                            .map(|table_id| table_id.table_id)
-                            .collect(),
-                        ..Default::default()
-                    })),
-                });
-
-                for table_id in &new_fragment_table_info.internal_table_ids {
-                    table_compaction_group_mapping
-                        .insert(*table_id, StaticCompactionGroupId::StateDefault as u64);
-                }
-            }
-
-            if let Some(table_id) = new_fragment_table_info.mv_table_id {
-                if let Some(levels) = old_version
-                    .levels
-                    .get(&(StaticCompactionGroupId::MaterializedView as u64))
-                {
-                    if levels.member_table_ids.contains(&table_id.table_id) {
-                        return Err(Error::CompactionGroup(format!(
-                            "table {} already in group {}",
-                            table_id,
-                            StaticCompactionGroupId::MaterializedView as u64
-                        )));
-                    }
-                }
-                let group_deltas = &mut new_version_delta
-                    .group_deltas
-                    .entry(StaticCompactionGroupId::MaterializedView as u64)
-                    .or_default()
-                    .group_deltas;
-                group_deltas.push(GroupDelta {
-                    delta_type: Some(DeltaType::GroupMetaChange(GroupMetaChange {
-                        table_ids_add: vec![table_id.table_id],
-                        ..Default::default()
-                    })),
-                });
-                let _ = table_compaction_group_mapping
-                    .insert(table_id, StaticCompactionGroupId::MaterializedView as u64);
-            }
-        }
-
-        let mut incorrect_ssts = vec![];
-        let mut new_sst_id_number = 0;
-        for ExtendedSstableInfo {
-            compaction_group_id,
-            sst_info: sst,
-            ..
-        } in &mut sstables
-        {
-            let is_sst_belong_to_group_declared = match old_version.levels.get(compaction_group_id)
-            {
-                Some(compaction_group) => sst
-                    .table_ids
-                    .iter()
-                    .all(|t| compaction_group.member_table_ids.contains(t)),
-                None => false,
-            };
-            if !is_sst_belong_to_group_declared {
-                let mut group_table_ids: BTreeMap<_, Vec<u32>> = BTreeMap::new();
-                for table_id in sst.get_table_ids() {
-                    match table_compaction_group_mapping.get(&TableId::new(*table_id)) {
-                        Some(compaction_group_id) => {
-                            group_table_ids
-                                .entry(*compaction_group_id)
-                                .or_default()
-                                .push(*table_id);
-                        }
-                        None => {
-                            tracing::warn!(
-                                "table {} in SST {} doesn't belong to any compaction group",
-                                table_id,
-                                sst.get_object_id(),
-                            );
-                        }
-                    }
-                }
-                let is_trivial_adjust = group_table_ids.len() == 1
-                    && group_table_ids.first_key_value().unwrap().1.len()
-                        == sst.get_table_ids().len();
-                if is_trivial_adjust {
-                    *compaction_group_id = *group_table_ids.first_key_value().unwrap().0;
-                    // is_sst_belong_to_group_declared = true;
-                } else {
-                    new_sst_id_number += group_table_ids.len();
-                    incorrect_ssts.push((std::mem::take(sst), group_table_ids));
-                    *compaction_group_id =
-                        StaticCompactionGroupId::NewCompactionGroup as CompactionGroupId;
-                }
-            }
-        }
-        let mut new_sst_id = next_sstable_object_id(&self.env, new_sst_id_number).await?;
-        let mut branched_ssts = create_trx_wrapper!(
-            self.meta_store_ref(),
-            BTreeMapTransactionWrapper,
-            BTreeMapTransaction::new(&mut versioning.branched_ssts)
-        );
-        let original_sstables = std::mem::take(&mut sstables);
-        sstables.reserve_exact(original_sstables.len() - incorrect_ssts.len() + new_sst_id_number);
-        let mut incorrect_ssts = incorrect_ssts.into_iter();
-        for original_sstable in original_sstables {
-            if original_sstable.compaction_group_id
-                == StaticCompactionGroupId::NewCompactionGroup as CompactionGroupId
-            {
-                let (sst, group_table_ids) = incorrect_ssts.next().unwrap();
-                let mut branch_groups = HashMap::new();
-                for (group_id, _match_ids) in group_table_ids {
-                    let mut branch_sst = sst.clone();
-                    branch_sst.sst_id = new_sst_id;
-                    sstables.push(ExtendedSstableInfo::with_compaction_group(
-                        group_id, branch_sst,
-                    ));
-                    branch_groups.insert(group_id, new_sst_id);
-                    new_sst_id += 1;
-                }
-                if !branch_groups.is_empty() {
-                    branched_ssts.insert(sst.get_object_id(), branch_groups);
-                }
-            } else {
-                sstables.push(original_sstable);
-            }
-        }
-
-        let mut modified_compaction_groups = vec![];
-        // Append SSTs to a new version.
-        for (compaction_group_id, sstables) in &sstables
-            .into_iter()
-            // the sort is stable sort, and will not change the order within compaction group.
-            // Do a sort so that sst in the same compaction group can be consecutive
-            .sorted_by_key(
-                |ExtendedSstableInfo {
-                     compaction_group_id,
-                     ..
-                 }| *compaction_group_id,
-            )
-            .group_by(
-                |ExtendedSstableInfo {
-                     compaction_group_id,
-                     ..
-                 }| *compaction_group_id,
-            )
-        {
-            modified_compaction_groups.push(compaction_group_id);
-            let group_sstables = sstables
-                .into_iter()
-                .map(|ExtendedSstableInfo { sst_info, .. }| sst_info)
-                .collect_vec();
-
-            let group_deltas = &mut new_version_delta
-                .group_deltas
-                .entry(compaction_group_id)
-                .or_default()
-                .group_deltas;
-            let l0_sub_level_id = epoch;
-            let group_delta = GroupDelta {
-                delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta {
-                    level_idx: 0,
-                    inserted_table_infos: group_sstables.clone(),
-                    l0_sub_level_id,
-                    ..Default::default()
-                })),
-            };
-            group_deltas.push(group_delta);
-        }
-
-        let mut new_hummock_version = old_version.clone();
-        // Create a new_version, possibly merely to bump up the version id and max_committed_epoch.
-        new_hummock_version.apply_version_delta(new_version_delta.deref());
-
-        // Apply stats changes.
-        let mut version_stats = create_trx_wrapper!(
-            self.meta_store_ref(),
-            VarTransactionWrapper,
-            VarTransaction::new(&mut versioning.version_stats)
-        );
-        add_prost_table_stats_map(&mut version_stats.table_stats, &table_stats_change);
-        if purge_prost_table_stats(&mut version_stats.table_stats, &new_hummock_version) {
-            self.metrics.version_stats.reset();
-            versioning.local_metrics.clear();
-        }
-
-        trigger_local_table_stat(
-            &self.metrics,
-            &mut versioning.local_metrics,
-            &version_stats,
-            &table_stats_change,
-        );
-        for (table_id, stats) in &table_stats_change {
-            if stats.total_key_size == 0
-                && stats.total_value_size == 0
-                && stats.total_key_count == 0
-            {
-                continue;
-            }
-            let stats_value = std::cmp::max(0, stats.total_key_size + stats.total_value_size);
-            let table_metrics = get_or_create_local_table_stat(
-                &self.metrics,
-                *table_id,
-                &mut versioning.local_metrics,
-            );
-            table_metrics.inc_write_throughput(stats_value as u64);
-        }
-        commit_multi_var!(self.meta_store_ref(), new_version_delta, version_stats)?;
-        branched_ssts.commit_memory();
-        versioning.current_version = new_hummock_version;
-
-        let snapshot = HummockSnapshot {
-            committed_epoch: epoch,
-            current_epoch: epoch,
-        };
-        let prev_snapshot = self.latest_snapshot.swap(snapshot.clone().into());
-        assert!(prev_snapshot.committed_epoch < epoch);
-        assert!(prev_snapshot.current_epoch < epoch);
-
-        trigger_version_stat(&self.metrics, &versioning.current_version);
-        for compaction_group_id in &modified_compaction_groups {
-            trigger_sst_stat(
-                &self.metrics,
-                None,
-                &versioning.current_version,
-                *compaction_group_id,
-            );
-        }
-
-        tracing::trace!("new committed epoch {}", epoch);
-
-        self.notify_last_version_delta(versioning);
-        trigger_delta_log_stats(&self.metrics, versioning.hummock_version_deltas.len());
-        self.notify_stats(&versioning.version_stats);
-        let mut table_groups = HashMap::<u32, usize>::default();
-        for group in versioning.current_version.levels.values() {
-            for table_id in &group.member_table_ids {
-                table_groups.insert(*table_id, group.member_table_ids.len());
-            }
-        }
-        drop(versioning_guard);
-        // Don't trigger compactions if we enable deterministic compaction
-        if !self.env.opts.compaction_deterministic_test {
-            // commit_epoch may contains SSTs from any compaction group
-            for id in &modified_compaction_groups {
-                self.try_send_compaction_request(*id, compact_task::TaskType::Dynamic);
-            }
-            if !table_stats_change.is_empty() {
-                table_stats_change.retain(|table_id, _| {
-                    table_groups
-                        .get(table_id)
-                        .map(|table_count| *table_count > 1)
-                        .unwrap_or(false)
-                });
-            }
-            if !table_stats_change.is_empty() {
-                self.collect_table_write_throughput(table_stats_change);
-            }
-        }
-        if !modified_compaction_groups.is_empty() {
-            self.try_update_write_limits(&modified_compaction_groups)
-                .await;
-        }
-        #[cfg(test)]
-        {
-            self.check_state_consistency().await;
-        }
-        Ok(Some(snapshot))
-    }
-
-    /// We don't commit an epoch without checkpoint. We will only update the `max_current_epoch`.
-    pub fn update_current_epoch(&self, max_current_epoch: HummockEpoch) -> HummockSnapshot {
-        // We only update `max_current_epoch`!
-        let prev_snapshot = self.latest_snapshot.rcu(|snapshot| HummockSnapshot {
-            committed_epoch: snapshot.committed_epoch,
-            current_epoch: max_current_epoch,
-        });
-        assert!(prev_snapshot.current_epoch < max_current_epoch);
-
-        tracing::trace!("new current epoch {}", max_current_epoch);
-        HummockSnapshot {
-            committed_epoch: prev_snapshot.committed_epoch,
-            current_epoch: max_current_epoch,
-        }
-    }
-
-    pub async fn get_new_sst_ids(&self, number: u32) -> Result<SstObjectIdRange> {
-        let start_id = next_sstable_object_id(&self.env, number).await?;
-        Ok(SstObjectIdRange::new(start_id, start_id + number as u64))
-    }
-
-    #[named]
-    pub async fn get_min_pinned_version_id(&self) -> HummockVersionId {
-        read_lock!(self, versioning).await.min_pinned_version_id()
-    }
-
-    #[named]
-    #[cfg(test)]
-    pub async fn check_state_consistency(&self) {
-        let mut compaction_guard = write_lock!(self, compaction).await;
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        let objects_to_delete = versioning_guard.objects_to_delete.clone();
-        // We don't check `checkpoint` because it's allowed to update its in memory state without
-        // persisting to object store.
-        let get_state =
-            |compaction_guard: &RwLockWriteGuard<'_, Compaction>,
-             versioning_guard: &RwLockWriteGuard<'_, Versioning>| {
-                let compact_statuses_copy = compaction_guard.compaction_statuses.clone();
-                let compact_task_assignment_copy = compaction_guard.compact_task_assignment.clone();
-                let pinned_versions_copy = versioning_guard.pinned_versions.clone();
-                let pinned_snapshots_copy = versioning_guard.pinned_snapshots.clone();
-                let hummock_version_deltas_copy = versioning_guard.hummock_version_deltas.clone();
-                let version_stats_copy = versioning_guard.version_stats.clone();
-                let branched_ssts = versioning_guard.branched_ssts.clone();
-                (
-                    (
-                        compact_statuses_copy,
-                        compact_task_assignment_copy,
-                        pinned_versions_copy,
-                        pinned_snapshots_copy,
-                        hummock_version_deltas_copy,
-                        version_stats_copy,
-                    ),
-                    branched_ssts,
-                )
-            };
-        let (mem_state, branched_ssts) = get_state(&compaction_guard, &versioning_guard);
-        self.load_meta_store_state_impl(
-            compaction_guard.borrow_mut(),
-            versioning_guard.borrow_mut(),
-        )
-        .await
-        .expect("Failed to load state from meta store");
-        let (loaded_state, load_branched_ssts) = get_state(&compaction_guard, &versioning_guard);
-        assert_eq!(branched_ssts, load_branched_ssts);
-        assert_eq!(
-            mem_state, loaded_state,
-            "hummock in-mem state is inconsistent with meta store state",
-        );
-        versioning_guard.objects_to_delete = objects_to_delete;
-    }
-
-    /// Gets current version without pinning it.
-    /// Should not be called inside [`HummockManager`], because it requests locks internally.
-    ///
-    /// Note: this method can hurt performance because it will clone a large object.
-    #[named]
-    pub async fn get_current_version(&self) -> HummockVersion {
-        read_lock!(self, versioning).await.current_version.clone()
-    }
-
-    #[named]
-    pub async fn get_current_max_committed_epoch(&self) -> HummockEpoch {
-        read_lock!(self, versioning)
-            .await
-            .current_version
-            .max_committed_epoch
-    }
-
-    /// Gets branched sstable infos
-    /// Should not be called inside [`HummockManager`], because it requests locks internally.
-    #[named]
-    pub async fn get_branched_ssts_info(&self) -> BTreeMap<HummockSstableId, BranchedSstInfo> {
-        read_lock!(self, versioning).await.branched_ssts.clone()
-    }
-
-    #[named]
-    /// Gets the mapping from table id to compaction group id
-    pub async fn get_table_compaction_group_id_mapping(
-        &self,
-    ) -> HashMap<StateTableId, CompactionGroupId> {
-        get_table_compaction_group_id_mapping(&read_lock!(self, versioning).await.current_version)
-    }
-
-    /// Get version deltas from meta store
-    #[cfg_attr(coverage, coverage(off))]
-    #[named]
-    pub async fn list_version_deltas(
-        &self,
-        start_id: u64,
-        num_limit: u32,
-        committed_epoch_limit: HummockEpoch,
-    ) -> Result<Vec<HummockVersionDelta>> {
-        let versioning = read_lock!(self, versioning).await;
-        let version_deltas = versioning
-            .hummock_version_deltas
-            .range(start_id..)
-            .map(|(_id, delta)| delta)
-            .filter(|delta| delta.max_committed_epoch <= committed_epoch_limit)
-            .take(num_limit as _)
-            .cloned()
-            .collect();
-        Ok(version_deltas)
-    }
-
-    pub async fn init_metadata_for_version_replay(
-        &self,
-        table_catalogs: Vec<Table>,
-        compaction_groups: Vec<PbCompactionGroupInfo>,
-    ) -> Result<()> {
-        for table in &table_catalogs {
-            table.insert(self.env.meta_store().as_kv()).await?;
-        }
-        for group in &compaction_groups {
-            assert!(
-                group.id == StaticCompactionGroupId::NewCompactionGroup as u64
-                    || (group.id >= StaticCompactionGroupId::StateDefault as u64
-                    && group.id <= StaticCompactionGroupId::MaterializedView as u64),
-                "compaction group id should be either NewCompactionGroup to create new one, or predefined static ones."
-            );
-        }
-
-        for group in &compaction_groups {
-            let mut pairs = vec![];
-            for table_id in group.member_table_ids.clone() {
-                pairs.push((table_id as StateTableId, group.id));
-            }
-            let group_config = group.compaction_config.clone().unwrap();
-            self.compaction_group_manager
-                .write()
-                .await
-                .init_compaction_config_for_replay(group.id, group_config)
-                .await
-                .unwrap();
-            self.register_table_ids(&pairs).await?;
-            tracing::info!("Registered table ids {:?}", pairs);
-        }
-
-        // Notify that tables have created
-        for table in table_catalogs {
-            self.env
-                .notification_manager()
-                .notify_hummock_relation_info(Operation::Add, RelationInfo::Table(table.clone()))
-                .await;
-            self.env
-                .notification_manager()
-                .notify_compactor_relation_info(Operation::Add, RelationInfo::Table(table))
-                .await;
-        }
-
-        tracing::info!("Inited compaction groups:");
-        for group in compaction_groups {
-            tracing::info!("{:?}", group);
-        }
-        Ok(())
-    }
-
-    /// Replay a version delta to current hummock version.
-    /// Returns the `version_id`, `max_committed_epoch` of the new version and the modified
-    /// compaction groups
-    #[named]
-    pub async fn replay_version_delta(
-        &self,
-        mut version_delta: HummockVersionDelta,
-    ) -> Result<(HummockVersion, Vec<CompactionGroupId>)> {
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        // ensure the version id is ascending after replay
-        version_delta.id = versioning_guard.current_version.id + 1;
-        version_delta.prev_id = version_delta.id - 1;
-        versioning_guard
-            .current_version
-            .apply_version_delta(&version_delta);
-
-        let version_new = versioning_guard.current_version.clone();
-        let compaction_group_ids = version_delta.group_deltas.keys().cloned().collect_vec();
-        Ok((version_new, compaction_group_ids))
-    }
-
-    #[named]
-    pub async fn disable_commit_epoch(&self) -> HummockVersion {
-        let mut versioning_guard = write_lock!(self, versioning).await;
-        versioning_guard.disable_commit_epochs = true;
-        versioning_guard.current_version.clone()
-    }
-
-    /// Triggers compacitons to specified compaction groups.
-    /// Don't wait for compaction finish
-    pub async fn trigger_compaction_deterministic(
-        &self,
-        _base_version_id: HummockVersionId,
-        compaction_groups: Vec<CompactionGroupId>,
-    ) -> Result<()> {
-        let old_version = self.get_current_version().await;
-        tracing::info!(
-            "Trigger compaction for version {}, epoch {}, groups {:?}",
-            old_version.id,
-            old_version.max_committed_epoch,
-            compaction_groups
-        );
-
-        if compaction_groups.is_empty() {
-            return Ok(());
-        }
-        for compaction_group in compaction_groups {
-            self.try_send_compaction_request(compaction_group, compact_task::TaskType::Dynamic);
-        }
-        Ok(())
-    }
-
-    /// Sends a compaction request.
-    pub fn try_send_compaction_request(
-        &self,
-        compaction_group: CompactionGroupId,
-        task_type: compact_task::TaskType,
-    ) -> bool {
-        match self
-            .compaction_state
-            .try_sched_compaction(compaction_group, task_type)
-        {
-            Ok(_) => true,
-            Err(e) => {
-                tracing::error!(
-                    error = %e.as_report(),
-                    "failed to send compaction request for compaction group {}",
-                    compaction_group,
-                );
-                false
-            }
-        }
-    }
-
-    pub async fn trigger_manual_compaction(
-        &self,
-        compaction_group: CompactionGroupId,
-        manual_compaction_option: ManualCompactionOption,
-    ) -> Result<()> {
-        let start_time = Instant::now();
-
-        // 1. Get idle compactor.
-        let compactor = match self.compactor_manager.next_compactor() {
-            Some(compactor) => compactor,
-            None => {
-                tracing::warn!("trigger_manual_compaction No compactor is available.");
-                return Err(anyhow::anyhow!(
-                    "trigger_manual_compaction No compactor is available. compaction_group {}",
-                    compaction_group
-                )
-                .into());
-            }
-        };
-
-        // 2. Get manual compaction task.
-        let compact_task = self
-            .manual_get_compact_task(compaction_group, manual_compaction_option)
-            .await;
-        let compact_task = match compact_task {
-            Ok(Some(compact_task)) => compact_task,
-            Ok(None) => {
-                // No compaction task available.
-                return Err(anyhow::anyhow!(
-                    "trigger_manual_compaction No compaction_task is available. compaction_group {}",
-                    compaction_group
-                )
-                .into());
-            }
-            Err(err) => {
-                tracing::warn!(error = %err.as_report(), "Failed to get compaction task");
-
-                return Err(anyhow::anyhow!(err)
-                    .context(format!(
-                        "Failed to get compaction task for compaction_group {}",
-                        compaction_group,
-                    ))
-                    .into());
-            }
-        };
-
-        // 3. send task to compactor
-        let compact_task_string = compact_task_to_string(&compact_task);
-        // TODO: shall we need to cancel on meta ?
-        compactor
-            .send_event(ResponseEvent::CompactTask(compact_task))
-            .with_context(|| {
-                format!(
-                    "Failed to trigger compaction task for compaction_group {}",
-                    compaction_group,
-                )
-            })?;
-
-        tracing::info!(
-            "Trigger manual compaction task. {}. cost time: {:?}",
-            &compact_task_string,
-            start_time.elapsed(),
-        );
-
-        Ok(())
-    }
-
-    #[cfg(any(test, feature = "test"))]
-    pub fn compactor_manager_ref_for_test(&self) -> CompactorManagerRef {
-        self.compactor_manager.clone()
-    }
-
-    #[cfg(any(test, feature = "test"))]
-    #[named]
-    pub async fn compaction_task_from_assignment_for_test(
-        &self,
-        task_id: u64,
-    ) -> Option<CompactTaskAssignment> {
-        let compaction_guard = read_lock!(self, compaction).await;
-        let assignment_ref = &compaction_guard.compact_task_assignment;
-        assignment_ref.get(&task_id).cloned()
-    }
-
-    #[cfg(any(test, feature = "test"))]
-    #[named]
-    pub async fn report_compact_task_for_test(
-        &self,
-        task_id: u64,
-        compact_task: Option<CompactTask>,
-        task_status: TaskStatus,
-        sorted_output_ssts: Vec<SstableInfo>,
-        table_stats_change: Option<PbTableStatsMap>,
-    ) -> Result<()> {
-        if let Some(task) = compact_task {
-            let mut guard = write_lock!(self, compaction).await;
-            guard.compact_task_assignment.insert(
-                task_id,
-                CompactTaskAssignment {
-                    compact_task: Some(task),
-                    context_id: 0,
-                },
-            );
-        }
-
-        // In the test, the contents of the compact task may have been modified directly, while the contents of compact_task_assignment were not modified.
-        // So we pass the modified compact_task directly into the `report_compact_task_impl`
-        self.report_compact_tasks(vec![ReportTask {
-            task_id,
-            task_status: task_status as i32,
-            sorted_output_ssts,
-            table_stats_change: table_stats_change.unwrap_or_default(),
-        }])
-        .await?;
-        Ok(())
-    }
-
-    pub fn metadata_manager(&self) -> &MetadataManager {
-        &self.metadata_manager
-    }
-
-    fn notify_last_version_delta(&self, versioning: &Versioning) {
-        self.env
-            .notification_manager()
-            .notify_hummock_without_version(
-                Operation::Add,
-                Info::HummockVersionDeltas(risingwave_pb::hummock::HummockVersionDeltas {
-                    version_deltas: vec![versioning
-                        .hummock_version_deltas
-                        .last_key_value()
-                        .unwrap()
-                        .1
-                        .to_protobuf()],
-                }),
-            );
-    }
-
-    fn notify_version_deltas(&self, versioning: &Versioning, last_version_id: u64) {
-        let start_version_id = last_version_id + 1;
-        let version_deltas = versioning
-            .hummock_version_deltas
-            .range(start_version_id..)
-            .map(|(_, delta)| delta.to_protobuf())
-            .collect_vec();
-        self.env
-            .notification_manager()
-            .notify_hummock_without_version(
-                Operation::Add,
-                Info::HummockVersionDeltas(risingwave_pb::hummock::HummockVersionDeltas {
-                    version_deltas,
-                }),
-            );
-    }
-
-    fn notify_stats(&self, stats: &HummockVersionStats) {
-        self.env
-            .notification_manager()
-            .notify_frontend_without_version(Operation::Update, Info::HummockStats(stats.clone()));
-    }
-
-    #[named]
-    pub fn hummock_timer_task(hummock_manager: Arc<Self>) -> (JoinHandle<()>, Sender<()>) {
-        use futures::{FutureExt, StreamExt};
-
-        let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
-        let join_handle = tokio::spawn(async move {
-            const CHECK_PENDING_TASK_PERIOD_SEC: u64 = 300;
-            const STAT_REPORT_PERIOD_SEC: u64 = 20;
-            const COMPACTION_HEARTBEAT_PERIOD_SEC: u64 = 1;
-
-            pub enum HummockTimerEvent {
-                GroupSplit,
-                CheckDeadTask,
-                Report,
-                CompactionHeartBeatExpiredCheck,
-
-                DynamicCompactionTrigger,
-                SpaceReclaimCompactionTrigger,
-                TtlCompactionTrigger,
-                TombstoneCompactionTrigger,
-
-                FullGc,
-            }
-            let mut check_compact_trigger_interval =
-                tokio::time::interval(Duration::from_secs(CHECK_PENDING_TASK_PERIOD_SEC));
-            check_compact_trigger_interval
-                .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
-            check_compact_trigger_interval.reset();
-
-            let check_compact_trigger = IntervalStream::new(check_compact_trigger_interval)
-                .map(|_| HummockTimerEvent::CheckDeadTask);
-
-            let mut stat_report_interval =
-                tokio::time::interval(std::time::Duration::from_secs(STAT_REPORT_PERIOD_SEC));
-            stat_report_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
-            stat_report_interval.reset();
-            let stat_report_trigger =
-                IntervalStream::new(stat_report_interval).map(|_| HummockTimerEvent::Report);
-
-            let mut compaction_heartbeat_interval = tokio::time::interval(
-                std::time::Duration::from_secs(COMPACTION_HEARTBEAT_PERIOD_SEC),
-            );
-            compaction_heartbeat_interval
-                .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
-            compaction_heartbeat_interval.reset();
-            let compaction_heartbeat_trigger = IntervalStream::new(compaction_heartbeat_interval)
-                .map(|_| HummockTimerEvent::CompactionHeartBeatExpiredCheck);
-
-            let mut min_trigger_interval = tokio::time::interval(Duration::from_secs(
-                hummock_manager.env.opts.periodic_compaction_interval_sec,
-            ));
-            min_trigger_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
-            min_trigger_interval.reset();
-            let dynamic_tick_trigger = IntervalStream::new(min_trigger_interval)
-                .map(|_| HummockTimerEvent::DynamicCompactionTrigger);
-
-            let mut min_space_reclaim_trigger_interval =
-                tokio::time::interval(Duration::from_secs(
-                    hummock_manager
-                        .env
-                        .opts
-                        .periodic_space_reclaim_compaction_interval_sec,
-                ));
-            min_space_reclaim_trigger_interval
-                .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
-            min_space_reclaim_trigger_interval.reset();
-            let space_reclaim_trigger = IntervalStream::new(min_space_reclaim_trigger_interval)
-                .map(|_| HummockTimerEvent::SpaceReclaimCompactionTrigger);
-
-            let mut min_ttl_reclaim_trigger_interval = tokio::time::interval(Duration::from_secs(
-                hummock_manager
-                    .env
-                    .opts
-                    .periodic_ttl_reclaim_compaction_interval_sec,
-            ));
-            min_ttl_reclaim_trigger_interval
-                .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
-            min_ttl_reclaim_trigger_interval.reset();
-            let ttl_reclaim_trigger = IntervalStream::new(min_ttl_reclaim_trigger_interval)
-                .map(|_| HummockTimerEvent::TtlCompactionTrigger);
-
-            let mut full_gc_interval = tokio::time::interval(Duration::from_secs(
-                hummock_manager.env.opts.full_gc_interval_sec,
-            ));
-            full_gc_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
-            full_gc_interval.reset();
-            let full_gc_trigger =
-                IntervalStream::new(full_gc_interval).map(|_| HummockTimerEvent::FullGc);
-
-            let mut tombstone_reclaim_trigger_interval =
-                tokio::time::interval(Duration::from_secs(
-                    hummock_manager
-                        .env
-                        .opts
-                        .periodic_tombstone_reclaim_compaction_interval_sec,
-                ));
-            tombstone_reclaim_trigger_interval
-                .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
-            tombstone_reclaim_trigger_interval.reset();
-            let tombstone_reclaim_trigger = IntervalStream::new(tombstone_reclaim_trigger_interval)
-                .map(|_| HummockTimerEvent::TombstoneCompactionTrigger);
-
-            let mut triggers: Vec<BoxStream<'static, HummockTimerEvent>> = vec![
-                Box::pin(check_compact_trigger),
-                Box::pin(stat_report_trigger),
-                Box::pin(compaction_heartbeat_trigger),
-                Box::pin(dynamic_tick_trigger),
-                Box::pin(space_reclaim_trigger),
-                Box::pin(ttl_reclaim_trigger),
-                Box::pin(full_gc_trigger),
-                Box::pin(tombstone_reclaim_trigger),
-            ];
-
-            let periodic_check_split_group_interval_sec = hummock_manager
-                .env
-                .opts
-                .periodic_split_compact_group_interval_sec;
-
-            if periodic_check_split_group_interval_sec > 0 {
-                let mut split_group_trigger_interval = tokio::time::interval(Duration::from_secs(
-                    periodic_check_split_group_interval_sec,
-                ));
-                split_group_trigger_interval
-                    .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
-
-                let split_group_trigger = IntervalStream::new(split_group_trigger_interval)
-                    .map(|_| HummockTimerEvent::GroupSplit);
-                triggers.push(Box::pin(split_group_trigger));
-            }
-
-            let event_stream = select_all(triggers);
-            use futures::pin_mut;
-            pin_mut!(event_stream);
-
-            let shutdown_rx_shared = shutdown_rx.shared();
-
-            tracing::info!(
-                "Hummock timer task tracing [GroupSplit interval {} sec] [CheckDeadTask interval {} sec] [Report interval {} sec] [CompactionHeartBeat interval {} sec]",
-                    periodic_check_split_group_interval_sec, CHECK_PENDING_TASK_PERIOD_SEC, STAT_REPORT_PERIOD_SEC, COMPACTION_HEARTBEAT_PERIOD_SEC
-            );
-
-            loop {
-                let item =
-                    futures::future::select(event_stream.next(), shutdown_rx_shared.clone()).await;
-
-                match item {
-                    Either::Left((event, _)) => {
-                        if let Some(event) = event {
-                            match event {
-                                HummockTimerEvent::CheckDeadTask => {
-                                    if hummock_manager.env.opts.compaction_deterministic_test {
-                                        continue;
-                                    }
-
-                                    hummock_manager.check_dead_task().await;
-                                }
-
-                                HummockTimerEvent::GroupSplit => {
-                                    if hummock_manager.env.opts.compaction_deterministic_test {
-                                        continue;
-                                    }
-
-                                    hummock_manager.on_handle_check_split_multi_group().await;
-                                }
-
-                                HummockTimerEvent::Report => {
-                                    let (
-                                        current_version,
-                                        id_to_config,
-                                        branched_sst,
-                                        version_stats,
-                                    ) = {
-                                        let versioning_guard =
-                                            read_lock!(hummock_manager.as_ref(), versioning).await;
-
-                                        let configs =
-                                            hummock_manager.get_compaction_group_map().await;
-                                        let versioning_deref = versioning_guard;
-                                        (
-                                            versioning_deref.current_version.clone(),
-                                            configs,
-                                            versioning_deref.branched_ssts.clone(),
-                                            versioning_deref.version_stats.clone(),
-                                        )
-                                    };
-
-                                    if let Some(mv_id_to_all_table_ids) = hummock_manager
-                                        .metadata_manager
-                                        .get_job_id_to_internal_table_ids_mapping()
-                                        .await
-                                    {
-                                        trigger_mv_stat(
-                                            &hummock_manager.metrics,
-                                            &version_stats,
-                                            mv_id_to_all_table_ids,
-                                        );
-                                    }
-
-                                    for compaction_group_id in
-                                        get_compaction_group_ids(&current_version)
-                                    {
-                                        let compaction_group_config =
-                                            &id_to_config[&compaction_group_id];
-
-                                        let group_levels = current_version
-                                            .get_compaction_group_levels(
-                                                compaction_group_config.group_id(),
-                                            );
-
-                                        trigger_split_stat(
-                                            &hummock_manager.metrics,
-                                            compaction_group_config.group_id(),
-                                            group_levels.member_table_ids.len(),
-                                            &branched_sst,
-                                        );
-
-                                        trigger_lsm_stat(
-                                            &hummock_manager.metrics,
-                                            compaction_group_config.compaction_config(),
-                                            group_levels,
-                                            compaction_group_config.group_id(),
-                                        )
-                                    }
-                                }
-
-                                HummockTimerEvent::CompactionHeartBeatExpiredCheck => {
-                                    let compactor_manager =
-                                        hummock_manager.compactor_manager.clone();
-
-                                    // TODO: add metrics to track expired tasks
-                                    // The cancel task has two paths
-                                    // 1. compactor heartbeat cancels the expired task based on task
-                                    // progress (meta + compactor)
-                                    // 2. meta periodically scans the task and performs a cancel on
-                                    // the meta side for tasks that are not updated by heartbeat
-                                    for task in compactor_manager.get_heartbeat_expired_tasks() {
-                                        if let Err(e) = hummock_manager
-                                            .cancel_compact_task(
-                                                task.task_id,
-                                                TaskStatus::HeartbeatCanceled,
-                                            )
-                                            .await
-                                        {
-                                            tracing::error!(
-                                                task_id = task.task_id,
-                                                error = %e.as_report(),
-                                                "Attempt to remove compaction task due to elapsed heartbeat failed. We will continue to track its heartbeat
-                                                until we can successfully report its status",
-                                            );
-                                        }
-                                    }
-                                }
-
-                                HummockTimerEvent::DynamicCompactionTrigger => {
-                                    // Disable periodic trigger for compaction_deterministic_test.
-                                    if hummock_manager.env.opts.compaction_deterministic_test {
-                                        continue;
-                                    }
-
-                                    hummock_manager
-                                        .on_handle_trigger_multi_group(
-                                            compact_task::TaskType::Dynamic,
-                                        )
-                                        .await;
-                                }
-
-                                HummockTimerEvent::SpaceReclaimCompactionTrigger => {
-                                    // Disable periodic trigger for compaction_deterministic_test.
-                                    if hummock_manager.env.opts.compaction_deterministic_test {
-                                        continue;
-                                    }
-
-                                    hummock_manager
-                                        .on_handle_trigger_multi_group(
-                                            compact_task::TaskType::SpaceReclaim,
-                                        )
-                                        .await;
-                                }
-
-                                HummockTimerEvent::TtlCompactionTrigger => {
-                                    // Disable periodic trigger for compaction_deterministic_test.
-                                    if hummock_manager.env.opts.compaction_deterministic_test {
-                                        continue;
-                                    }
-
-                                    hummock_manager
-                                        .on_handle_trigger_multi_group(compact_task::TaskType::Ttl)
-                                        .await;
-                                }
-
-                                HummockTimerEvent::TombstoneCompactionTrigger => {
-                                    // Disable periodic trigger for compaction_deterministic_test.
-                                    if hummock_manager.env.opts.compaction_deterministic_test {
-                                        continue;
-                                    }
-
-                                    hummock_manager
-                                        .on_handle_trigger_multi_group(
-                                            compact_task::TaskType::Tombstone,
-                                        )
-                                        .await;
-                                }
-
-                                HummockTimerEvent::FullGc => {
-                                    if hummock_manager
-                                        .start_full_gc(Duration::from_secs(3600))
-                                        .is_ok()
-                                    {
-                                        tracing::info!("Start full GC from meta node.");
-                                    }
-                                }
-                            }
-                        }
-                    }
-
-                    Either::Right((_, _shutdown)) => {
-                        tracing::info!("Hummock timer loop is stopped");
-                        break;
-                    }
-                }
-            }
-        });
-        (join_handle, shutdown_tx)
-    }
-
-    #[named]
-    pub async fn check_dead_task(&self) {
-        const MAX_COMPACTION_L0_MULTIPLIER: u64 = 32;
-        const MAX_COMPACTION_DURATION_SEC: u64 = 20 * 60;
-        let (groups, configs) = {
-            let versioning_guard = read_lock!(self, versioning).await;
-            let g = versioning_guard
-                .current_version
-                .levels
-                .iter()
-                .map(|(id, group)| {
-                    (
-                        *id,
-                        group
-                            .l0
-                            .as_ref()
-                            .unwrap()
-                            .sub_levels
-                            .iter()
-                            .map(|level| level.total_file_size)
-                            .sum::<u64>(),
-                    )
-                })
-                .collect_vec();
-            let c = self.get_compaction_group_map().await;
-            (g, c)
-        };
-        let mut slowdown_groups: HashMap<u64, u64> = HashMap::default();
-        {
-            for (group_id, l0_file_size) in groups {
-                let group = &configs[&group_id];
-                if l0_file_size
-                    > MAX_COMPACTION_L0_MULTIPLIER
-                        * group.compaction_config.max_bytes_for_level_base
-                {
-                    slowdown_groups.insert(group_id, l0_file_size);
-                }
-            }
-        }
-        if slowdown_groups.is_empty() {
-            return;
-        }
-        let mut pending_tasks: HashMap<u64, (u64, usize, RunningCompactTask)> = HashMap::default();
-        {
-            let compaction_guard = read_lock!(self, compaction).await;
-            for group_id in slowdown_groups.keys() {
-                if let Some(status) = compaction_guard.compaction_statuses.get(group_id) {
-                    for (idx, level_handler) in status.level_handlers.iter().enumerate() {
-                        let tasks = level_handler.get_pending_tasks().to_vec();
-                        if tasks.is_empty() {
-                            continue;
-                        }
-                        for task in tasks {
-                            pending_tasks.insert(task.task_id, (*group_id, idx, task));
-                        }
-                    }
-                }
-            }
-        }
-        let task_ids = pending_tasks.keys().cloned().collect_vec();
-        let task_infos = self
-            .compactor_manager
-            .check_tasks_status(&task_ids, Duration::from_secs(MAX_COMPACTION_DURATION_SEC));
-        for (task_id, (compact_time, status)) in task_infos {
-            if status == TASK_NORMAL {
-                continue;
-            }
-            if let Some((group_id, level_id, task)) = pending_tasks.get(&task_id) {
-                let group_size = *slowdown_groups.get(group_id).unwrap();
-                warn!("COMPACTION SLOW: the task-{} of group-{}(size: {}MB) level-{} has not finished after {:?}, {}, it may cause pending sstable files({:?}) blocking other task.",
-                    task_id, *group_id,group_size / 1024 / 1024,*level_id, compact_time, status, task.ssts);
-            }
-        }
-    }
-
-    fn collect_table_write_throughput(&self, table_stats: PbTableStatsMap) {
-        let mut table_infos = self.history_table_throughput.write();
-        for (table_id, stat) in table_stats {
-            let throughput = (stat.total_value_size + stat.total_key_size) as u64;
-            let entry = table_infos.entry(table_id).or_default();
-            entry.push_back(throughput);
-            if entry.len() > HISTORY_TABLE_INFO_STATISTIC_TIME {
-                entry.pop_front();
-            }
-        }
-    }
-
-    /// * For compaction group with only one single state-table, do not change it again.
-    /// * For state-table which only write less than `HISTORY_TABLE_INFO_WINDOW_SIZE` times, do not
-    ///   change it. Because we need more statistic data to decide split strategy.
-    /// * For state-table with low throughput which write no more than
-    ///   `min_table_split_write_throughput` data, never split it.
-    /// * For state-table whose size less than `min_table_split_size`, do not split it unless its
-    ///   throughput keep larger than `table_write_throughput_threshold` for a long time.
-    /// * For state-table whose throughput less than `min_table_split_write_throughput`, do not
-    ///   increase it size of base-level.
-    async fn on_handle_check_split_multi_group(&self) {
-        let params = self.env.system_params_reader().await;
-        let barrier_interval_ms = params.barrier_interval_ms() as u64;
-        let checkpoint_secs = std::cmp::max(
-            1,
-            params.checkpoint_frequency() * barrier_interval_ms / 1000,
-        );
-        let created_tables = match self.metadata_manager.get_created_table_ids().await {
-            Ok(created_tables) => created_tables,
-            Err(err) => {
-                tracing::warn!(error = %err.as_report(), "failed to fetch created table ids");
-                return;
-            }
-        };
-        let created_tables: HashSet<u32> = HashSet::from_iter(created_tables);
-        let table_write_throughput = self.history_table_throughput.read().clone();
-        let mut group_infos = self.calculate_compaction_group_statistic().await;
-        group_infos.sort_by_key(|group| group.group_size);
-        group_infos.reverse();
-        const SPLIT_BY_TABLE: u32 = 1;
-
-        let mut group_to_table_vnode_partition = self.group_to_table_vnode_partition.read().clone();
-        for group in &group_infos {
-            if group.table_statistic.len() == 1 {
-                // no need to handle the separate compaciton group
-                continue;
-            }
-
-            let mut table_vnode_partition_mappoing = group_to_table_vnode_partition
-                .entry(group.group_id)
-                .or_default();
-
-            for (table_id, table_size) in &group.table_statistic {
-                let rule = self
-                    .calculate_table_align_rule(
-                        &table_write_throughput,
-                        table_id,
-                        table_size,
-                        !created_tables.contains(table_id),
-                        checkpoint_secs,
-                        group.group_id,
-                        group.group_size,
-                    )
-                    .await;
-
-                match rule {
-                    TableAlignRule::NoOptimization => {
-                        table_vnode_partition_mappoing.remove(table_id);
-                        continue;
-                    }
-
-                    TableAlignRule::SplitByTable(table_id) => {
-                        if self.env.opts.hybird_partition_vnode_count > 0 {
-                            table_vnode_partition_mappoing.insert(table_id, SPLIT_BY_TABLE);
-                        } else {
-                            table_vnode_partition_mappoing.remove(&table_id);
-                        }
-                    }
-
-                    TableAlignRule::SplitByVnode((table_id, vnode)) => {
-                        if self.env.opts.hybird_partition_vnode_count > 0 {
-                            table_vnode_partition_mappoing.insert(table_id, vnode);
-                        } else {
-                            table_vnode_partition_mappoing.remove(&table_id);
-                        }
-                    }
-
-                    TableAlignRule::SplitToDedicatedCg((
-                        new_group_id,
-                        table_vnode_partition_count,
-                    )) => {
-                        let _ = table_vnode_partition_mappoing; // drop
-                        group_to_table_vnode_partition
-                            .insert(new_group_id, table_vnode_partition_count);
-
-                        table_vnode_partition_mappoing = group_to_table_vnode_partition
-                            .entry(group.group_id)
-                            .or_default();
-                    }
-                }
-            }
-        }
-
-        tracing::trace!(
-            "group_to_table_vnode_partition {:?}",
-            group_to_table_vnode_partition
-        );
-
-        // batch update group_to_table_vnode_partition
-        *self.group_to_table_vnode_partition.write() = group_to_table_vnode_partition;
-    }
-
-    /// dedicated event runtime for CPU/IO bound event
-    pub fn compaction_event_loop(
-        hummock_manager: Arc<Self>,
-        mut compactor_streams_change_rx: UnboundedReceiver<(
-            u32,
-            Streaming<SubscribeCompactionEventRequest>,
-        )>,
-    ) -> Vec<(JoinHandle<()>, Sender<()>)> {
-        let mut compactor_request_streams = FuturesUnordered::new();
-        let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
-        let (shutdown_tx_dedicated, shutdown_rx_dedicated) = tokio::sync::oneshot::channel();
-        let shutdown_rx_shared = shutdown_rx.shared();
-        let shutdown_rx_dedicated_shared = shutdown_rx_dedicated.shared();
-
-        let (tx, rx) = unbounded_channel();
-
-        let mut join_handle_vec = Vec::default();
-
-        let hummock_manager_dedicated = hummock_manager.clone();
-        let compact_task_event_handler_join_handle = tokio::spawn(async move {
-            Self::compact_task_dedicated_event_handler(
-                hummock_manager_dedicated,
-                rx,
-                shutdown_rx_dedicated_shared,
-            )
-            .await;
-        });
-
-        join_handle_vec.push((
-            compact_task_event_handler_join_handle,
-            shutdown_tx_dedicated,
-        ));
-
-        let join_handle = tokio::spawn(async move {
-            let push_stream =
-                |context_id: u32,
-                 stream: Streaming<SubscribeCompactionEventRequest>,
-                 compactor_request_streams: &mut FuturesUnordered<_>| {
-                    let future = stream
-                        .into_future()
-                        .map(move |stream_future| (context_id, stream_future));
-
-                    compactor_request_streams.push(future);
-                };
-
-            let mut event_loop_iteration_now = Instant::now();
-
-            loop {
-                let shutdown_rx_shared = shutdown_rx_shared.clone();
-                let hummock_manager = hummock_manager.clone();
-                hummock_manager
-                    .metrics
-                    .compaction_event_loop_iteration_latency
-                    .observe(event_loop_iteration_now.elapsed().as_millis() as _);
-                event_loop_iteration_now = Instant::now();
-
-                tokio::select! {
-                    _ = shutdown_rx_shared => { return; },
-
-                    compactor_stream = compactor_streams_change_rx.recv() => {
-                        if let Some((context_id, stream)) = compactor_stream {
-                            tracing::info!("compactor {} enters the cluster", context_id);
-                            push_stream(context_id, stream, &mut compactor_request_streams);
-                        }
-                    },
-
-                    result = pending_on_none(compactor_request_streams.next()) => {
-                        let mut compactor_alive = true;
-
-                        let (context_id, compactor_stream_req): (_, (std::option::Option<std::result::Result<SubscribeCompactionEventRequest, _>>, _)) = result;
-                        let (event, create_at, stream) = match compactor_stream_req {
-                            (Some(Ok(req)), stream) => {
-                                (req.event.unwrap(), req.create_at, stream)
-                            }
-
-                            (Some(Err(err)), _stream) => {
-                                tracing::warn!(error = %err.as_report(), "compactor stream {} poll with err, recv stream may be destroyed", context_id);
-                                continue
-                            }
-
-                            _ => {
-                                tracing::warn!("compactor stream {} poll err, recv stream may be destroyed", context_id);
-                                continue
-                            },
-                        };
-
-                        {
-                            let consumed_latency_ms = SystemTime::now()
-                                .duration_since(std::time::UNIX_EPOCH)
-                                .expect("Clock may have gone backwards")
-                                .as_millis()
-                                as u64
-                            - create_at;
-                            hummock_manager.metrics
-                                .compaction_event_consumed_latency
-                                .observe(consumed_latency_ms as _);
-                        }
-
-                        match event {
-                            RequestEvent::HeartBeat(HeartBeat {
-                                progress,
-                            }) => {
-                                let compactor_manager = hummock_manager.compactor_manager.clone();
-                                let cancel_tasks = compactor_manager.update_task_heartbeats(&progress);
-                                if let Some(compactor) = compactor_manager.get_compactor(context_id) {
-                                    // TODO: task cancellation can be batched
-                                    for task in cancel_tasks {
-                                        tracing::info!(
-                                            "Task with group_id {} task_id {} with context_id {} has expired due to lack of visible progress",
-                                            task.compaction_group_id,
-                                            task.task_id,
-                                            context_id,
-                                        );
-
-                                        if let Err(e) =
-                                            hummock_manager
-                                            .cancel_compact_task(task.task_id, TaskStatus::HeartbeatCanceled)
-                                            .await
-                                        {
-                                            tracing::error!(
-                                                task_id = task.task_id,
-                                                error = %e.as_report(),
-                                                "Attempt to remove compaction task due to elapsed heartbeat failed. We will continue to track its heartbeat
-                                                until we can successfully report its status."
-                                            );
-                                        }
-
-                                        // Forcefully cancel the task so that it terminates
-                                        // early on the compactor
-                                        // node.
-                                        let _ = compactor.cancel_task(task.task_id);
-                                        tracing::info!(
-                                            "CancelTask operation for task_id {} has been sent to node with context_id {}",
-                                            context_id,
-                                            task.task_id
-                                        );
-                                    }
-                                } else {
-                                    // Determine the validity of the compactor streaming rpc. When the compactor no longer exists in the manager, the stream will be removed.
-                                    // Tip: Connectivity to the compactor will be determined through the `send_event` operation. When send fails, it will be removed from the manager
-                                    compactor_alive = false;
-                                }
-                            },
-
-                            RequestEvent::Register(_) => {
-                                unreachable!()
-                            }
-
-                            e @ (RequestEvent::PullTask(_) | RequestEvent::ReportTask(_)) => {
-                                let _ = tx.send((context_id, e));
-                            }
-                        }
-
-                        if compactor_alive {
-                            push_stream(context_id, stream, &mut compactor_request_streams);
-                        } else {
-                            tracing::warn!("compactor stream {} error, send stream may be destroyed", context_id);
-                        }
-                    },
-                }
-            }
-        });
-
-        join_handle_vec.push((join_handle, shutdown_tx));
-
-        join_handle_vec
-    }
-
-    pub fn add_compactor_stream(
-        &self,
-        context_id: u32,
-        req_stream: Streaming<SubscribeCompactionEventRequest>,
-    ) {
-        self.compactor_streams_change_tx
-            .send((context_id, req_stream))
-            .unwrap();
-    }
-
-    async fn on_handle_trigger_multi_group(&self, task_type: compact_task::TaskType) {
-        for cg_id in self.compaction_group_ids().await {
-            if let Err(e) = self.compaction_state.try_sched_compaction(cg_id, task_type) {
-                tracing::warn!(
-                    error = %e.as_report(),
-                    "Failed to schedule {:?} compaction for compaction group {}",
-                    task_type,
-                    cg_id,
-                );
-            }
-        }
-    }
-
-    pub async fn auto_pick_compaction_group_and_type(
-        &self,
-    ) -> Option<(CompactionGroupId, compact_task::TaskType)> {
-        let mut compaction_group_ids = self.compaction_group_ids().await;
-        compaction_group_ids.shuffle(&mut thread_rng());
-
-        for cg_id in compaction_group_ids {
-            if let Some(pick_type) = self.compaction_state.auto_pick_type(cg_id) {
-                return Some((cg_id, pick_type));
-            }
-        }
-
-        None
-    }
-
-    /// This method will return all compaction group id in a random order and task type. If there are any group block by `write_limit`, it will return a single array with `TaskType::Emergency`.
-    /// If these groups get different task-type, it will return all group id with `TaskType::Dynamic` if the first group get `TaskType::Dynamic`, otherwise it will return the single group with other task type.
-    pub async fn auto_pick_compaction_groups_and_type(
-        &self,
-    ) -> (Vec<CompactionGroupId>, compact_task::TaskType) {
-        let mut compaction_group_ids = self.compaction_group_ids().await;
-        compaction_group_ids.shuffle(&mut thread_rng());
-
-        let mut normal_groups = vec![];
-        for cg_id in compaction_group_ids {
-            if let Some(pick_type) = self.compaction_state.auto_pick_type(cg_id) {
-                if pick_type == TaskType::Dynamic {
-                    normal_groups.push(cg_id);
-                } else if normal_groups.is_empty() {
-                    return (vec![cg_id], pick_type);
-                }
-            }
-        }
-        (normal_groups, TaskType::Dynamic)
-    }
-
-    async fn calculate_table_align_rule(
-        &self,
-        table_write_throughput: &HashMap<u32, VecDeque<u64>>,
-        table_id: &u32,
-        table_size: &u64,
-        is_creating_table: bool,
-        checkpoint_secs: u64,
-        parent_group_id: u64,
-        group_size: u64,
-    ) -> TableAlignRule {
-        let default_group_id: CompactionGroupId = StaticCompactionGroupId::StateDefault.into();
-        let mv_group_id: CompactionGroupId = StaticCompactionGroupId::MaterializedView.into();
-        let partition_vnode_count = self.env.opts.partition_vnode_count;
-        let hybrid_vnode_count: u32 = self.env.opts.hybird_partition_vnode_count;
-        let window_size = HISTORY_TABLE_INFO_STATISTIC_TIME / (checkpoint_secs as usize);
-
-        let mut is_high_write_throughput = false;
-        let mut is_low_write_throughput = true;
-        if let Some(history) = table_write_throughput.get(table_id) {
-            if !is_creating_table {
-                if history.len() >= window_size {
-                    is_high_write_throughput = history.iter().all(|throughput| {
-                        *throughput / checkpoint_secs
-                            > self.env.opts.table_write_throughput_threshold
-                    });
-                    is_low_write_throughput = history.iter().any(|throughput| {
-                        *throughput / checkpoint_secs
-                            < self.env.opts.min_table_split_write_throughput
-                    });
-                }
-            } else {
-                // For creating table, relax the checking restrictions to make the data alignment behavior more sensitive.
-                let sum = history.iter().sum::<u64>();
-                is_low_write_throughput = sum
-                    < self.env.opts.min_table_split_write_throughput
-                        * history.len() as u64
-                        * checkpoint_secs;
-            }
-        }
-
-        let state_table_size = *table_size;
-        let result = {
-            // When in a hybrid compaction group, data from multiple state tables may exist in a single sst, and in order to make the data in the sub level more aligned, a proactive cut is made for the data.
-            // https://github.com/risingwavelabs/risingwave/issues/13037
-            // 1. In some scenario (like backfill), the creating state_table / mv may have high write throughput (creating table ). Therefore, we relax the limit of `is_low_write_throughput` and partition the table with high write throughput by vnode to improve the parallel efficiency of compaction.
-            // Add: creating table is not allowed to be split
-            // 2. For table with low throughput, partition by table_id to minimize amplification.
-            // 3. When the write mode is changed (the above conditions are not met), the default behavior is restored
-            if !is_low_write_throughput {
-                TableAlignRule::SplitByVnode((*table_id, hybrid_vnode_count))
-            } else if state_table_size > self.env.opts.cut_table_size_limit {
-                TableAlignRule::SplitByTable(*table_id)
-            } else {
-                TableAlignRule::NoOptimization
-            }
-        };
-
-        // 1. Avoid splitting a creating table
-        // 2. Avoid splitting a is_low_write_throughput creating table
-        // 3. Avoid splitting a non-high throughput medium-sized table
-        if is_creating_table
-            || (is_low_write_throughput)
-            || (state_table_size < self.env.opts.min_table_split_size && !is_high_write_throughput)
-        {
-            return result;
-        }
-
-        // do not split a large table and a small table because it would increase IOPS
-        // of small table.
-        if parent_group_id != default_group_id && parent_group_id != mv_group_id {
-            let rest_group_size = group_size - state_table_size;
-            if rest_group_size < state_table_size
-                && rest_group_size < self.env.opts.min_table_split_size
-            {
-                return result;
-            }
-        }
-
-        let ret = self
-            .move_state_table_to_compaction_group(
-                parent_group_id,
-                &[*table_id],
-                None,
-                partition_vnode_count,
-            )
-            .await;
-        match ret {
-            Ok((new_group_id, table_vnode_partition_count)) => {
-                tracing::info!("move state table [{}] from group-{} to group-{} success table_vnode_partition_count {:?}", table_id, parent_group_id, new_group_id, table_vnode_partition_count);
-                return TableAlignRule::SplitToDedicatedCg((
-                    new_group_id,
-                    table_vnode_partition_count,
-                ));
-            }
-            Err(e) => {
-                tracing::info!(
-                    error = %e.as_report(),
-                    "failed to move state table [{}] from group-{}",
-                    table_id,
-                    parent_group_id,
-                )
-            }
-        }
-
-        TableAlignRule::NoOptimization
-    }
-
-    async fn initial_compaction_group_config_after_load(
-        &self,
-        versioning_guard: &mut RwLockWriteGuard<'_, Versioning>,
-    ) -> Result<()> {
-        // 1. Due to version compatibility, we fix some of the configuration of older versions after hummock starts.
-        let current_version = &versioning_guard.current_version;
-        let all_group_ids = get_compaction_group_ids(current_version);
-        let mut configs = self
-            .compaction_group_manager
-            .write()
-            .await
-            .get_or_insert_compaction_group_configs(&all_group_ids.collect_vec())
-            .await?;
-
-        // We've already lowered the default limit for write limit in PR-12183, and to prevent older clusters from continuing to use the outdated configuration, we've introduced a new logic to rewrite it in a uniform way.
-        let mut rewrite_cg_ids = vec![];
-        let mut restore_cg_to_partition_vnode: HashMap<u64, BTreeMap<u32, u32>> =
-            HashMap::default();
-        for (cg_id, compaction_group_config) in &mut configs {
-            // update write limit
-            let relaxed_default_write_stop_level_count = 1000;
-            if compaction_group_config
-                .compaction_config
-                .level0_sub_level_compact_level_count
-                == relaxed_default_write_stop_level_count
-            {
-                rewrite_cg_ids.push(*cg_id);
-            }
-
-            if let Some(levels) = current_version.levels.get(cg_id) {
-                if levels.member_table_ids.len() == 1 {
-                    restore_cg_to_partition_vnode.insert(
-                        *cg_id,
-                        vec![(
-                            levels.member_table_ids[0],
-                            compaction_group_config
-                                .compaction_config
-                                .split_weight_by_vnode,
-                        )]
-                        .into_iter()
-                        .collect(),
-                    );
-                }
-            }
-        }
-
-        if !rewrite_cg_ids.is_empty() {
-            tracing::info!("Compaction group {:?} configs rewrite ", rewrite_cg_ids);
-
-            // update meta store
-            let result = self
-                .compaction_group_manager
-                .write()
-                .await
-                .update_compaction_config(
-                    &rewrite_cg_ids,
-                    &[
-                        mutable_config::MutableConfig::Level0StopWriteThresholdSubLevelNumber(
-                            compaction_config::level0_stop_write_threshold_sub_level_number(),
-                        ),
-                    ],
-                )
-                .await?;
-
-            // update memory
-            for new_config in result {
-                configs.insert(new_config.group_id(), new_config);
-            }
-        }
-
-        versioning_guard.write_limit =
-            calc_new_write_limits(configs, HashMap::new(), &versioning_guard.current_version);
-        trigger_write_stop_stats(&self.metrics, &versioning_guard.write_limit);
-        tracing::debug!("Hummock stopped write: {:#?}", versioning_guard.write_limit);
-
-        {
-            // 2. Restore the memory data structure according to the memory of the compaction group config.
-            let mut group_to_table_vnode_partition = self.group_to_table_vnode_partition.write();
-            for (cg_id, table_vnode_partition) in restore_cg_to_partition_vnode {
-                group_to_table_vnode_partition.insert(cg_id, table_vnode_partition);
-            }
-        }
-
-        Ok(())
-    }
-}
-
-// This structure describes how hummock handles sst switching in a compaction group. A better sst cut will result in better data alignment, which in turn will improve the efficiency of the compaction.
-// By adopting certain rules, a better sst cut will lead to better data alignment and thus improve the efficiency of the compaction.
-pub enum TableAlignRule {
-    // The table_id is not optimized for alignment.
-    NoOptimization,
-    // Move the table_id to a separate compaction group. Currently, the system only supports separate compaction with one table.
-    SplitToDedicatedCg((CompactionGroupId, BTreeMap<StateTableId, u32>)),
-    // In the current group, partition the table's data according to the granularity of the vnode.
-    SplitByVnode((StateTableId, u32)),
-    // In the current group, partition the table's data at the granularity of the table.
-    SplitByTable(StateTableId),
-}
-
-fn drop_sst(
-    branched_ssts: &mut BTreeMapTransactionWrapper<'_, HummockSstableObjectId, BranchedSstInfo>,
-    group_id: CompactionGroupId,
-    object_id: HummockSstableObjectId,
-    sst_id: HummockSstableId,
-) -> bool {
-    match branched_ssts.get_mut(object_id) {
-        Some(mut entry) => {
-            // if group_id not exist, it would not pass the stale check before.
-            let removed_sst_id = entry.get(&group_id).unwrap();
-            assert_eq!(*removed_sst_id, sst_id);
-            entry.remove(&group_id);
-            if entry.is_empty() {
-                branched_ssts.remove(object_id);
-                true
-            } else {
-                false
-            }
-        }
-        None => true,
-    }
-}
-
-fn gen_version_delta<'a>(
-    txn: &mut BTreeMapTransactionWrapper<'a, HummockVersionId, HummockVersionDelta>,
-    branched_ssts: &mut BTreeMapTransactionWrapper<'a, HummockSstableObjectId, BranchedSstInfo>,
-    old_version: &HummockVersion,
-    compact_task: &CompactTask,
-    deterministic_mode: bool,
-) -> HummockVersionDelta {
-    let trivial_move = CompactStatus::is_trivial_move_task(compact_task);
-
-    let mut version_delta = HummockVersionDelta {
-        id: old_version.id + 1,
-        prev_id: old_version.id,
-        max_committed_epoch: old_version.max_committed_epoch,
-        trivial_move,
-        ..Default::default()
-    };
-    let group_deltas = &mut version_delta
-        .group_deltas
-        .entry(compact_task.compaction_group_id)
-        .or_default()
-        .group_deltas;
-    let mut removed_table_ids_map: BTreeMap<u32, Vec<u64>> = BTreeMap::default();
-
-    for level in &compact_task.input_ssts {
-        let level_idx = level.level_idx;
-        let mut removed_table_ids = level
-            .table_infos
-            .iter()
-            .map(|sst| {
-                let object_id = sst.get_object_id();
-                let sst_id = sst.get_sst_id();
-                if !trivial_move {
-                    drop_sst(
-                        branched_ssts,
-                        compact_task.compaction_group_id,
-                        object_id,
-                        sst_id,
-                    );
-                }
-                sst_id
-            })
-            .collect_vec();
-
-        removed_table_ids_map
-            .entry(level_idx)
-            .or_default()
-            .append(&mut removed_table_ids);
-    }
-
-    for (level_idx, removed_table_ids) in removed_table_ids_map {
-        let group_delta = GroupDelta {
-            delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta {
-                level_idx,
-                removed_table_ids,
-                ..Default::default()
-            })),
-        };
-        group_deltas.push(group_delta);
-    }
-
-    let group_delta = GroupDelta {
-        delta_type: Some(DeltaType::IntraLevel(IntraLevelDelta {
-            level_idx: compact_task.target_level,
-            inserted_table_infos: compact_task.sorted_output_ssts.clone(),
-            l0_sub_level_id: compact_task.target_sub_level_id,
-            vnode_partition_count: compact_task.split_weight_by_vnode,
-            ..Default::default()
-        })),
-    };
-    group_deltas.push(group_delta);
-    version_delta.safe_epoch = std::cmp::max(old_version.safe_epoch, compact_task.watermark);
-
-    // Don't persist version delta generated by compaction to meta store in deterministic mode.
-    // Because it will override existing version delta that has same ID generated in the data
-    // ingestion phase.
-    if !deterministic_mode {
-        txn.insert(version_delta.id, version_delta.clone());
-    }
-
-    version_delta
-}
-
-async fn write_exclusive_cluster_id(
-    state_store_dir: &str,
-    cluster_id: ClusterId,
-    object_store: ObjectStoreRef,
-) -> Result<()> {
-    const CLUSTER_ID_DIR: &str = "cluster_id";
-    const CLUSTER_ID_NAME: &str = "0";
-    let cluster_id_dir = format!("{}/{}/", state_store_dir, CLUSTER_ID_DIR);
-    let cluster_id_full_path = format!("{}{}", cluster_id_dir, CLUSTER_ID_NAME);
-    match object_store.read(&cluster_id_full_path, ..).await {
-        Ok(stored_cluster_id) => {
-            let stored_cluster_id = String::from_utf8(stored_cluster_id.to_vec()).unwrap();
-            if cluster_id.deref() == stored_cluster_id {
-                return Ok(());
-            }
-
-            Err(ObjectError::internal(format!(
-                "Data directory is already used by another cluster with id {:?}, path {}.",
-                stored_cluster_id, cluster_id_full_path,
-            ))
-            .into())
-        }
-        Err(e) => {
-            if e.is_object_not_found_error() {
-                object_store
-                    .upload(&cluster_id_full_path, Bytes::from(String::from(cluster_id)))
-                    .await?;
-                return Ok(());
+        Err(e) => {
+            if e.is_object_not_found_error() {
+                object_store
+                    .upload(&cluster_id_full_path, Bytes::from(String::from(cluster_id)))
+                    .await?;
+                return Ok(());
             }
             Err(e.into())
         }
     }
 }
-
-fn init_selectors() -> HashMap<compact_task::TaskType, Box<dyn CompactionSelector>> {
-    let mut compaction_selectors: HashMap<compact_task::TaskType, Box<dyn CompactionSelector>> =
-        HashMap::default();
-    compaction_selectors.insert(
-        compact_task::TaskType::Dynamic,
-        Box::<DynamicLevelSelector>::default(),
-    );
-    compaction_selectors.insert(
-        compact_task::TaskType::SpaceReclaim,
-        Box::<SpaceReclaimCompactionSelector>::default(),
-    );
-    compaction_selectors.insert(
-        compact_task::TaskType::Ttl,
-        Box::<TtlCompactionSelector>::default(),
-    );
-    compaction_selectors.insert(
-        compact_task::TaskType::Tombstone,
-        Box::<TombstoneCompactionSelector>::default(),
-    );
-    compaction_selectors
-}
-
-type CompactionRequestChannelItem = (CompactionGroupId, compact_task::TaskType);
-use risingwave_hummock_sdk::table_watermark::TableWatermarks;
-use risingwave_hummock_sdk::version::HummockVersion;
-use tokio::sync::mpsc::error::SendError;
-
-use super::compaction::CompactionSelector;
-use crate::hummock::manager::checkpoint::HummockVersionCheckpoint;
-use crate::hummock::sequence::next_sstable_object_id;
-
-#[derive(Debug, Default)]
-pub struct CompactionState {
-    scheduled: Mutex<HashSet<(CompactionGroupId, compact_task::TaskType)>>,
-}
-
-impl CompactionState {
-    pub fn new() -> Self {
-        Self {
-            scheduled: Default::default(),
-        }
-    }
-
-    /// Enqueues only if the target is not yet in queue.
-    pub fn try_sched_compaction(
-        &self,
-        compaction_group: CompactionGroupId,
-        task_type: TaskType,
-    ) -> std::result::Result<bool, SendError<CompactionRequestChannelItem>> {
-        let mut guard = self.scheduled.lock();
-        let key = (compaction_group, task_type);
-        if guard.contains(&key) {
-            return Ok(false);
-        }
-        guard.insert(key);
-        Ok(true)
-    }
-
-    pub fn unschedule(
-        &self,
-        compaction_group: CompactionGroupId,
-        task_type: compact_task::TaskType,
-    ) {
-        self.scheduled.lock().remove(&(compaction_group, task_type));
-    }
-
-    pub fn auto_pick_type(&self, group: CompactionGroupId) -> Option<TaskType> {
-        let guard = self.scheduled.lock();
-        if guard.contains(&(group, compact_task::TaskType::Dynamic)) {
-            Some(compact_task::TaskType::Dynamic)
-        } else if guard.contains(&(group, compact_task::TaskType::SpaceReclaim)) {
-            Some(compact_task::TaskType::SpaceReclaim)
-        } else if guard.contains(&(group, compact_task::TaskType::Ttl)) {
-            Some(compact_task::TaskType::Ttl)
-        } else if guard.contains(&(group, compact_task::TaskType::Tombstone)) {
-            Some(compact_task::TaskType::Tombstone)
-        } else {
-            None
-        }
-    }
-}
diff --git a/src/meta/src/hummock/manager/tests.rs b/src/meta/src/hummock/manager/tests.rs
index 65ddef44af8a1..8d4fcf89b33be 100644
--- a/src/meta/src/hummock/manager/tests.rs
+++ b/src/meta/src/hummock/manager/tests.rs
@@ -12,18 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![cfg(test)]
+
 use std::borrow::Borrow;
 use std::cmp::Ordering;
-use std::collections::{BTreeMap, HashMap};
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use itertools::Itertools;
 use prometheus::Registry;
 use risingwave_common::util::epoch::{test_epoch, EpochExt, INVALID_EPOCH};
 use risingwave_hummock_sdk::compact::compact_task_to_string;
-use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
-    get_compaction_group_ssts, BranchedSstInfo,
-};
+use risingwave_hummock_sdk::compaction_group::hummock_version_ext::get_compaction_group_ssts;
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_sdk::table_stats::{to_prost_table_stats_map, TableStats, TableStatsMap};
 use risingwave_hummock_sdk::version::HummockVersion;
@@ -100,7 +100,7 @@ fn get_compaction_group_object_ids(
 }
 
 async fn list_pinned_snapshot_from_meta_store(env: &MetaSrvEnv) -> Vec<HummockPinnedSnapshot> {
-    match env.meta_store() {
+    match env.meta_store_ref() {
         MetaStoreImpl::Kv(meta_store) => HummockPinnedSnapshot::list(meta_store).await.unwrap(),
         MetaStoreImpl::Sql(sql_meta_store) => {
             use risingwave_meta_model_v2::hummock_pinned_snapshot;
@@ -117,7 +117,7 @@ async fn list_pinned_snapshot_from_meta_store(env: &MetaSrvEnv) -> Vec<HummockPi
 }
 
 async fn list_pinned_version_from_meta_store(env: &MetaSrvEnv) -> Vec<HummockPinnedVersion> {
-    match env.meta_store() {
+    match env.meta_store_ref() {
         MetaStoreImpl::Kv(meta_store) => HummockPinnedVersion::list(meta_store).await.unwrap(),
         MetaStoreImpl::Sql(sql_meta_store) => {
             use risingwave_meta_model_v2::hummock_pinned_version;
@@ -590,7 +590,7 @@ async fn test_hummock_manager_basic() {
         );
     }
     // objects_to_delete is always empty because no compaction is ever invoked.
-    assert!(hummock_manager.get_objects_to_delete().await.is_empty());
+    assert!(hummock_manager.get_objects_to_delete().is_empty());
     assert_eq!(
         hummock_manager
             .delete_version_deltas(usize::MAX)
@@ -602,7 +602,7 @@ async fn test_hummock_manager_basic() {
         hummock_manager.create_version_checkpoint(1).await.unwrap(),
         commit_log_count + register_log_count
     );
-    assert!(hummock_manager.get_objects_to_delete().await.is_empty());
+    assert!(hummock_manager.get_objects_to_delete().is_empty());
     assert_eq!(
         hummock_manager
             .delete_version_deltas(usize::MAX)
@@ -1127,7 +1127,7 @@ async fn test_extend_objects_to_delete() {
         .map(|s| s.get_object_id())
         .chain(max_committed_object_id + 1..=max_committed_object_id + orphan_sst_num)
         .collect_vec();
-    assert!(hummock_manager.get_objects_to_delete().await.is_empty());
+    assert!(hummock_manager.get_objects_to_delete().is_empty());
     assert_eq!(
         hummock_manager
             .extend_objects_to_delete_from_scan(&all_object_ids)
@@ -1135,7 +1135,7 @@ async fn test_extend_objects_to_delete() {
         orphan_sst_num as usize
     );
     assert_eq!(
-        hummock_manager.get_objects_to_delete().await.len(),
+        hummock_manager.get_objects_to_delete().len(),
         orphan_sst_num as usize
     );
 
@@ -1145,7 +1145,7 @@ async fn test_extend_objects_to_delete() {
         6
     );
     assert_eq!(
-        hummock_manager.get_objects_to_delete().await.len(),
+        hummock_manager.get_objects_to_delete().len(),
         orphan_sst_num as usize
     );
     // since version1 is still pinned, the sst removed in compaction can not be reclaimed.
@@ -1155,10 +1155,10 @@ async fn test_extend_objects_to_delete() {
             .await,
         orphan_sst_num as usize
     );
-    let objects_to_delete = hummock_manager.get_objects_to_delete().await;
+    let objects_to_delete = hummock_manager.get_objects_to_delete();
     assert_eq!(objects_to_delete.len(), orphan_sst_num as usize);
     let pinned_version2: HummockVersion = hummock_manager.pin_version(context_id).await.unwrap();
-    let objects_to_delete = hummock_manager.get_objects_to_delete().await;
+    let objects_to_delete = hummock_manager.get_objects_to_delete();
     assert_eq!(
         objects_to_delete.len(),
         orphan_sst_num as usize,
@@ -1169,7 +1169,7 @@ async fn test_extend_objects_to_delete() {
         .unpin_version_before(context_id, pinned_version2.id)
         .await
         .unwrap();
-    let objects_to_delete = hummock_manager.get_objects_to_delete().await;
+    let objects_to_delete = hummock_manager.get_objects_to_delete();
     assert_eq!(
         objects_to_delete.len(),
         orphan_sst_num as usize,
@@ -1184,7 +1184,7 @@ async fn test_extend_objects_to_delete() {
             .await,
         orphan_sst_num as usize
     );
-    let objects_to_delete = hummock_manager.get_objects_to_delete().await;
+    let objects_to_delete = hummock_manager.get_objects_to_delete();
     assert_eq!(objects_to_delete.len(), orphan_sst_num as usize);
     let new_epoch = pinned_version2.max_committed_epoch.next_epoch();
     hummock_manager
@@ -1208,7 +1208,7 @@ async fn test_extend_objects_to_delete() {
             .await,
         orphan_sst_num as usize + 3
     );
-    let objects_to_delete = hummock_manager.get_objects_to_delete().await;
+    let objects_to_delete = hummock_manager.get_objects_to_delete();
     assert_eq!(objects_to_delete.len(), orphan_sst_num as usize + 3);
 }
 
@@ -1339,11 +1339,11 @@ async fn test_split_compaction_group_on_commit() {
     let (_env, hummock_manager, _, worker_node) = setup_compute_env(80).await;
     let context_id = worker_node.id;
     hummock_manager
-        .register_table_ids(&[(100, 2)])
+        .register_table_ids_for_test(&[(100, 2)])
         .await
         .unwrap();
     hummock_manager
-        .register_table_ids(&[(101, 3)])
+        .register_table_ids_for_test(&[(101, 3)])
         .await
         .unwrap();
     let sst_1 = ExtendedSstableInfo {
@@ -1378,51 +1378,22 @@ async fn test_split_compaction_group_on_commit() {
     );
     assert_eq!(
         current_version
-            .get_compaction_group_levels(2)
-            .member_table_ids,
+            .state_table_info
+            .compaction_group_member_table_ids(2)
+            .iter()
+            .map(|table_id| table_id.table_id)
+            .collect_vec(),
         vec![100]
     );
     assert_eq!(
         current_version
-            .get_compaction_group_levels(3)
-            .member_table_ids,
+            .state_table_info
+            .compaction_group_member_table_ids(3)
+            .iter()
+            .map(|table_id| table_id.table_id)
+            .collect_vec(),
         vec![101]
     );
-    let branched_ssts = hummock_manager
-        .versioning
-        .read(&["", "", ""])
-        .await
-        .branched_ssts
-        .clone();
-    assert_eq!(branched_ssts.len(), 1);
-    assert_eq!(branched_ssts.values().next().unwrap().len(), 2);
-    assert_ne!(
-        branched_ssts
-            .values()
-            .next()
-            .unwrap()
-            .get(&2)
-            .cloned()
-            .unwrap(),
-        branched_ssts
-            .values()
-            .next()
-            .unwrap()
-            .get(&3)
-            .cloned()
-            .unwrap(),
-    );
-}
-
-async fn get_branched_ssts(
-    hummock_manager: &HummockManager,
-) -> BTreeMap<HummockSstableObjectId, BranchedSstInfo> {
-    hummock_manager
-        .versioning
-        .read(&["", "", ""])
-        .await
-        .branched_ssts
-        .clone()
 }
 
 #[tokio::test]
@@ -1460,11 +1431,11 @@ async fn test_split_compaction_group_on_demand_basic() {
     );
 
     hummock_manager
-        .register_table_ids(&[(100, 2)])
+        .register_table_ids_for_test(&[(100, 2)])
         .await
         .unwrap();
     hummock_manager
-        .register_table_ids(&[(101, 2)])
+        .register_table_ids_for_test(&[(101, 2)])
         .await
         .unwrap();
     let sst_1 = ExtendedSstableInfo {
@@ -1524,7 +1495,7 @@ async fn test_split_compaction_group_on_demand_basic() {
     // Now group 2 has member tables [100,101,102], so split [100, 101] can succeed even though
     // there is no data of 102.
     hummock_manager
-        .register_table_ids(&[(102, 2)])
+        .register_table_ids_for_test(&[(102, 2)])
         .await
         .unwrap();
 
@@ -1546,31 +1517,23 @@ async fn test_split_compaction_group_on_demand_basic() {
     );
     assert_eq!(
         current_version
-            .get_compaction_group_levels(2)
-            .member_table_ids,
+            .state_table_info
+            .compaction_group_member_table_ids(2)
+            .iter()
+            .map(|table_id| table_id.table_id)
+            .collect_vec(),
         vec![102]
     );
     assert_eq!(
         current_version
-            .get_compaction_group_levels(new_group_id)
-            .member_table_ids,
+            .state_table_info
+            .compaction_group_member_table_ids(new_group_id)
+            .iter()
+            .map(|table_id| table_id.table_id)
+            .sorted()
+            .collect_vec(),
         vec![100, 101]
     );
-    let branched_ssts = get_branched_ssts(&hummock_manager).await;
-    assert_eq!(branched_ssts.len(), 2);
-    for object_id in [10, 11] {
-        assert_eq!(branched_ssts.get(&object_id).unwrap().len(), 2);
-        assert_ne!(
-            branched_ssts
-                .get(&object_id)
-                .unwrap()
-                .get(&new_group_id)
-                .cloned()
-                .unwrap(),
-            object_id,
-            "trivial adjust should also generate a new SST id"
-        );
-    }
 }
 
 #[tokio::test]
@@ -1591,11 +1554,11 @@ async fn test_split_compaction_group_on_demand_non_trivial() {
         table_stats: Default::default(),
     };
     hummock_manager
-        .register_table_ids(&[(100, 2)])
+        .register_table_ids_for_test(&[(100, 2)])
         .await
         .unwrap();
     hummock_manager
-        .register_table_ids(&[(101, 2)])
+        .register_table_ids_for_test(&[(101, 2)])
         .await
         .unwrap();
     hummock_manager
@@ -1625,30 +1588,22 @@ async fn test_split_compaction_group_on_demand_non_trivial() {
     );
     assert_eq!(
         current_version
-            .get_compaction_group_levels(2)
-            .member_table_ids,
+            .state_table_info
+            .compaction_group_member_table_ids(2)
+            .iter()
+            .map(|table_id| table_id.table_id)
+            .collect_vec(),
         vec![101]
     );
     assert_eq!(
         current_version
-            .get_compaction_group_levels(new_group_id)
-            .member_table_ids,
+            .state_table_info
+            .compaction_group_member_table_ids(new_group_id)
+            .iter()
+            .map(|table_id| table_id.table_id)
+            .collect_vec(),
         vec![100]
     );
-    let branched_ssts = get_branched_ssts(&hummock_manager).await;
-    assert_eq!(branched_ssts.len(), 1);
-    assert_eq!(branched_ssts.get(&10).unwrap().len(), 2);
-    let sst_ids = branched_ssts.get(&10).unwrap().get(&2).cloned().unwrap();
-    assert_ne!(sst_ids, 10);
-    assert_ne!(
-        branched_ssts
-            .get(&10)
-            .unwrap()
-            .get(&new_group_id)
-            .cloned()
-            .unwrap(),
-        sst_ids,
-    );
 }
 
 #[tokio::test]
@@ -1667,11 +1622,11 @@ async fn test_split_compaction_group_trivial_expired() {
     hummock_manager.compactor_manager.add_compactor(context_id);
 
     hummock_manager
-        .register_table_ids(&[(100, 2)])
+        .register_table_ids_for_test(&[(100, 2)])
         .await
         .unwrap();
     hummock_manager
-        .register_table_ids(&[(101, 2)])
+        .register_table_ids_for_test(&[(101, 2)])
         .await
         .unwrap();
     let sst_1 = ExtendedSstableInfo {
@@ -1733,7 +1688,7 @@ async fn test_split_compaction_group_trivial_expired() {
     // Now group 2 has member tables [100,101,102], so split [100, 101] can succeed even though
     // there is no data of 102.
     hummock_manager
-        .register_table_ids(&[(102, 2)])
+        .register_table_ids_for_test(&[(102, 2)])
         .await
         .unwrap();
     let task = hummock_manager
@@ -1762,14 +1717,21 @@ async fn test_split_compaction_group_trivial_expired() {
     assert!(new_group_id > StaticCompactionGroupId::End as u64);
     assert_eq!(
         current_version
-            .get_compaction_group_levels(2)
-            .member_table_ids,
+            .state_table_info
+            .compaction_group_member_table_ids(2)
+            .iter()
+            .map(|table_id| table_id.table_id)
+            .sorted()
+            .collect_vec(),
         vec![101, 102]
     );
     assert_eq!(
         current_version
-            .get_compaction_group_levels(new_group_id)
-            .member_table_ids,
+            .state_table_info
+            .compaction_group_member_table_ids(new_group_id)
+            .iter()
+            .map(|table_id| table_id.table_id)
+            .collect_vec(),
         vec![100]
     );
 
@@ -1829,11 +1791,11 @@ async fn test_split_compaction_group_on_demand_bottom_levels() {
     let context_id = worker_node.id;
 
     hummock_manager
-        .register_table_ids(&[(100, 2)])
+        .register_table_ids_for_test(&[(100, 2)])
         .await
         .unwrap();
     hummock_manager
-        .register_table_ids(&[(101, 2)])
+        .register_table_ids_for_test(&[(101, 2)])
         .await
         .unwrap();
 
@@ -1927,14 +1889,6 @@ async fn test_split_compaction_group_on_demand_bottom_levels() {
         2
     );
 
-    let branched_ssts = hummock_manager.get_branched_ssts_info().await;
-    // object-11 and object-12
-    assert_eq!(branched_ssts.len(), 2);
-    let info = branched_ssts.get(&11).unwrap();
-    assert_eq!(
-        info.keys().sorted().cloned().collect_vec(),
-        vec![2, new_group_id]
-    );
     assert_eq!(
         current_version.get_compaction_group_levels(2).levels[base_level - 1].table_infos[0]
             .object_id,
@@ -1977,11 +1931,11 @@ async fn test_compaction_task_expiration_due_to_split_group() {
     let context_id = worker_node.id;
 
     hummock_manager
-        .register_table_ids(&[(100, 2)])
+        .register_table_ids_for_test(&[(100, 2)])
         .await
         .unwrap();
     hummock_manager
-        .register_table_ids(&[(101, 2)])
+        .register_table_ids_for_test(&[(101, 2)])
         .await
         .unwrap();
     let sst_1 = ExtendedSstableInfo {
@@ -2068,15 +2022,15 @@ async fn test_move_tables_between_compaction_group() {
     let context_id = worker_node.id;
 
     hummock_manager
-        .register_table_ids(&[(100, 2)])
+        .register_table_ids_for_test(&[(100, 2)])
         .await
         .unwrap();
     hummock_manager
-        .register_table_ids(&[(101, 2)])
+        .register_table_ids_for_test(&[(101, 2)])
         .await
         .unwrap();
     hummock_manager
-        .register_table_ids(&[(102, 2)])
+        .register_table_ids_for_test(&[(102, 2)])
         .await
         .unwrap();
     let sst_1 = gen_extend_sstable_info(10, 2, 1, vec![100, 101, 102]);
@@ -2140,12 +2094,6 @@ async fn test_move_tables_between_compaction_group() {
     assert_eq!(level.table_infos[0].table_ids, vec![100]);
     assert_eq!(level.table_infos[1].table_ids, vec![100, 101]);
     assert_eq!(level.table_infos.len(), 2);
-    let branched_ssts = hummock_manager.get_branched_ssts_info().await;
-    // object-id 11 and 12.
-    assert_eq!(branched_ssts.len(), 2);
-    let info = branched_ssts.get(&12).unwrap();
-    let groups = info.keys().sorted().cloned().collect_vec();
-    assert_eq!(groups, vec![2, new_group_id]);
 
     let mut selector: Box<dyn CompactionSelector> =
         Box::<SpaceReclaimCompactionSelector>::default();
@@ -2155,9 +2103,9 @@ async fn test_move_tables_between_compaction_group() {
         .await
         .unwrap()
         .unwrap();
-    assert_eq!(compaction_task.existing_table_ids, vec![101, 102]);
     assert_eq!(compaction_task.input_ssts[0].table_infos.len(), 1);
     assert_eq!(compaction_task.input_ssts[0].table_infos[0].object_id, 12);
+    assert_eq!(compaction_task.existing_table_ids, vec![101]);
 
     let ret = hummock_manager
         .report_compact_task(
@@ -2169,9 +2117,6 @@ async fn test_move_tables_between_compaction_group() {
         .await
         .unwrap();
     assert!(ret);
-    let branched_ssts = hummock_manager.get_branched_ssts_info().await;
-    // there is still left one sst for object-12 in branched-sst.
-    assert_eq!(branched_ssts.len(), 2);
 }
 
 #[tokio::test]
@@ -2259,11 +2204,11 @@ async fn test_partition_level() {
     let context_id = worker_node.id;
 
     hummock_manager
-        .register_table_ids(&[(100, 2)])
+        .register_table_ids_for_test(&[(100, 2)])
         .await
         .unwrap();
     hummock_manager
-        .register_table_ids(&[(101, 2)])
+        .register_table_ids_for_test(&[(101, 2)])
         .await
         .unwrap();
     let sst_1 = gen_extend_sstable_info(10, 2, 1, vec![100, 101]);
@@ -2354,3 +2299,135 @@ async fn test_partition_level() {
         }
     }
 }
+
+#[tokio::test]
+async fn test_unregister_moved_table() {
+    let (_env, hummock_manager, _, worker_node) = setup_compute_env(80).await;
+    let context_id = worker_node.id;
+    let original_groups = hummock_manager
+        .get_current_version()
+        .await
+        .levels
+        .keys()
+        .cloned()
+        .sorted()
+        .collect_vec();
+    assert_eq!(
+        original_groups,
+        vec![
+            StaticCompactionGroupId::StateDefault as u64,
+            StaticCompactionGroupId::MaterializedView as u64
+        ]
+    );
+
+    hummock_manager
+        .register_table_ids_for_test(&[(100, 2)])
+        .await
+        .unwrap();
+    hummock_manager
+        .register_table_ids_for_test(&[(101, 2)])
+        .await
+        .unwrap();
+    let sst_1 = ExtendedSstableInfo {
+        compaction_group_id: 2,
+        sst_info: SstableInfo {
+            object_id: 10,
+            sst_id: 10,
+            key_range: Some(KeyRange {
+                left: iterator_test_key_of_epoch(100, 1, 20),
+                right: iterator_test_key_of_epoch(100, 100, 20),
+                right_exclusive: false,
+            }),
+            table_ids: vec![100],
+            min_epoch: 20,
+            max_epoch: 20,
+            ..Default::default()
+        },
+        table_stats: Default::default(),
+    };
+    let sst_2 = ExtendedSstableInfo {
+        compaction_group_id: 2,
+        sst_info: SstableInfo {
+            object_id: 11,
+            sst_id: 11,
+            key_range: Some(KeyRange {
+                left: iterator_test_key_of_epoch(100, 101, 20),
+                right: iterator_test_key_of_epoch(101, 100, 20),
+                right_exclusive: false,
+            }),
+            table_ids: vec![100, 101],
+            min_epoch: 20,
+            max_epoch: 20,
+            ..Default::default()
+        },
+        table_stats: Default::default(),
+    };
+    hummock_manager
+        .commit_epoch(
+            30,
+            CommitEpochInfo::for_test(
+                vec![sst_1, sst_2],
+                HashMap::from([(10, context_id), (11, context_id)]),
+            ),
+        )
+        .await
+        .unwrap();
+
+    let new_group_id = hummock_manager
+        .split_compaction_group(2, &[100])
+        .await
+        .unwrap();
+    assert_ne!(new_group_id, 2);
+    assert!(new_group_id > StaticCompactionGroupId::End as u64);
+
+    let current_version = hummock_manager.get_current_version().await;
+    assert_eq!(
+        new_group_id,
+        current_version.levels.keys().max().cloned().unwrap()
+    );
+    assert_eq!(current_version.levels.len(), 3);
+    assert_eq!(
+        get_compaction_group_object_ids(&current_version, 2),
+        vec![10, 11]
+    );
+    assert_eq!(
+        get_compaction_group_object_ids(&current_version, new_group_id),
+        vec![10, 11]
+    );
+    assert_eq!(
+        current_version
+            .state_table_info
+            .compaction_group_member_table_ids(2)
+            .iter()
+            .map(|table_id| table_id.table_id)
+            .collect_vec(),
+        vec![101]
+    );
+    assert_eq!(
+        current_version
+            .state_table_info
+            .compaction_group_member_table_ids(new_group_id)
+            .iter()
+            .map(|table_id| table_id.table_id)
+            .collect_vec(),
+        vec![100]
+    );
+
+    hummock_manager.unregister_table_ids(&[100]).await.unwrap();
+    let current_version = hummock_manager.get_current_version().await;
+    assert_eq!(current_version.levels.len(), 2);
+    assert!(!current_version.levels.contains_key(&new_group_id));
+    assert_eq!(
+        get_compaction_group_object_ids(&current_version, 2),
+        vec![10, 11]
+    );
+    assert_eq!(
+        current_version
+            .state_table_info
+            .compaction_group_member_table_ids(2)
+            .iter()
+            .map(|table_id| table_id.table_id)
+            .collect_vec(),
+        vec![101]
+    );
+}
diff --git a/src/meta/src/hummock/manager/timer_task.rs b/src/meta/src/hummock/manager/timer_task.rs
new file mode 100644
index 0000000000000..b7c8cae4b260e
--- /dev/null
+++ b/src/meta/src/hummock/manager/timer_task.rs
@@ -0,0 +1,586 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{HashMap, HashSet, VecDeque};
+use std::sync::Arc;
+use std::time::Duration;
+
+use futures::future::Either;
+use futures::stream::BoxStream;
+use futures::{FutureExt, StreamExt};
+use itertools::Itertools;
+use risingwave_common::system_param::reader::SystemParamsRead;
+use risingwave_hummock_sdk::compaction_group::hummock_version_ext::get_compaction_group_ids;
+use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
+use risingwave_hummock_sdk::CompactionGroupId;
+use risingwave_pb::hummock::compact_task::{self, TaskStatus};
+use risingwave_pb::hummock::level_handler::RunningCompactTask;
+use rw_futures_util::select_all;
+use thiserror_ext::AsReport;
+use tokio::sync::oneshot::Sender;
+use tokio::task::JoinHandle;
+use tokio_stream::wrappers::IntervalStream;
+use tracing::warn;
+
+use crate::hummock::manager::HISTORY_TABLE_INFO_STATISTIC_TIME;
+use crate::hummock::metrics_utils::{trigger_lsm_stat, trigger_mv_stat};
+use crate::hummock::{HummockManager, TASK_NORMAL};
+
+impl HummockManager {
+    pub fn hummock_timer_task(hummock_manager: Arc<Self>) -> (JoinHandle<()>, Sender<()>) {
+        let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
+        let join_handle = tokio::spawn(async move {
+            const CHECK_PENDING_TASK_PERIOD_SEC: u64 = 300;
+            const STAT_REPORT_PERIOD_SEC: u64 = 20;
+            const COMPACTION_HEARTBEAT_PERIOD_SEC: u64 = 1;
+
+            pub enum HummockTimerEvent {
+                GroupSplit,
+                CheckDeadTask,
+                Report,
+                CompactionHeartBeatExpiredCheck,
+
+                DynamicCompactionTrigger,
+                SpaceReclaimCompactionTrigger,
+                TtlCompactionTrigger,
+                TombstoneCompactionTrigger,
+
+                FullGc,
+            }
+            let mut check_compact_trigger_interval =
+                tokio::time::interval(Duration::from_secs(CHECK_PENDING_TASK_PERIOD_SEC));
+            check_compact_trigger_interval
+                .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+            check_compact_trigger_interval.reset();
+
+            let check_compact_trigger = IntervalStream::new(check_compact_trigger_interval)
+                .map(|_| HummockTimerEvent::CheckDeadTask);
+
+            let mut stat_report_interval =
+                tokio::time::interval(std::time::Duration::from_secs(STAT_REPORT_PERIOD_SEC));
+            stat_report_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+            stat_report_interval.reset();
+            let stat_report_trigger =
+                IntervalStream::new(stat_report_interval).map(|_| HummockTimerEvent::Report);
+
+            let mut compaction_heartbeat_interval = tokio::time::interval(
+                std::time::Duration::from_secs(COMPACTION_HEARTBEAT_PERIOD_SEC),
+            );
+            compaction_heartbeat_interval
+                .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+            compaction_heartbeat_interval.reset();
+            let compaction_heartbeat_trigger = IntervalStream::new(compaction_heartbeat_interval)
+                .map(|_| HummockTimerEvent::CompactionHeartBeatExpiredCheck);
+
+            let mut min_trigger_interval = tokio::time::interval(Duration::from_secs(
+                hummock_manager.env.opts.periodic_compaction_interval_sec,
+            ));
+            min_trigger_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+            min_trigger_interval.reset();
+            let dynamic_tick_trigger = IntervalStream::new(min_trigger_interval)
+                .map(|_| HummockTimerEvent::DynamicCompactionTrigger);
+
+            let mut min_space_reclaim_trigger_interval =
+                tokio::time::interval(Duration::from_secs(
+                    hummock_manager
+                        .env
+                        .opts
+                        .periodic_space_reclaim_compaction_interval_sec,
+                ));
+            min_space_reclaim_trigger_interval
+                .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+            min_space_reclaim_trigger_interval.reset();
+            let space_reclaim_trigger = IntervalStream::new(min_space_reclaim_trigger_interval)
+                .map(|_| HummockTimerEvent::SpaceReclaimCompactionTrigger);
+
+            let mut min_ttl_reclaim_trigger_interval = tokio::time::interval(Duration::from_secs(
+                hummock_manager
+                    .env
+                    .opts
+                    .periodic_ttl_reclaim_compaction_interval_sec,
+            ));
+            min_ttl_reclaim_trigger_interval
+                .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+            min_ttl_reclaim_trigger_interval.reset();
+            let ttl_reclaim_trigger = IntervalStream::new(min_ttl_reclaim_trigger_interval)
+                .map(|_| HummockTimerEvent::TtlCompactionTrigger);
+
+            let mut full_gc_interval = tokio::time::interval(Duration::from_secs(
+                hummock_manager.env.opts.full_gc_interval_sec,
+            ));
+            full_gc_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+            full_gc_interval.reset();
+            let full_gc_trigger =
+                IntervalStream::new(full_gc_interval).map(|_| HummockTimerEvent::FullGc);
+
+            let mut tombstone_reclaim_trigger_interval =
+                tokio::time::interval(Duration::from_secs(
+                    hummock_manager
+                        .env
+                        .opts
+                        .periodic_tombstone_reclaim_compaction_interval_sec,
+                ));
+            tombstone_reclaim_trigger_interval
+                .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+            tombstone_reclaim_trigger_interval.reset();
+            let tombstone_reclaim_trigger = IntervalStream::new(tombstone_reclaim_trigger_interval)
+                .map(|_| HummockTimerEvent::TombstoneCompactionTrigger);
+
+            let mut triggers: Vec<BoxStream<'static, HummockTimerEvent>> = vec![
+                Box::pin(check_compact_trigger),
+                Box::pin(stat_report_trigger),
+                Box::pin(compaction_heartbeat_trigger),
+                Box::pin(dynamic_tick_trigger),
+                Box::pin(space_reclaim_trigger),
+                Box::pin(ttl_reclaim_trigger),
+                Box::pin(full_gc_trigger),
+                Box::pin(tombstone_reclaim_trigger),
+            ];
+
+            let periodic_check_split_group_interval_sec = hummock_manager
+                .env
+                .opts
+                .periodic_split_compact_group_interval_sec;
+
+            if periodic_check_split_group_interval_sec > 0 {
+                let mut split_group_trigger_interval = tokio::time::interval(Duration::from_secs(
+                    periodic_check_split_group_interval_sec,
+                ));
+                split_group_trigger_interval
+                    .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+
+                let split_group_trigger = IntervalStream::new(split_group_trigger_interval)
+                    .map(|_| HummockTimerEvent::GroupSplit);
+                triggers.push(Box::pin(split_group_trigger));
+            }
+
+            let event_stream = select_all(triggers);
+            use futures::pin_mut;
+            pin_mut!(event_stream);
+
+            let shutdown_rx_shared = shutdown_rx.shared();
+
+            tracing::info!(
+                "Hummock timer task tracing [GroupSplit interval {} sec] [CheckDeadTask interval {} sec] [Report interval {} sec] [CompactionHeartBeat interval {} sec]",
+                    periodic_check_split_group_interval_sec, CHECK_PENDING_TASK_PERIOD_SEC, STAT_REPORT_PERIOD_SEC, COMPACTION_HEARTBEAT_PERIOD_SEC
+            );
+
+            loop {
+                let item =
+                    futures::future::select(event_stream.next(), shutdown_rx_shared.clone()).await;
+
+                match item {
+                    Either::Left((event, _)) => {
+                        if let Some(event) = event {
+                            match event {
+                                HummockTimerEvent::CheckDeadTask => {
+                                    if hummock_manager.env.opts.compaction_deterministic_test {
+                                        continue;
+                                    }
+
+                                    hummock_manager.check_dead_task().await;
+                                }
+
+                                HummockTimerEvent::GroupSplit => {
+                                    if hummock_manager.env.opts.compaction_deterministic_test {
+                                        continue;
+                                    }
+
+                                    hummock_manager.on_handle_check_split_multi_group().await;
+                                }
+
+                                HummockTimerEvent::Report => {
+                                    let (current_version, id_to_config, version_stats) = {
+                                        let versioning_guard =
+                                            hummock_manager.versioning.read().await;
+
+                                        let configs =
+                                            hummock_manager.get_compaction_group_map().await;
+                                        let versioning_deref = versioning_guard;
+                                        (
+                                            versioning_deref.current_version.clone(),
+                                            configs,
+                                            versioning_deref.version_stats.clone(),
+                                        )
+                                    };
+
+                                    if let Some(mv_id_to_all_table_ids) = hummock_manager
+                                        .metadata_manager
+                                        .get_job_id_to_internal_table_ids_mapping()
+                                        .await
+                                    {
+                                        trigger_mv_stat(
+                                            &hummock_manager.metrics,
+                                            &version_stats,
+                                            mv_id_to_all_table_ids,
+                                        );
+                                    }
+
+                                    for compaction_group_id in
+                                        get_compaction_group_ids(&current_version)
+                                    {
+                                        let compaction_group_config =
+                                            &id_to_config[&compaction_group_id];
+
+                                        let group_levels = current_version
+                                            .get_compaction_group_levels(
+                                                compaction_group_config.group_id(),
+                                            );
+
+                                        trigger_lsm_stat(
+                                            &hummock_manager.metrics,
+                                            compaction_group_config.compaction_config(),
+                                            group_levels,
+                                            compaction_group_config.group_id(),
+                                        )
+                                    }
+                                }
+
+                                HummockTimerEvent::CompactionHeartBeatExpiredCheck => {
+                                    let compactor_manager =
+                                        hummock_manager.compactor_manager.clone();
+
+                                    // TODO: add metrics to track expired tasks
+                                    // The cancel task has two paths
+                                    // 1. compactor heartbeat cancels the expired task based on task
+                                    // progress (meta + compactor)
+                                    // 2. meta periodically scans the task and performs a cancel on
+                                    // the meta side for tasks that are not updated by heartbeat
+                                    let expired_tasks: Vec<u64> = compactor_manager
+                                        .get_heartbeat_expired_tasks()
+                                        .into_iter()
+                                        .map(|task| task.task_id)
+                                        .collect();
+                                    if !expired_tasks.is_empty() {
+                                        tracing::info!(
+                                            expired_tasks = ?expired_tasks,
+                                            "Heartbeat expired compaction tasks detected. Attempting to cancel tasks.",
+                                        );
+                                        if let Err(e) = hummock_manager
+                                            .cancel_compact_tasks(
+                                                expired_tasks.clone(),
+                                                TaskStatus::HeartbeatCanceled,
+                                            )
+                                            .await
+                                        {
+                                            tracing::error!(
+                                                expired_tasks = ?expired_tasks,
+                                                error = %e.as_report(),
+                                                "Attempt to remove compaction task due to elapsed heartbeat failed. We will continue to track its heartbeat
+                                                until we can successfully report its status",
+                                            );
+                                        }
+                                    }
+                                }
+
+                                HummockTimerEvent::DynamicCompactionTrigger => {
+                                    // Disable periodic trigger for compaction_deterministic_test.
+                                    if hummock_manager.env.opts.compaction_deterministic_test {
+                                        continue;
+                                    }
+
+                                    hummock_manager
+                                        .on_handle_trigger_multi_group(
+                                            compact_task::TaskType::Dynamic,
+                                        )
+                                        .await;
+                                }
+
+                                HummockTimerEvent::SpaceReclaimCompactionTrigger => {
+                                    // Disable periodic trigger for compaction_deterministic_test.
+                                    if hummock_manager.env.opts.compaction_deterministic_test {
+                                        continue;
+                                    }
+
+                                    hummock_manager
+                                        .on_handle_trigger_multi_group(
+                                            compact_task::TaskType::SpaceReclaim,
+                                        )
+                                        .await;
+                                }
+
+                                HummockTimerEvent::TtlCompactionTrigger => {
+                                    // Disable periodic trigger for compaction_deterministic_test.
+                                    if hummock_manager.env.opts.compaction_deterministic_test {
+                                        continue;
+                                    }
+
+                                    hummock_manager
+                                        .on_handle_trigger_multi_group(compact_task::TaskType::Ttl)
+                                        .await;
+                                }
+
+                                HummockTimerEvent::TombstoneCompactionTrigger => {
+                                    // Disable periodic trigger for compaction_deterministic_test.
+                                    if hummock_manager.env.opts.compaction_deterministic_test {
+                                        continue;
+                                    }
+
+                                    hummock_manager
+                                        .on_handle_trigger_multi_group(
+                                            compact_task::TaskType::Tombstone,
+                                        )
+                                        .await;
+                                }
+
+                                HummockTimerEvent::FullGc => {
+                                    if hummock_manager
+                                        .start_full_gc(Duration::from_secs(3600))
+                                        .is_ok()
+                                    {
+                                        tracing::info!("Start full GC from meta node.");
+                                    }
+                                }
+                            }
+                        }
+                    }
+
+                    Either::Right((_, _shutdown)) => {
+                        tracing::info!("Hummock timer loop is stopped");
+                        break;
+                    }
+                }
+            }
+        });
+        (join_handle, shutdown_tx)
+    }
+}
+
+impl HummockManager {
+    async fn check_dead_task(&self) {
+        const MAX_COMPACTION_L0_MULTIPLIER: u64 = 32;
+        const MAX_COMPACTION_DURATION_SEC: u64 = 20 * 60;
+        let (groups, configs) = {
+            let versioning_guard = self.versioning.read().await;
+            let g = versioning_guard
+                .current_version
+                .levels
+                .iter()
+                .map(|(id, group)| {
+                    (
+                        *id,
+                        group
+                            .l0
+                            .as_ref()
+                            .unwrap()
+                            .sub_levels
+                            .iter()
+                            .map(|level| level.total_file_size)
+                            .sum::<u64>(),
+                    )
+                })
+                .collect_vec();
+            let c = self.get_compaction_group_map().await;
+            (g, c)
+        };
+        let mut slowdown_groups: HashMap<u64, u64> = HashMap::default();
+        {
+            for (group_id, l0_file_size) in groups {
+                let group = &configs[&group_id];
+                if l0_file_size
+                    > MAX_COMPACTION_L0_MULTIPLIER
+                        * group.compaction_config.max_bytes_for_level_base
+                {
+                    slowdown_groups.insert(group_id, l0_file_size);
+                }
+            }
+        }
+        if slowdown_groups.is_empty() {
+            return;
+        }
+        let mut pending_tasks: HashMap<u64, (u64, usize, RunningCompactTask)> = HashMap::default();
+        {
+            let compaction_guard = self.compaction.read().await;
+            for group_id in slowdown_groups.keys() {
+                if let Some(status) = compaction_guard.compaction_statuses.get(group_id) {
+                    for (idx, level_handler) in status.level_handlers.iter().enumerate() {
+                        let tasks = level_handler.get_pending_tasks().to_vec();
+                        if tasks.is_empty() {
+                            continue;
+                        }
+                        for task in tasks {
+                            pending_tasks.insert(task.task_id, (*group_id, idx, task));
+                        }
+                    }
+                }
+            }
+        }
+        let task_ids = pending_tasks.keys().cloned().collect_vec();
+        let task_infos = self
+            .compactor_manager
+            .check_tasks_status(&task_ids, Duration::from_secs(MAX_COMPACTION_DURATION_SEC));
+        for (task_id, (compact_time, status)) in task_infos {
+            if status == TASK_NORMAL {
+                continue;
+            }
+            if let Some((group_id, level_id, task)) = pending_tasks.get(&task_id) {
+                let group_size = *slowdown_groups.get(group_id).unwrap();
+                warn!("COMPACTION SLOW: the task-{} of group-{}(size: {}MB) level-{} has not finished after {:?}, {}, it may cause pending sstable files({:?}) blocking other task.",
+                    task_id, *group_id,group_size / 1024 / 1024,*level_id, compact_time, status, task.ssts);
+            }
+        }
+    }
+
+    /// * For compaction group with only one single state-table, do not change it again.
+    /// * For state-table which only write less than `HISTORY_TABLE_INFO_WINDOW_SIZE` times, do not
+    ///   change it. Because we need more statistic data to decide split strategy.
+    /// * For state-table with low throughput which write no more than
+    ///   `min_table_split_write_throughput` data, never split it.
+    /// * For state-table whose size less than `min_table_split_size`, do not split it unless its
+    ///   throughput keep larger than `table_write_throughput_threshold` for a long time.
+    /// * For state-table whose throughput less than `min_table_split_write_throughput`, do not
+    ///   increase it size of base-level.
+    async fn on_handle_check_split_multi_group(&self) {
+        let params = self.env.system_params_reader().await;
+        let barrier_interval_ms = params.barrier_interval_ms() as u64;
+        let checkpoint_secs = std::cmp::max(
+            1,
+            params.checkpoint_frequency() * barrier_interval_ms / 1000,
+        );
+        let created_tables = match self.metadata_manager.get_created_table_ids().await {
+            Ok(created_tables) => created_tables,
+            Err(err) => {
+                tracing::warn!(error = %err.as_report(), "failed to fetch created table ids");
+                return;
+            }
+        };
+        let created_tables: HashSet<u32> = HashSet::from_iter(created_tables);
+        let table_write_throughput = self.history_table_throughput.read().clone();
+        let mut group_infos = self.calculate_compaction_group_statistic().await;
+        group_infos.sort_by_key(|group| group.group_size);
+        group_infos.reverse();
+
+        for group in &group_infos {
+            if group.table_statistic.len() == 1 {
+                // no need to handle the separate compaciton group
+                continue;
+            }
+
+            for (table_id, table_size) in &group.table_statistic {
+                self.calculate_table_align_rule(
+                    &table_write_throughput,
+                    table_id,
+                    table_size,
+                    !created_tables.contains(table_id),
+                    checkpoint_secs,
+                    group.group_id,
+                    group.group_size,
+                )
+                .await;
+            }
+        }
+    }
+
+    async fn on_handle_trigger_multi_group(&self, task_type: compact_task::TaskType) {
+        for cg_id in self.compaction_group_ids().await {
+            if let Err(e) = self.compaction_state.try_sched_compaction(cg_id, task_type) {
+                tracing::warn!(
+                    error = %e.as_report(),
+                    "Failed to schedule {:?} compaction for compaction group {}",
+                    task_type,
+                    cg_id,
+                );
+            }
+        }
+    }
+
+    async fn calculate_table_align_rule(
+        &self,
+        table_write_throughput: &HashMap<u32, VecDeque<u64>>,
+        table_id: &u32,
+        table_size: &u64,
+        is_creating_table: bool,
+        checkpoint_secs: u64,
+        parent_group_id: u64,
+        group_size: u64,
+    ) {
+        let default_group_id: CompactionGroupId = StaticCompactionGroupId::StateDefault.into();
+        let mv_group_id: CompactionGroupId = StaticCompactionGroupId::MaterializedView.into();
+        let partition_vnode_count = self.env.opts.partition_vnode_count;
+        let window_size = HISTORY_TABLE_INFO_STATISTIC_TIME / (checkpoint_secs as usize);
+
+        let mut is_high_write_throughput = false;
+        let mut is_low_write_throughput = true;
+        if let Some(history) = table_write_throughput.get(table_id) {
+            if !is_creating_table {
+                if history.len() >= window_size {
+                    is_high_write_throughput = history.iter().all(|throughput| {
+                        *throughput / checkpoint_secs
+                            > self.env.opts.table_write_throughput_threshold
+                    });
+                    is_low_write_throughput = history.iter().any(|throughput| {
+                        *throughput / checkpoint_secs
+                            < self.env.opts.min_table_split_write_throughput
+                    });
+                }
+            } else {
+                // For creating table, relax the checking restrictions to make the data alignment behavior more sensitive.
+                let sum = history.iter().sum::<u64>();
+                is_low_write_throughput = sum
+                    < self.env.opts.min_table_split_write_throughput
+                        * history.len() as u64
+                        * checkpoint_secs;
+            }
+        }
+
+        let state_table_size = *table_size;
+
+        // 1. Avoid splitting a creating table
+        // 2. Avoid splitting a is_low_write_throughput creating table
+        // 3. Avoid splitting a non-high throughput medium-sized table
+        if is_creating_table
+            || (is_low_write_throughput)
+            || (state_table_size < self.env.opts.min_table_split_size && !is_high_write_throughput)
+        {
+            return;
+        }
+
+        // do not split a large table and a small table because it would increase IOPS
+        // of small table.
+        if parent_group_id != default_group_id && parent_group_id != mv_group_id {
+            let rest_group_size = group_size - state_table_size;
+            if rest_group_size < state_table_size
+                && rest_group_size < self.env.opts.min_table_split_size
+            {
+                return;
+            }
+        }
+
+        let ret = self
+            .move_state_table_to_compaction_group(
+                parent_group_id,
+                &[*table_id],
+                partition_vnode_count,
+            )
+            .await;
+        match ret {
+            Ok(new_group_id) => {
+                tracing::info!(
+                    "move state table [{}] from group-{} to group-{} success",
+                    table_id,
+                    parent_group_id,
+                    new_group_id
+                );
+            }
+            Err(e) => {
+                tracing::info!(
+                    error = %e.as_report(),
+                    "failed to move state table [{}] from group-{}",
+                    table_id,
+                    parent_group_id,
+                )
+            }
+        }
+    }
+}
diff --git a/src/meta/src/hummock/manager/transaction.rs b/src/meta/src/hummock/manager/transaction.rs
new file mode 100644
index 0000000000000..c467e95adfdbe
--- /dev/null
+++ b/src/meta/src/hummock/manager/transaction.rs
@@ -0,0 +1,245 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::BTreeMap;
+use std::ops::{Deref, DerefMut};
+
+use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
+use risingwave_hummock_sdk::HummockVersionId;
+use risingwave_pb::hummock::HummockVersionStats;
+use risingwave_pb::meta::subscribe_response::{Info, Operation};
+
+use crate::manager::NotificationManager;
+use crate::model::{
+    InMemValTransaction, MetadataModelResult, Transactional, ValTransaction, VarTransaction,
+};
+use crate::rpc::metrics::MetaMetrics;
+
+fn trigger_delta_log_stats(metrics: &MetaMetrics, total_number: usize) {
+    metrics.delta_log_count.set(total_number as _);
+}
+
+fn trigger_version_stat(metrics: &MetaMetrics, current_version: &HummockVersion) {
+    metrics
+        .max_committed_epoch
+        .set(current_version.max_committed_epoch as i64);
+    metrics
+        .version_size
+        .set(current_version.estimated_encode_len() as i64);
+    metrics
+        .safe_epoch
+        .set(current_version.visible_table_safe_epoch() as i64);
+    metrics.current_version_id.set(current_version.id as i64);
+}
+
+pub(super) struct HummockVersionTransaction<'a> {
+    orig_version: &'a mut HummockVersion,
+    orig_deltas: &'a mut BTreeMap<HummockVersionId, HummockVersionDelta>,
+    notification_manager: &'a NotificationManager,
+    meta_metrics: &'a MetaMetrics,
+
+    pre_applied_version: Option<(HummockVersion, Vec<HummockVersionDelta>)>,
+    disable_apply_to_txn: bool,
+}
+
+impl<'a> HummockVersionTransaction<'a> {
+    pub(super) fn new(
+        version: &'a mut HummockVersion,
+        deltas: &'a mut BTreeMap<HummockVersionId, HummockVersionDelta>,
+        notification_manager: &'a NotificationManager,
+        meta_metrics: &'a MetaMetrics,
+    ) -> Self {
+        Self {
+            orig_version: version,
+            orig_deltas: deltas,
+            pre_applied_version: None,
+            disable_apply_to_txn: false,
+            notification_manager,
+            meta_metrics,
+        }
+    }
+
+    pub(super) fn disable_apply_to_txn(&mut self) {
+        assert!(
+            self.pre_applied_version.is_none(),
+            "should only call disable at the beginning of txn"
+        );
+        self.disable_apply_to_txn = true;
+    }
+
+    pub(super) fn latest_version(&self) -> &HummockVersion {
+        if let Some((version, _)) = &self.pre_applied_version {
+            version
+        } else {
+            self.orig_version
+        }
+    }
+
+    pub(super) fn new_delta<'b>(&'b mut self) -> SingleDeltaTransaction<'a, 'b> {
+        let delta = self.latest_version().version_delta_after();
+        SingleDeltaTransaction {
+            version_txn: self,
+            delta: Some(delta),
+        }
+    }
+
+    fn pre_apply(&mut self, delta: HummockVersionDelta) {
+        let (version, deltas) = self
+            .pre_applied_version
+            .get_or_insert_with(|| (self.orig_version.clone(), Vec::with_capacity(1)));
+        version.apply_version_delta(&delta);
+        deltas.push(delta);
+    }
+}
+
+impl<'a> InMemValTransaction for HummockVersionTransaction<'a> {
+    fn commit(self) {
+        if let Some((version, deltas)) = self.pre_applied_version {
+            *self.orig_version = version;
+            if !self.disable_apply_to_txn {
+                let pb_deltas = deltas.iter().map(|delta| delta.to_protobuf()).collect();
+                self.notification_manager.notify_hummock_without_version(
+                    Operation::Add,
+                    Info::HummockVersionDeltas(risingwave_pb::hummock::HummockVersionDeltas {
+                        version_deltas: pb_deltas,
+                    }),
+                );
+            }
+            for delta in deltas {
+                assert!(self.orig_deltas.insert(delta.id, delta.clone()).is_none());
+            }
+
+            trigger_delta_log_stats(self.meta_metrics, self.orig_deltas.len());
+            trigger_version_stat(self.meta_metrics, self.orig_version);
+        }
+    }
+}
+
+impl<'a, TXN> ValTransaction<TXN> for HummockVersionTransaction<'a>
+where
+    HummockVersionDelta: Transactional<TXN>,
+    HummockVersionStats: Transactional<TXN>,
+{
+    async fn apply_to_txn(&self, txn: &mut TXN) -> MetadataModelResult<()> {
+        if self.disable_apply_to_txn {
+            return Ok(());
+        }
+        for delta in self
+            .pre_applied_version
+            .iter()
+            .flat_map(|(_, deltas)| deltas.iter())
+        {
+            delta.upsert_in_transaction(txn).await?;
+        }
+        Ok(())
+    }
+}
+
+pub(super) struct SingleDeltaTransaction<'a, 'b> {
+    version_txn: &'b mut HummockVersionTransaction<'a>,
+    delta: Option<HummockVersionDelta>,
+}
+
+impl<'a, 'b> SingleDeltaTransaction<'a, 'b> {
+    pub(super) fn latest_version(&self) -> &HummockVersion {
+        self.version_txn.latest_version()
+    }
+
+    pub(super) fn pre_apply(mut self) {
+        self.version_txn.pre_apply(self.delta.take().unwrap());
+    }
+
+    pub(super) fn with_latest_version(
+        &mut self,
+        f: impl FnOnce(&HummockVersion, &mut HummockVersionDelta),
+    ) {
+        f(
+            self.version_txn.latest_version(),
+            self.delta.as_mut().expect("should exist"),
+        )
+    }
+}
+
+impl<'a, 'b> Deref for SingleDeltaTransaction<'a, 'b> {
+    type Target = HummockVersionDelta;
+
+    fn deref(&self) -> &Self::Target {
+        self.delta.as_ref().expect("should exist")
+    }
+}
+
+impl<'a, 'b> DerefMut for SingleDeltaTransaction<'a, 'b> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.delta.as_mut().expect("should exist")
+    }
+}
+
+impl<'a, 'b> Drop for SingleDeltaTransaction<'a, 'b> {
+    fn drop(&mut self) {
+        if let Some(delta) = self.delta.take() {
+            self.version_txn.pre_apply(delta);
+        }
+    }
+}
+
+pub(super) struct HummockVersionStatsTransaction<'a> {
+    stats: VarTransaction<'a, HummockVersionStats>,
+    notification_manager: &'a NotificationManager,
+}
+
+impl<'a> HummockVersionStatsTransaction<'a> {
+    pub(super) fn new(
+        stats: &'a mut HummockVersionStats,
+        notification_manager: &'a NotificationManager,
+    ) -> Self {
+        Self {
+            stats: VarTransaction::new(stats),
+            notification_manager,
+        }
+    }
+}
+
+impl<'a> InMemValTransaction for HummockVersionStatsTransaction<'a> {
+    fn commit(self) {
+        if self.stats.has_new_value() {
+            let stats = self.stats.clone();
+            self.stats.commit();
+            self.notification_manager
+                .notify_frontend_without_version(Operation::Update, Info::HummockStats(stats));
+        }
+    }
+}
+
+impl<'a, TXN> ValTransaction<TXN> for HummockVersionStatsTransaction<'a>
+where
+    HummockVersionStats: Transactional<TXN>,
+{
+    async fn apply_to_txn(&self, txn: &mut TXN) -> MetadataModelResult<()> {
+        self.stats.apply_to_txn(txn).await
+    }
+}
+
+impl<'a> Deref for HummockVersionStatsTransaction<'a> {
+    type Target = HummockVersionStats;
+
+    fn deref(&self) -> &Self::Target {
+        self.stats.deref()
+    }
+}
+
+impl<'a> DerefMut for HummockVersionStatsTransaction<'a> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.stats.deref_mut()
+    }
+}
diff --git a/src/meta/src/hummock/manager/utils.rs b/src/meta/src/hummock/manager/utils.rs
index 45d75d4774e90..3d8cb04546284 100644
--- a/src/meta/src/hummock/manager/utils.rs
+++ b/src/meta/src/hummock/manager/utils.rs
@@ -19,16 +19,18 @@ macro_rules! commit_multi_var {
     ($meta_store:expr, $($val_txn:expr),*) => {
         {
             async {
+                use crate::model::{InMemValTransaction, ValTransaction};
                 match &$meta_store {
-                    crate::manager::MetaStoreImpl::Kv(meta_store) => {
+                    $crate::manager::MetaStoreImpl::Kv(meta_store) => {
                         use crate::storage::Transaction;
+                        use crate::storage::meta_store::MetaStore;
                         let mut trx = Transaction::default();
                         $(
-                            $val_txn.as_v1_ref().apply_to_txn(&mut trx).await?;
+                            $val_txn.apply_to_txn(&mut trx).await?;
                         )*
                         meta_store.txn(trx).await?;
                         $(
-                            $val_txn.into_v1().commit();
+                            $val_txn.commit();
                         )*
                         Result::Ok(())
                     }
@@ -37,11 +39,11 @@ macro_rules! commit_multi_var {
                         use crate::model::MetadataModelError;
                         let mut trx = sql_meta_store.conn.begin().await.map_err(MetadataModelError::from)?;
                         $(
-                            $val_txn.as_v2_ref().apply_to_txn(&mut trx).await?;
+                            $val_txn.apply_to_txn(&mut trx).await?;
                         )*
                         trx.commit().await.map_err(MetadataModelError::from)?;
                         $(
-                            $val_txn.into_v2().commit();
+                            $val_txn.commit();
                         )*
                         Result::Ok(())
                     }
@@ -51,14 +53,70 @@ macro_rules! commit_multi_var {
     };
 }
 pub(crate) use commit_multi_var;
+use risingwave_hummock_sdk::SstObjectIdRange;
 
-macro_rules! create_trx_wrapper {
-    ($meta_store:expr, $wrapper:ident, $inner:expr) => {{
-        match &$meta_store {
-            crate::manager::MetaStoreImpl::Kv(_) => $wrapper::V1($inner),
-            crate::manager::MetaStoreImpl::Sql(_) => $wrapper::V2($inner),
-        }
-    }};
-}
+use crate::hummock::error::Result;
+use crate::hummock::sequence::next_sstable_object_id;
+use crate::hummock::HummockManager;
 
-pub(crate) use create_trx_wrapper;
+impl HummockManager {
+    #[cfg(test)]
+    pub(super) async fn check_state_consistency(&self) {
+        use crate::hummock::manager::compaction::Compaction;
+        use crate::hummock::manager::context::ContextInfo;
+        use crate::hummock::manager::versioning::Versioning;
+        let mut compaction_guard = self.compaction.write().await;
+        let mut versioning_guard = self.versioning.write().await;
+        let mut context_info_guard = self.context_info.write().await;
+        let objects_to_delete = self.delete_object_tracker.current();
+        // We don't check `checkpoint` because it's allowed to update its in memory state without
+        // persisting to object store.
+        let get_state = |compaction_guard: &mut Compaction,
+                         versioning_guard: &mut Versioning,
+                         context_info_guard: &mut ContextInfo| {
+            let compact_statuses_copy = compaction_guard.compaction_statuses.clone();
+            let compact_task_assignment_copy = compaction_guard.compact_task_assignment.clone();
+            let pinned_versions_copy = context_info_guard.pinned_versions.clone();
+            let pinned_snapshots_copy = context_info_guard.pinned_snapshots.clone();
+            let hummock_version_deltas_copy = versioning_guard.hummock_version_deltas.clone();
+            let version_stats_copy = versioning_guard.version_stats.clone();
+            ((
+                compact_statuses_copy,
+                compact_task_assignment_copy,
+                pinned_versions_copy,
+                pinned_snapshots_copy,
+                hummock_version_deltas_copy,
+                version_stats_copy,
+            ),)
+        };
+        let mem_state = get_state(
+            &mut compaction_guard,
+            &mut versioning_guard,
+            &mut context_info_guard,
+        );
+        self.load_meta_store_state_impl(
+            &mut compaction_guard,
+            &mut versioning_guard,
+            &mut context_info_guard,
+        )
+        .await
+        .expect("Failed to load state from meta store");
+        let loaded_state = get_state(
+            &mut compaction_guard,
+            &mut versioning_guard,
+            &mut context_info_guard,
+        );
+        assert_eq!(
+            mem_state, loaded_state,
+            "hummock in-mem state is inconsistent with meta store state",
+        );
+        self.delete_object_tracker.clear();
+        self.delete_object_tracker
+            .add(objects_to_delete.into_iter());
+    }
+
+    pub async fn get_new_sst_ids(&self, number: u32) -> Result<SstObjectIdRange> {
+        let start_id = next_sstable_object_id(&self.env, number).await?;
+        Ok(SstObjectIdRange::new(start_id, start_id + number as u64))
+    }
+}
diff --git a/src/meta/src/hummock/manager/versioning.rs b/src/meta/src/hummock/manager/versioning.rs
index feeacc98b558a..2e6b2512a8be0 100644
--- a/src/meta/src/hummock/manager/versioning.rs
+++ b/src/meta/src/hummock/manager/versioning.rs
@@ -15,92 +15,56 @@
 use std::cmp;
 use std::collections::{BTreeMap, HashMap, HashSet};
 
-use function_name::named;
 use itertools::Itertools;
-use risingwave_common::util::epoch::INVALID_EPOCH;
+use risingwave_common::catalog::TableId;
 use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
-    build_initial_compaction_group_levels, get_compaction_group_ids, BranchedSstInfo,
+    get_compaction_group_ids, get_table_compaction_group_id_mapping, BranchedSstInfo,
 };
-use risingwave_hummock_sdk::compaction_group::{StateTableId, StaticCompactionGroupId};
+use risingwave_hummock_sdk::compaction_group::StateTableId;
 use risingwave_hummock_sdk::table_stats::add_prost_table_stats_map;
 use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
 use risingwave_hummock_sdk::{
-    CompactionGroupId, HummockContextId, HummockSstableObjectId, HummockVersionId, FIRST_VERSION_ID,
+    CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId,
 };
 use risingwave_pb::common::WorkerNode;
 use risingwave_pb::hummock::write_limits::WriteLimit;
 use risingwave_pb::hummock::{
-    CompactionConfig, HummockPinnedSnapshot, HummockPinnedVersion, HummockVersionStats,
-    SstableInfo, TableStats,
+    HummockPinnedSnapshot, HummockPinnedVersion, HummockSnapshot, HummockVersionStats, SstableInfo,
+    TableStats,
 };
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
 
 use super::check_cg_write_limit;
 use crate::hummock::error::Result;
 use crate::hummock::manager::checkpoint::HummockVersionCheckpoint;
-use crate::hummock::manager::worker::{HummockManagerEvent, HummockManagerEventSender};
-use crate::hummock::manager::{commit_multi_var, create_trx_wrapper, read_lock, write_lock};
-use crate::hummock::metrics_utils::{
-    trigger_safepoint_stat, trigger_write_stop_stats, LocalTableMetrics,
-};
+use crate::hummock::manager::commit_multi_var;
+use crate::hummock::manager::context::ContextInfo;
+use crate::hummock::manager::gc::DeleteObjectTracker;
+use crate::hummock::manager::transaction::HummockVersionTransaction;
+use crate::hummock::metrics_utils::{trigger_write_stop_stats, LocalTableMetrics};
 use crate::hummock::model::CompactionGroup;
 use crate::hummock::HummockManager;
-use crate::model::{VarTransaction, VarTransactionWrapper};
-use crate::storage::MetaStore;
+use crate::model::VarTransaction;
 use crate::MetaResult;
 
-/// `HummockVersionSafePoint` prevents hummock versions GE than it from being GC.
-/// It's used by meta node itself to temporarily pin versions.
-pub struct HummockVersionSafePoint {
-    pub id: HummockVersionId,
-    event_sender: HummockManagerEventSender,
-}
-
-impl Drop for HummockVersionSafePoint {
-    fn drop(&mut self) {
-        if self
-            .event_sender
-            .send(HummockManagerEvent::DropSafePoint(self.id))
-            .is_err()
-        {
-            tracing::debug!("failed to drop hummock version safe point {}", self.id);
-        }
-    }
-}
-
 #[derive(Default)]
 pub struct Versioning {
     // Volatile states below
-    /// Avoide commit epoch epochs
+    /// Avoid commit epoch epochs
     /// Don't persist compaction version delta to meta store
     pub disable_commit_epochs: bool,
     /// Latest hummock version
     pub current_version: HummockVersion,
-    /// Objects that waits to be deleted from object store. It comes from either compaction, or
-    /// full GC (listing object store).
-    pub objects_to_delete: HashSet<HummockSstableObjectId>,
-    /// SST whose `object_id` != `sst_id`
-    pub branched_ssts: BTreeMap<
-        // SST object id
-        HummockSstableObjectId,
-        BranchedSstInfo,
-    >,
-    /// `version_safe_points` is similar to `pinned_versions` expect for being a transient state.
-    pub version_safe_points: Vec<HummockVersionId>,
-    /// Tables that write limit is trigger for.
-    pub write_limit: HashMap<CompactionGroupId, WriteLimit>,
+    pub local_metrics: HashMap<u32, LocalTableMetrics>,
 
     // Persistent states below
     pub hummock_version_deltas: BTreeMap<HummockVersionId, HummockVersionDelta>,
-    pub pinned_versions: BTreeMap<HummockContextId, HummockPinnedVersion>,
-    pub pinned_snapshots: BTreeMap<HummockContextId, HummockPinnedSnapshot>,
     /// Stats for latest hummock version.
     pub version_stats: HummockVersionStats,
     pub checkpoint: HummockVersionCheckpoint,
-    pub local_metrics: HashMap<u32, LocalTableMetrics>,
 }
 
-impl Versioning {
+impl ContextInfo {
     pub fn min_pinned_version_id(&self) -> HummockVersionId {
         let mut min_pinned_version_id = HummockVersionId::MAX;
         for id in self
@@ -113,64 +77,30 @@ impl Versioning {
         }
         min_pinned_version_id
     }
+}
 
+impl Versioning {
     /// Marks all objects <= `min_pinned_version_id` for deletion.
-    pub(super) fn mark_objects_for_deletion(&mut self) {
-        let min_pinned_version_id = self.min_pinned_version_id();
-        self.objects_to_delete.extend(
+    pub(super) fn mark_objects_for_deletion(
+        &self,
+        context_info: &ContextInfo,
+        delete_object_tracker: &DeleteObjectTracker,
+    ) {
+        let min_pinned_version_id = context_info.min_pinned_version_id();
+        delete_object_tracker.add(
             self.checkpoint
                 .stale_objects
                 .iter()
                 .filter(|(version_id, _)| **version_id <= min_pinned_version_id)
-                .flat_map(|(_, stale_objects)| stale_objects.id.clone()),
+                .flat_map(|(_, stale_objects)| stale_objects.id.iter().cloned()),
         );
     }
-
-    /// If there is some sst in the target group which is just split but we have not compact it, we
-    ///  can not split or move state-table to those group, because it may cause data overlap.
-    pub fn check_branched_sst_in_target_group(
-        &self,
-        table_ids: &[StateTableId],
-        source_group_id: &CompactionGroupId,
-        target_group_id: &CompactionGroupId,
-    ) -> bool {
-        for groups in self.branched_ssts.values() {
-            if groups.contains_key(target_group_id) && groups.contains_key(source_group_id) {
-                return false;
-            }
-        }
-        let mut found_sstable_repeated = false;
-        let moving_table_ids: HashSet<&u32> = HashSet::from_iter(table_ids);
-        if let Some(group) = self.current_version.levels.get(target_group_id) {
-            let target_member_table_ids: HashSet<u32> =
-                HashSet::from_iter(group.member_table_ids.clone());
-            self.current_version.level_iter(*source_group_id, |level| {
-                for sst in &level.table_infos {
-                    if sst
-                        .table_ids
-                        .iter()
-                        .all(|table_id| !moving_table_ids.contains(table_id))
-                    {
-                        continue;
-                    }
-                    for table_id in &sst.table_ids {
-                        if target_member_table_ids.contains(table_id) {
-                            found_sstable_repeated = true;
-                            return false;
-                        }
-                    }
-                }
-                true
-            });
-        }
-        !found_sstable_repeated
-    }
 }
 
 impl HummockManager {
-    #[named]
     pub async fn list_pinned_version(&self) -> Vec<HummockPinnedVersion> {
-        read_lock!(self, versioning)
+        self.context_info
+            .read()
             .await
             .pinned_versions
             .values()
@@ -178,9 +108,9 @@ impl HummockManager {
             .collect_vec()
     }
 
-    #[named]
     pub async fn list_pinned_snapshot(&self) -> Vec<HummockPinnedSnapshot> {
-        read_lock!(self, versioning)
+        self.context_info
+            .read()
             .await
             .pinned_snapshots
             .values()
@@ -205,71 +135,90 @@ impl HummockManager {
         Ok(workers)
     }
 
-    #[named]
-    pub async fn get_version_stats(&self) -> HummockVersionStats {
-        read_lock!(self, versioning).await.version_stats.clone()
+    /// Gets current version without pinning it.
+    /// Should not be called inside [`HummockManager`], because it requests locks internally.
+    ///
+    /// Note: this method can hurt performance because it will clone a large object.
+    pub async fn get_current_version(&self) -> HummockVersion {
+        self.versioning.read().await.current_version.clone()
     }
 
-    #[named]
-    pub async fn register_safe_point(&self) -> HummockVersionSafePoint {
-        let mut wl = write_lock!(self, versioning).await;
-        let safe_point = HummockVersionSafePoint {
-            id: wl.current_version.id,
-            event_sender: self.event_sender.clone(),
-        };
-        wl.version_safe_points.push(safe_point.id);
-        trigger_safepoint_stat(&self.metrics, &wl.version_safe_points);
-        safe_point
+    pub async fn get_current_max_committed_epoch(&self) -> HummockEpoch {
+        self.versioning
+            .read()
+            .await
+            .current_version
+            .max_committed_epoch
     }
 
-    #[named]
-    pub async fn unregister_safe_point(&self, safe_point: HummockVersionId) {
-        let mut wl = write_lock!(self, versioning).await;
-        let version_safe_points = &mut wl.version_safe_points;
-        if let Some(pos) = version_safe_points.iter().position(|sp| *sp == safe_point) {
-            version_safe_points.remove(pos);
-        }
-        trigger_safepoint_stat(&self.metrics, &wl.version_safe_points);
+    /// Gets the mapping from table id to compaction group id
+    pub async fn get_table_compaction_group_id_mapping(
+        &self,
+    ) -> HashMap<StateTableId, CompactionGroupId> {
+        get_table_compaction_group_id_mapping(&self.versioning.read().await.current_version)
+    }
+
+    /// Get version deltas from meta store
+    #[cfg_attr(coverage, coverage(off))]
+    pub async fn list_version_deltas(
+        &self,
+        start_id: u64,
+        num_limit: u32,
+        committed_epoch_limit: HummockEpoch,
+    ) -> Result<Vec<HummockVersionDelta>> {
+        let versioning = self.versioning.read().await;
+        let version_deltas = versioning
+            .hummock_version_deltas
+            .range(start_id..)
+            .map(|(_id, delta)| delta)
+            .filter(|delta| delta.max_committed_epoch <= committed_epoch_limit)
+            .take(num_limit as _)
+            .cloned()
+            .collect();
+        Ok(version_deltas)
+    }
+
+    pub async fn get_version_stats(&self) -> HummockVersionStats {
+        self.versioning.read().await.version_stats.clone()
     }
 
     /// Updates write limits for `target_groups` and sends notification.
     /// Returns true if `write_limit` has been modified.
     /// The implementation acquires `versioning` lock and `compaction_group_manager` lock.
-    #[named]
     pub(super) async fn try_update_write_limits(
         &self,
         target_group_ids: &[CompactionGroupId],
     ) -> bool {
-        let mut guard = write_lock!(self, versioning).await;
-        let config_mgr = self.compaction_group_manager.read().await;
+        let versioning = self.versioning.read().await;
+        let mut cg_manager = self.compaction_group_manager.write().await;
         let target_group_configs = target_group_ids
             .iter()
             .filter_map(|id| {
-                config_mgr
+                cg_manager
                     .try_get_compaction_group_config(*id)
                     .map(|config| (*id, config))
             })
             .collect();
         let mut new_write_limits = calc_new_write_limits(
             target_group_configs,
-            guard.write_limit.clone(),
-            &guard.current_version,
+            cg_manager.write_limit.clone(),
+            &versioning.current_version,
         );
         let all_group_ids: HashSet<_> =
-            HashSet::from_iter(get_compaction_group_ids(&guard.current_version));
+            HashSet::from_iter(get_compaction_group_ids(&versioning.current_version));
         new_write_limits.retain(|group_id, _| all_group_ids.contains(group_id));
-        if new_write_limits == guard.write_limit {
+        if new_write_limits == cg_manager.write_limit {
             return false;
         }
         tracing::debug!("Hummock stopped write is updated: {:#?}", new_write_limits);
         trigger_write_stop_stats(&self.metrics, &new_write_limits);
-        guard.write_limit = new_write_limits;
+        cg_manager.write_limit = new_write_limits;
         self.env
             .notification_manager()
             .notify_hummock_without_version(
                 Operation::Add,
                 Info::HummockWriteLimits(risingwave_pb::hummock::WriteLimits {
-                    write_limits: guard.write_limit.clone(),
+                    write_limits: cg_manager.write_limit.clone(),
                 }),
             );
         true
@@ -277,33 +226,88 @@ impl HummockManager {
 
     /// Gets write limits.
     /// The implementation acquires `versioning` lock.
-    #[named]
     pub async fn write_limits(&self) -> HashMap<CompactionGroupId, WriteLimit> {
-        let guard = read_lock!(self, versioning).await;
+        let guard = self.compaction_group_manager.read().await;
         guard.write_limit.clone()
     }
 
-    #[named]
     pub async fn list_branched_objects(&self) -> BTreeMap<HummockSstableObjectId, BranchedSstInfo> {
-        let guard = read_lock!(self, versioning).await;
-        guard.branched_ssts.clone()
+        let guard = self.versioning.read().await;
+        guard.current_version.build_branched_sst_info()
     }
 
-    #[named]
     pub async fn rebuild_table_stats(&self) -> Result<()> {
-        use crate::model::ValTransaction;
-        let mut versioning = write_lock!(self, versioning).await;
+        let mut versioning = self.versioning.write().await;
         let new_stats = rebuild_table_stats(&versioning.current_version);
-        let mut version_stats = create_trx_wrapper!(
-            self.meta_store_ref(),
-            VarTransactionWrapper,
-            VarTransaction::new(&mut versioning.version_stats)
-        );
+        let mut version_stats = VarTransaction::new(&mut versioning.version_stats);
         // version_stats.hummock_version_id is always 0 in meta store.
         version_stats.table_stats = new_stats.table_stats;
         commit_multi_var!(self.meta_store_ref(), version_stats)?;
         Ok(())
     }
+
+    pub async fn may_fill_backward_state_table_info(&self) -> Result<()> {
+        let mut versioning = self.versioning.write().await;
+        if versioning
+            .current_version
+            .need_fill_backward_compatible_state_table_info_delta()
+        {
+            let versioning: &mut Versioning = &mut versioning;
+            let mut version = HummockVersionTransaction::new(
+                &mut versioning.current_version,
+                &mut versioning.hummock_version_deltas,
+                self.env.notification_manager(),
+                &self.metrics,
+            );
+            let mut new_version_delta = version.new_delta();
+            new_version_delta.with_latest_version(|version, delta| {
+                version.may_fill_backward_compatible_state_table_info_delta(delta)
+            });
+            new_version_delta.pre_apply();
+            commit_multi_var!(self.meta_store_ref(), version)?;
+        }
+        Ok(())
+    }
+
+    pub fn latest_snapshot(&self) -> HummockSnapshot {
+        let snapshot = self.latest_snapshot.load();
+        HummockSnapshot::clone(&snapshot)
+    }
+
+    /// We don't commit an epoch without checkpoint. We will only update the `max_current_epoch`.
+    pub fn update_current_epoch(&self, max_current_epoch: HummockEpoch) -> HummockSnapshot {
+        // We only update `max_current_epoch`!
+        let prev_snapshot = self.latest_snapshot.rcu(|snapshot| HummockSnapshot {
+            committed_epoch: snapshot.committed_epoch,
+            current_epoch: max_current_epoch,
+        });
+        assert!(prev_snapshot.current_epoch < max_current_epoch);
+
+        tracing::trace!("new current epoch {}", max_current_epoch);
+        HummockSnapshot {
+            committed_epoch: prev_snapshot.committed_epoch,
+            current_epoch: max_current_epoch,
+        }
+    }
+
+    pub async fn list_change_log_epochs(
+        &self,
+        table_id: u32,
+        min_epoch: u64,
+        max_count: u32,
+    ) -> Vec<u64> {
+        let versioning = self.versioning.read().await;
+        if let Some(table_change_log) = versioning
+            .current_version
+            .table_change_log
+            .get(&TableId::new(table_id))
+        {
+            let table_change_log = table_change_log.clone();
+            table_change_log.get_epochs(min_epoch, max_count as usize)
+        } else {
+            vec![]
+        }
+    }
 }
 
 /// Calculates write limits for `target_groups`.
@@ -328,7 +332,12 @@ pub(super) fn calc_new_write_limits(
             new_write_limits.insert(
                 *id,
                 WriteLimit {
-                    table_ids: levels.member_table_ids.clone(),
+                    table_ids: version
+                        .state_table_info
+                        .compaction_group_member_table_ids(*id)
+                        .iter()
+                        .map(|table_id| table_id.table_id)
+                        .collect(),
                     reason: write_limit_type.as_str(),
                 },
             );
@@ -340,27 +349,6 @@ pub(super) fn calc_new_write_limits(
     new_write_limits
 }
 
-pub(super) fn create_init_version(default_compaction_config: CompactionConfig) -> HummockVersion {
-    let mut init_version = HummockVersion {
-        id: FIRST_VERSION_ID,
-        levels: Default::default(),
-        max_committed_epoch: INVALID_EPOCH,
-        safe_epoch: INVALID_EPOCH,
-        table_watermarks: HashMap::new(),
-        table_change_log: HashMap::new(),
-    };
-    for group_id in [
-        StaticCompactionGroupId::StateDefault as CompactionGroupId,
-        StaticCompactionGroupId::MaterializedView as CompactionGroupId,
-    ] {
-        init_version.levels.insert(
-            group_id,
-            build_initial_compaction_group_levels(group_id, &default_compaction_config),
-        );
-    }
-    init_version
-}
-
 /// Rebuilds table stats from the given version.
 /// Note that the result is approximate value. See `estimate_table_stats`.
 fn rebuild_table_stats(version: &HummockVersion) -> HummockVersionStats {
@@ -416,29 +404,30 @@ mod tests {
     };
 
     use crate::hummock::compaction::compaction_config::CompactionConfigBuilder;
+    use crate::hummock::manager::context::ContextInfo;
     use crate::hummock::manager::versioning::{
-        calc_new_write_limits, estimate_table_stats, rebuild_table_stats, Versioning,
+        calc_new_write_limits, estimate_table_stats, rebuild_table_stats,
     };
     use crate::hummock::model::CompactionGroup;
 
     #[test]
     fn test_min_pinned_version_id() {
-        let mut versioning = Versioning::default();
-        assert_eq!(versioning.min_pinned_version_id(), HummockVersionId::MAX);
-        versioning.pinned_versions.insert(
+        let mut context_info = ContextInfo::default();
+        assert_eq!(context_info.min_pinned_version_id(), HummockVersionId::MAX);
+        context_info.pinned_versions.insert(
             1,
             HummockPinnedVersion {
                 context_id: 1,
                 min_pinned_id: 10,
             },
         );
-        assert_eq!(versioning.min_pinned_version_id(), 10);
-        versioning.version_safe_points.push(5);
-        assert_eq!(versioning.min_pinned_version_id(), 5);
-        versioning.version_safe_points.clear();
-        assert_eq!(versioning.min_pinned_version_id(), 10);
-        versioning.pinned_versions.clear();
-        assert_eq!(versioning.min_pinned_version_id(), HummockVersionId::MAX);
+        assert_eq!(context_info.min_pinned_version_id(), 10);
+        context_info.version_safe_points.push(5);
+        assert_eq!(context_info.min_pinned_version_id(), 5);
+        context_info.version_safe_points.clear();
+        assert_eq!(context_info.min_pinned_version_id(), 10);
+        context_info.pinned_versions.clear();
+        assert_eq!(context_info.min_pinned_version_id(), HummockVersionId::MAX);
     }
 
     #[test]
@@ -564,14 +553,9 @@ mod tests {
             );
         }
 
-        let mut version = HummockVersion {
-            id: 123,
-            levels: Default::default(),
-            max_committed_epoch: 0,
-            safe_epoch: 0,
-            table_watermarks: HashMap::new(),
-            table_change_log: HashMap::new(),
-        };
+        let mut version = HummockVersion::default();
+        version.id = 123;
+
         for cg in 1..3 {
             version.levels.insert(
                 cg,
diff --git a/src/meta/src/hummock/metrics_utils.rs b/src/meta/src/hummock/metrics_utils.rs
index be25c5cf452b2..3779ff5b2be97 100644
--- a/src/meta/src/hummock/metrics_utils.rs
+++ b/src/meta/src/hummock/metrics_utils.rs
@@ -20,14 +20,10 @@ use std::time::{SystemTime, UNIX_EPOCH};
 use itertools::{enumerate, Itertools};
 use prometheus::core::{AtomicU64, GenericCounter};
 use prometheus::IntGauge;
-use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
-    object_size_map, BranchedSstInfo,
-};
+use risingwave_hummock_sdk::compaction_group::hummock_version_ext::object_size_map;
 use risingwave_hummock_sdk::table_stats::PbTableStatsMap;
 use risingwave_hummock_sdk::version::HummockVersion;
-use risingwave_hummock_sdk::{
-    CompactionGroupId, HummockContextId, HummockEpoch, HummockSstableObjectId, HummockVersionId,
-};
+use risingwave_hummock_sdk::{CompactionGroupId, HummockContextId, HummockEpoch, HummockVersionId};
 use risingwave_pb::hummock::hummock_version::Levels;
 use risingwave_pb::hummock::write_limits::WriteLimit;
 use risingwave_pb::hummock::{
@@ -113,17 +109,6 @@ pub fn trigger_local_table_stat(
     }
 }
 
-pub fn trigger_version_stat(metrics: &MetaMetrics, current_version: &HummockVersion) {
-    metrics
-        .max_committed_epoch
-        .set(current_version.max_committed_epoch as i64);
-    metrics
-        .version_size
-        .set(current_version.estimated_encode_len() as i64);
-    metrics.safe_epoch.set(current_version.safe_epoch as i64);
-    metrics.current_version_id.set(current_version.id as i64);
-}
-
 pub fn trigger_mv_stat(
     metrics: &MetaMetrics,
     version_stats: &HummockVersionStats,
@@ -431,16 +416,6 @@ pub fn trigger_pin_unpin_snapshot_state(
     }
 }
 
-pub fn trigger_safepoint_stat(metrics: &MetaMetrics, safepoints: &[HummockVersionId]) {
-    if let Some(sp) = safepoints.iter().min() {
-        metrics.min_safepoint_version_id.set(*sp as _);
-    } else {
-        metrics
-            .min_safepoint_version_id
-            .set(HummockVersionId::MAX as _);
-    }
-}
-
 pub fn trigger_gc_stat(
     metrics: &MetaMetrics,
     checkpoint: &HummockVersionCheckpoint,
@@ -478,10 +453,6 @@ pub fn trigger_gc_stat(
     metrics.stale_object_count.set(stale_object_count as _);
 }
 
-pub fn trigger_delta_log_stats(metrics: &MetaMetrics, total_number: usize) {
-    metrics.delta_log_count.set(total_number as _);
-}
-
 // Triggers a report on compact_pending_bytes_needed
 pub fn trigger_lsm_stat(
     metrics: &MetaMetrics,
@@ -555,36 +526,36 @@ pub fn trigger_write_stop_stats(
     }
 }
 
-pub fn trigger_split_stat(
-    metrics: &MetaMetrics,
-    compaction_group_id: CompactionGroupId,
-    member_table_id_len: usize,
-    branched_ssts: &BTreeMap<
-        // SST object id
-        HummockSstableObjectId,
-        BranchedSstInfo,
-    >,
-) {
-    let group_label = compaction_group_id.to_string();
-    metrics
-        .state_table_count
-        .with_label_values(&[&group_label])
-        .set(member_table_id_len as _);
+pub fn trigger_split_stat(metrics: &MetaMetrics, version: &HummockVersion) {
+    let branched_ssts = version.build_branched_sst_info();
 
-    let branched_sst_count: usize = branched_ssts
-        .values()
-        .map(|branched_map| {
-            branched_map
-                .keys()
-                .filter(|group_id| **group_id == compaction_group_id)
-                .count()
-        })
-        .sum();
+    for compaction_group_id in version.levels.keys() {
+        let group_label = compaction_group_id.to_string();
+        metrics
+            .state_table_count
+            .with_label_values(&[&group_label])
+            .set(
+                version
+                    .state_table_info
+                    .compaction_group_member_table_ids(*compaction_group_id)
+                    .len() as _,
+            );
 
-    metrics
-        .branched_sst_count
-        .with_label_values(&[&group_label])
-        .set(branched_sst_count as _);
+        let branched_sst_count: usize = branched_ssts
+            .values()
+            .map(|branched_map| {
+                branched_map
+                    .keys()
+                    .filter(|group_id| *group_id == compaction_group_id)
+                    .count()
+            })
+            .sum();
+
+        metrics
+            .branched_sst_count
+            .with_label_values(&[&group_label])
+            .set(branched_sst_count as _);
+    }
 }
 
 pub fn build_level_metrics_label(compaction_group_id: u64, level_idx: usize) -> String {
diff --git a/src/meta/src/hummock/mock_hummock_meta_client.rs b/src/meta/src/hummock/mock_hummock_meta_client.rs
index 9a888e27bc78e..90f70a4f74f3a 100644
--- a/src/meta/src/hummock/mock_hummock_meta_client.rs
+++ b/src/meta/src/hummock/mock_hummock_meta_client.rs
@@ -21,6 +21,7 @@ use async_trait::async_trait;
 use fail::fail_point;
 use futures::stream::BoxStream;
 use futures::{Stream, StreamExt};
+use risingwave_hummock_sdk::change_log::build_table_change_log_delta;
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::{
@@ -156,13 +157,26 @@ impl HummockMetaClient for MockHummockMetaClient {
     }
 
     async fn commit_epoch(&self, epoch: HummockEpoch, sync_result: SyncResult) -> Result<()> {
+        let version: HummockVersion = self.hummock_manager.get_current_version().await;
         let sst_to_worker = sync_result
             .uncommitted_ssts
             .iter()
             .map(|LocalSstableInfo { sst_info, .. }| (sst_info.get_object_id(), self.context_id))
             .collect();
         let new_table_watermark = sync_result.table_watermarks;
-
+        let table_change_log = build_table_change_log_delta(
+            sync_result
+                .old_value_ssts
+                .into_iter()
+                .map(|sst| sst.sst_info),
+            sync_result.uncommitted_ssts.iter().map(|sst| &sst.sst_info),
+            &vec![epoch],
+            version
+                .state_table_info
+                .info()
+                .keys()
+                .map(|table_id| (table_id.table_id, 0)),
+        );
         self.hummock_manager
             .commit_epoch(
                 epoch,
@@ -175,6 +189,7 @@ impl HummockMetaClient for MockHummockMetaClient {
                     new_table_watermark,
                     sst_to_worker,
                     None,
+                    table_change_log,
                 ),
             )
             .await
diff --git a/src/meta/src/hummock/test_utils.rs b/src/meta/src/hummock/test_utils.rs
index 23898f6965ca1..0090cceeed1cd 100644
--- a/src/meta/src/hummock/test_utils.rs
+++ b/src/meta/src/hummock/test_utils.rs
@@ -236,7 +236,7 @@ pub async fn register_table_ids_to_compaction_group(
     compaction_group_id: CompactionGroupId,
 ) {
     hummock_manager_ref
-        .register_table_ids(
+        .register_table_ids_for_test(
             &table_ids
                 .iter()
                 .map(|table_id| (*table_id, compaction_group_id))
@@ -251,8 +251,9 @@ pub async fn unregister_table_ids_from_compaction_group(
     table_ids: &[u32],
 ) {
     hummock_manager_ref
-        .unregister_table_ids_fail_fast(table_ids)
-        .await;
+        .unregister_table_ids(table_ids)
+        .await
+        .unwrap();
 }
 
 /// Generate keys like `001_key_test_00002` with timestamp `epoch`.
diff --git a/src/meta/src/hummock/vacuum.rs b/src/meta/src/hummock/vacuum.rs
index 9e45fa5a2bcc9..6cde13507836b 100644
--- a/src/meta/src/hummock/vacuum.rs
+++ b/src/meta/src/hummock/vacuum.rs
@@ -95,7 +95,7 @@ impl VacuumManager {
                 pending_object_ids
             } else {
                 // 2. If no pending SST objects, then fetch new ones.
-                let mut objects_to_delete = self.hummock_manager.get_objects_to_delete().await;
+                let mut objects_to_delete = self.hummock_manager.get_objects_to_delete();
                 self.filter_out_pinned_ssts(&mut objects_to_delete).await?;
                 if objects_to_delete.is_empty() {
                     return Ok(vec![]);
@@ -237,13 +237,13 @@ mod tests {
         assert_eq!(vacuum.vacuum_metadata().await.unwrap(), 6);
         assert_eq!(vacuum.vacuum_metadata().await.unwrap(), 0);
 
-        assert!(hummock_manager.get_objects_to_delete().await.is_empty());
+        assert!(hummock_manager.get_objects_to_delete().is_empty());
         hummock_manager
             .unpin_version_before(context_id, HummockVersionId::MAX)
             .await
             .unwrap();
         hummock_manager.create_version_checkpoint(0).await.unwrap();
-        assert!(!hummock_manager.get_objects_to_delete().await.is_empty());
+        assert!(!hummock_manager.get_objects_to_delete().is_empty());
         // No SST deletion is scheduled because no available worker.
         assert_eq!(vacuum.vacuum_object().await.unwrap().len(), 0);
         let _receiver = compactor_manager.add_compactor(context_id);
diff --git a/src/meta/src/manager/catalog/database.rs b/src/meta/src/manager/catalog/database.rs
index aa4317add003b..5911188e2bfb5 100644
--- a/src/meta/src/manager/catalog/database.rs
+++ b/src/meta/src/manager/catalog/database.rs
@@ -18,17 +18,18 @@ use std::collections::{BTreeMap, HashMap, HashSet};
 use itertools::Itertools;
 use risingwave_common::bail;
 use risingwave_common::catalog::TableOption;
+use risingwave_pb::catalog::subscription::PbSubscriptionState;
 use risingwave_pb::catalog::table::TableType;
 use risingwave_pb::catalog::{
-    Connection, CreateType, Database, Function, Index, PbStreamJobStatus, Schema, Sink, Source,
-    StreamJobStatus, Subscription, Table, View,
+    Connection, CreateType, Database, Function, Index, PbStreamJobStatus, Schema, Secret, Sink,
+    Source, StreamJobStatus, Subscription, Table, View,
 };
 use risingwave_pb::data::DataType;
 use risingwave_pb::user::grant_privilege::PbObject;
 
 use super::{
-    ConnectionId, DatabaseId, FunctionId, RelationId, SchemaId, SinkId, SourceId, SubscriptionId,
-    ViewId,
+    ConnectionId, DatabaseId, FunctionId, RelationId, SchemaId, SecretId, SinkId, SourceId,
+    SubscriptionId, ViewId,
 };
 use crate::manager::{IndexId, MetaSrvEnv, TableId, UserId};
 use crate::model::MetadataModel;
@@ -45,6 +46,7 @@ pub type Catalog = (
     Vec<View>,
     Vec<Function>,
     Vec<Connection>,
+    Vec<Secret>,
 );
 
 type DatabaseKey = String;
@@ -75,10 +77,16 @@ pub struct DatabaseManager {
     pub(super) functions: BTreeMap<FunctionId, Function>,
     /// Cached connection information.
     pub(super) connections: BTreeMap<ConnectionId, Connection>,
+    /// Cached secret information.
+    pub(super) secrets: BTreeMap<SecretId, Secret>,
 
     /// Relation reference count mapping.
-    // TODO(zehua): avoid key conflicts after distinguishing table's and source's id generator.
     pub(super) relation_ref_count: HashMap<RelationId, usize>,
+
+    /// Secret reference count mapping
+    pub(super) secret_ref_count: HashMap<SecretId, usize>,
+    /// Connection reference count mapping.
+    pub(super) connection_ref_count: HashMap<ConnectionId, usize>,
     // In-progress creation tracker.
     pub(super) in_progress_creation_tracker: HashSet<RelationKey>,
     // In-progress creating streaming job tracker: this is a temporary workaround to avoid clean up
@@ -100,8 +108,11 @@ impl DatabaseManager {
         let functions = Function::list(env.meta_store().as_kv()).await?;
         let connections = Connection::list(env.meta_store().as_kv()).await?;
         let subscriptions = Subscription::list(env.meta_store().as_kv()).await?;
+        let secrets = Secret::list(env.meta_store().as_kv()).await?;
 
         let mut relation_ref_count = HashMap::new();
+        let mut connection_ref_count = HashMap::new();
+        let mut _secret_ref_count = HashMap::new();
 
         let databases = BTreeMap::from_iter(
             databases
@@ -110,9 +121,8 @@ impl DatabaseManager {
         );
         let schemas = BTreeMap::from_iter(schemas.into_iter().map(|schema| (schema.id, schema)));
         let sources = BTreeMap::from_iter(sources.into_iter().map(|source| {
-            // TODO(weili): wait for yezizp to refactor ref cnt
             if let Some(connection_id) = source.connection_id {
-                *relation_ref_count.entry(connection_id).or_default() += 1;
+                *connection_ref_count.entry(connection_id).or_default() += 1;
             }
             (source.id, source)
         }));
@@ -120,14 +130,18 @@ impl DatabaseManager {
             for depend_relation_id in &sink.dependent_relations {
                 *relation_ref_count.entry(*depend_relation_id).or_default() += 1;
             }
+            if let Some(connection_id) = sink.connection_id {
+                *connection_ref_count.entry(connection_id).or_default() += 1;
+            }
             (sink.id, sink)
         }));
         let subscriptions = BTreeMap::from_iter(subscriptions.into_iter().map(|subscription| {
-            for depend_relation_id in &subscription.dependent_relations {
-                *relation_ref_count.entry(*depend_relation_id).or_default() += 1;
-            }
+            *relation_ref_count
+                .entry(subscription.dependent_table_id)
+                .or_default() += 1;
             (subscription.id, subscription)
         }));
+        let secrets = BTreeMap::from_iter(secrets.into_iter().map(|secret| (secret.id, secret)));
         let indexes = BTreeMap::from_iter(indexes.into_iter().map(|index| (index.id, index)));
         let tables = BTreeMap::from_iter(tables.into_iter().map(|table| {
             for depend_relation_id in &table.dependent_relations {
@@ -144,6 +158,8 @@ impl DatabaseManager {
         let functions = BTreeMap::from_iter(functions.into_iter().map(|f| (f.id, f)));
         let connections = BTreeMap::from_iter(connections.into_iter().map(|c| (c.id, c)));
 
+        // todo: scan over stream source info and sink to update secret ref count `_secret_ref_count`
+
         Ok(Self {
             databases,
             schemas,
@@ -156,6 +172,9 @@ impl DatabaseManager {
             functions,
             connections,
             relation_ref_count,
+            connection_ref_count,
+            secrets,
+            secret_ref_count: _secret_ref_count,
             in_progress_creation_tracker: HashSet::default(),
             in_progress_creation_streaming_job: HashMap::default(),
             in_progress_creating_tables: HashMap::default(),
@@ -185,10 +204,7 @@ impl DatabaseManager {
                 .collect_vec(),
             self.subscriptions
                 .values()
-                .filter(|t| {
-                    t.stream_job_status == PbStreamJobStatus::Unspecified as i32
-                        || t.stream_job_status == PbStreamJobStatus::Created as i32
-                })
+                .filter(|t| t.subscription_state == PbSubscriptionState::Created as i32)
                 .cloned()
                 .collect_vec(),
             self.indexes
@@ -202,6 +218,7 @@ impl DatabaseManager {
             self.views.values().cloned().collect_vec(),
             self.functions.values().cloned().collect_vec(),
             self.connections.values().cloned().collect_vec(),
+            self.secrets.values().cloned().collect_vec(),
         )
     }
 
@@ -294,10 +311,24 @@ impl DatabaseManager {
         }
     }
 
+    pub fn check_secret_name_duplicated(&self, secret_key: &RelationKey) -> MetaResult<()> {
+        if self.secrets.values().any(|x| {
+            x.database_id == secret_key.0 && x.schema_id == secret_key.1 && x.name.eq(&secret_key.2)
+        }) {
+            Err(MetaError::catalog_duplicated("secret", &secret_key.2))
+        } else {
+            Ok(())
+        }
+    }
+
     pub fn list_databases(&self) -> Vec<Database> {
         self.databases.values().cloned().collect_vec()
     }
 
+    pub fn list_schemas(&self) -> Vec<Schema> {
+        self.schemas.values().cloned().collect_vec()
+    }
+
     pub fn list_creating_background_mvs(&self) -> Vec<Table> {
         self.tables
             .values()
@@ -361,6 +392,14 @@ impl DatabaseManager {
             .collect_vec()
     }
 
+    pub fn list_view_ids(&self, schema_id: SchemaId) -> Vec<ViewId> {
+        self.views
+            .values()
+            .filter(|view| view.schema_id == schema_id)
+            .map(|view| view.id)
+            .collect_vec()
+    }
+
     pub fn list_sources(&self) -> Vec<Source> {
         self.sources.values().cloned().collect_vec()
     }
@@ -398,7 +437,6 @@ impl DatabaseManager {
             .keys()
             .copied()
             .chain(self.sinks.keys().copied())
-            .chain(self.subscriptions.keys().copied())
             .chain(self.indexes.keys().copied())
             .chain(self.sources.keys().copied())
             .chain(
@@ -442,11 +480,11 @@ impl DatabaseManager {
             && self.views.values().all(|v| v.schema_id != schema_id)
     }
 
-    pub fn increase_ref_count(&mut self, relation_id: RelationId) {
+    pub fn increase_relation_ref_count(&mut self, relation_id: RelationId) {
         *self.relation_ref_count.entry(relation_id).or_insert(0) += 1;
     }
 
-    pub fn decrease_ref_count(&mut self, relation_id: RelationId) {
+    pub fn decrease_relation_ref_count(&mut self, relation_id: RelationId) {
         match self.relation_ref_count.entry(relation_id) {
             Entry::Occupied(mut o) => {
                 *o.get_mut() -= 1;
@@ -458,6 +496,38 @@ impl DatabaseManager {
         }
     }
 
+    pub fn increase_secret_ref_count(&mut self, secret_id: SecretId) {
+        *self.secret_ref_count.entry(secret_id).or_insert(0) += 1;
+    }
+
+    pub fn decrease_secret_ref_count(&mut self, secret_id: SecretId) {
+        match self.secret_ref_count.entry(secret_id) {
+            Entry::Occupied(mut o) => {
+                *o.get_mut() -= 1;
+                if *o.get() == 0 {
+                    o.remove_entry();
+                }
+            }
+            Entry::Vacant(_) => unreachable!(),
+        }
+    }
+
+    pub fn increase_connection_ref_count(&mut self, connection_id: ConnectionId) {
+        *self.connection_ref_count.entry(connection_id).or_insert(0) += 1;
+    }
+
+    pub fn decrease_connection_ref_count(&mut self, connection_id: ConnectionId) {
+        match self.connection_ref_count.entry(connection_id) {
+            Entry::Occupied(mut o) => {
+                *o.get_mut() -= 1;
+                if *o.get() == 0 {
+                    o.remove_entry();
+                }
+            }
+            Entry::Vacant(_) => unreachable!(),
+        }
+    }
+
     pub fn has_creation_in_database(&self, database_id: DatabaseId) -> bool {
         self.in_progress_creation_tracker
             .iter()
@@ -610,7 +680,6 @@ impl DatabaseManager {
         }
     }
 
-    // TODO(zehua): refactor when using SourceId.
     pub fn ensure_table_view_or_source_id(&self, table_id: &TableId) -> MetaResult<()> {
         if self.tables.contains_key(table_id)
             || self.sources.contains_key(table_id)
diff --git a/src/meta/src/manager/catalog/fragment.rs b/src/meta/src/manager/catalog/fragment.rs
index feb3dc3026dbf..9cb0d25a09ae0 100644
--- a/src/meta/src/manager/catalog/fragment.rs
+++ b/src/meta/src/manager/catalog/fragment.rs
@@ -21,18 +21,22 @@ use risingwave_common::bail;
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::{ActorMapping, ParallelUnitId, ParallelUnitMapping};
-use risingwave_common::util::stream_graph_visitor::{visit_stream_node, visit_stream_node_cont};
+use risingwave_common::util::stream_graph_visitor::{
+    visit_stream_node, visit_stream_node_cont, visit_stream_node_cont_mut,
+};
 use risingwave_connector::source::SplitImpl;
 use risingwave_meta_model_v2::SourceId;
+use risingwave_pb::common::{PbParallelUnitMapping, PbWorkerSlotMapping};
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
 use risingwave_pb::meta::table_fragments::actor_status::ActorState;
 use risingwave_pb::meta::table_fragments::{ActorStatus, Fragment, State};
-use risingwave_pb::meta::FragmentParallelUnitMapping;
+use risingwave_pb::meta::FragmentWorkerSlotMapping;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
 use risingwave_pb::stream_plan::update_mutation::MergeUpdate;
 use risingwave_pb::stream_plan::{
     DispatchStrategy, Dispatcher, DispatcherType, FragmentTypeFlag, StreamActor, StreamNode,
 };
+use risingwave_pb::stream_service::BuildActorInfo;
 use tokio::sync::{RwLock, RwLockReadGuard};
 
 use crate::barrier::Reschedule;
@@ -40,10 +44,10 @@ use crate::manager::cluster::WorkerId;
 use crate::manager::{commit_meta, commit_meta_with_trx, LocalNotification, MetaSrvEnv};
 use crate::model::{
     ActorId, BTreeMapTransaction, FragmentId, MetadataModel, MigrationPlan, TableFragments,
-    TableParallelism, ValTransaction,
+    TableParallelism,
 };
 use crate::storage::Transaction;
-use crate::stream::{SplitAssignment, TableRevision};
+use crate::stream::{to_build_actor_info, SplitAssignment, TableRevision};
 use crate::{MetaError, MetaResult};
 
 pub struct FragmentManagerCore {
@@ -55,18 +59,21 @@ impl FragmentManagerCore {
     /// List all fragment vnode mapping info that not in `State::Initial`.
     pub fn all_running_fragment_mappings(
         &self,
-    ) -> impl Iterator<Item = FragmentParallelUnitMapping> + '_ {
+    ) -> impl Iterator<Item = FragmentWorkerSlotMapping> + '_ {
         self.table_fragments
             .values()
             .filter(|tf| tf.state() != State::Initial)
             .flat_map(|table_fragments| {
-                table_fragments.fragments.values().map(|fragment| {
-                    let parallel_unit_mapping = fragment.vnode_mapping.clone().unwrap();
-                    FragmentParallelUnitMapping {
+                table_fragments
+                    .fragments
+                    .values()
+                    .map(move |fragment| FragmentWorkerSlotMapping {
                         fragment_id: fragment.fragment_id,
-                        mapping: Some(parallel_unit_mapping),
-                    }
-                })
+                        mapping: Some(FragmentManager::convert_mapping(
+                            &table_fragments.actor_status,
+                            fragment.vnode_mapping.as_ref().unwrap(),
+                        )),
+                    })
             })
     }
 
@@ -191,18 +198,23 @@ impl FragmentManager {
     async fn notify_fragment_mapping(&self, table_fragment: &TableFragments, operation: Operation) {
         // Notify all fragment mapping to frontend nodes
         for fragment in table_fragment.fragments.values() {
-            let mapping = fragment
-                .vnode_mapping
-                .clone()
-                .expect("no data distribution found");
-            let fragment_mapping = FragmentParallelUnitMapping {
+            let fragment_mapping = FragmentWorkerSlotMapping {
                 fragment_id: fragment.fragment_id,
-                mapping: Some(mapping),
+                mapping: Some(Self::convert_mapping(
+                    &table_fragment.actor_status,
+                    fragment
+                        .vnode_mapping
+                        .as_ref()
+                        .expect("no data distribution found"),
+                )),
             };
 
             self.env
                 .notification_manager()
-                .notify_frontend(operation, Info::ParallelUnitMapping(fragment_mapping))
+                .notify_frontend(
+                    operation,
+                    Info::StreamingWorkerSlotMapping(fragment_mapping),
+                )
                 .await;
         }
 
@@ -560,7 +572,6 @@ impl FragmentManager {
             .filter_map(|table_id| map.get(table_id).cloned())
             .collect_vec();
 
-        let mut dirty_sink_into_table_upstream_fragment_id = HashSet::new();
         let mut table_fragments = BTreeMapTransaction::new(map);
         let mut table_ids_to_unregister_from_hummock = vec![];
         for table_fragment in &to_delete_table_fragments {
@@ -595,32 +606,14 @@ impl FragmentManager {
                         })
                     });
             }
-
-            if let Some(sink_fragment) = table_fragment.sink_fragment() {
-                let dispatchers = sink_fragment
-                    .get_actors()
-                    .iter()
-                    .map(|actor| actor.get_dispatcher())
-                    .collect_vec();
-
-                if !dispatchers.is_empty() {
-                    dirty_sink_into_table_upstream_fragment_id.insert(sink_fragment.fragment_id);
-                }
-            }
         }
 
-        if !dirty_sink_into_table_upstream_fragment_id.is_empty() {
-            let to_delete_table_ids: HashSet<_> = to_delete_table_fragments
-                .iter()
-                .map(|table| table.table_id())
-                .collect();
-
-            Self::clean_dirty_table_sink_downstreams(
-                dirty_sink_into_table_upstream_fragment_id,
-                to_delete_table_ids,
-                &mut table_fragments,
-            )?;
-        }
+        let to_delete_table_ids: HashSet<_> = to_delete_table_fragments
+            .iter()
+            .map(|table| table.table_id())
+            .collect();
+
+        Self::clean_dirty_table_sink_downstreams(to_delete_table_ids, &mut table_fragments)?;
 
         if table_ids.is_empty() {
             commit_meta!(self, table_fragments)?;
@@ -646,36 +639,43 @@ impl FragmentManager {
     // but the union branch that attaches the downstream table to the sink fragment may still exist.
     // This could lead to issues. Therefore, we need to find the sink fragment’s downstream, then locate its union node and delete the dirty merge.
     fn clean_dirty_table_sink_downstreams(
-        dirty_sink_into_table_upstream_fragment_id: HashSet<u32>,
         to_delete_table_ids: HashSet<TableId>,
         table_fragments: &mut BTreeMapTransaction<'_, TableId, TableFragments>,
     ) -> MetaResult<()> {
         tracing::info!("cleaning dirty downstream merge nodes for table sink");
 
+        let mut all_fragment_ids = HashSet::new();
+
+        for (table_id, table_fragment) in table_fragments.tree_ref() {
+            if to_delete_table_ids.contains(table_id) {
+                continue;
+            }
+
+            all_fragment_ids.extend(table_fragment.fragment_ids());
+        }
+
         let mut dirty_downstream_table_ids = HashMap::new();
-        for (table_id, table_fragment) in table_fragments.tree_mut() {
+
+        for (table_id, table_fragment) in table_fragments.tree_ref() {
             if to_delete_table_ids.contains(table_id) {
                 continue;
             }
 
-            for fragment in table_fragment.fragments.values_mut() {
+            for fragment in table_fragment.fragments.values() {
                 if fragment
                     .get_upstream_fragment_ids()
                     .iter()
-                    .all(|upstream_fragment_id| {
-                        !dirty_sink_into_table_upstream_fragment_id.contains(upstream_fragment_id)
-                    })
+                    .all(|upstream_fragment_id| all_fragment_ids.contains(upstream_fragment_id))
                 {
                     continue;
                 }
 
-                for actor in &mut fragment.actors {
-                    visit_stream_node_cont(actor.nodes.as_mut().unwrap(), |node| {
+                for actor in &fragment.actors {
+                    visit_stream_node_cont(actor.nodes.as_ref().unwrap(), |node| {
                         if let Some(NodeBody::Union(_)) = node.node_body {
-                            for input in &mut node.input {
-                                if let Some(NodeBody::Merge(merge_node)) = &mut input.node_body
-                                    && dirty_sink_into_table_upstream_fragment_id
-                                        .contains(&merge_node.upstream_fragment_id)
+                            for input in &node.input {
+                                if let Some(NodeBody::Merge(merge_node)) = &input.node_body
+                                    && !all_fragment_ids.contains(&merge_node.upstream_fragment_id)
                                 {
                                     dirty_downstream_table_ids
                                         .insert(*table_id, fragment.fragment_id);
@@ -686,12 +686,6 @@ impl FragmentManager {
                         true
                     })
                 }
-
-                fragment
-                    .upstream_fragment_ids
-                    .retain(|upstream_fragment_id| {
-                        !dirty_sink_into_table_upstream_fragment_id.contains(upstream_fragment_id)
-                    });
             }
         }
 
@@ -705,13 +699,16 @@ impl FragmentManager {
                 .get_mut(&fragment_id)
                 .with_context(|| format!("fragment not exist: id={}", fragment_id))?;
 
+            fragment
+                .upstream_fragment_ids
+                .retain(|upstream_fragment_id| all_fragment_ids.contains(upstream_fragment_id));
+
             for actor in &mut fragment.actors {
-                visit_stream_node_cont(actor.nodes.as_mut().unwrap(), |node| {
+                visit_stream_node_cont_mut(actor.nodes.as_mut().unwrap(), |node| {
                     if let Some(NodeBody::Union(_)) = node.node_body {
                         node.input.retain_mut(|input| {
                             if let Some(NodeBody::Merge(merge_node)) = &mut input.node_body
-                                && dirty_sink_into_table_upstream_fragment_id
-                                    .contains(&merge_node.upstream_fragment_id)
+                                && !all_fragment_ids.contains(&merge_node.upstream_fragment_id)
                             {
                                 false
                             } else {
@@ -849,14 +846,20 @@ impl FragmentManager {
     pub async fn all_node_actors(
         &self,
         include_inactive: bool,
-    ) -> HashMap<WorkerId, Vec<StreamActor>> {
+        subscriptions: &HashMap<TableId, HashMap<u32, u64>>,
+    ) -> HashMap<WorkerId, Vec<BuildActorInfo>> {
         let mut actor_maps = HashMap::new();
 
         let map = &self.core.read().await.table_fragments;
         for fragments in map.values() {
+            let table_id = fragments.table_id();
             for (node_id, actors) in fragments.worker_actors(include_inactive) {
                 let node_actors = actor_maps.entry(node_id).or_insert_with(Vec::new);
-                node_actors.extend(actors);
+                node_actors.extend(
+                    actors
+                        .into_iter()
+                        .map(|actor| to_build_actor_info(actor, subscriptions, table_id)),
+                );
             }
         }
 
@@ -1270,11 +1273,14 @@ impl FragmentManager {
 
                 *fragment.vnode_mapping.as_mut().unwrap() = vnode_mapping.clone();
 
+                let worker_slot_mapping = Self::convert_mapping(&actor_status, &vnode_mapping);
+
                 // Notify fragment mapping to frontend nodes.
-                let fragment_mapping = FragmentParallelUnitMapping {
+                let fragment_mapping = FragmentWorkerSlotMapping {
                     fragment_id: *fragment_id as FragmentId,
-                    mapping: Some(vnode_mapping),
+                    mapping: Some(worker_slot_mapping),
                 };
+
                 fragment_mapping_to_notify.push(fragment_mapping);
             }
 
@@ -1394,13 +1400,30 @@ impl FragmentManager {
         for mapping in fragment_mapping_to_notify {
             self.env
                 .notification_manager()
-                .notify_frontend(Operation::Update, Info::ParallelUnitMapping(mapping))
+                .notify_frontend(Operation::Update, Info::StreamingWorkerSlotMapping(mapping))
                 .await;
         }
 
         Ok(())
     }
 
+    fn convert_mapping(
+        actor_status: &BTreeMap<ActorId, ActorStatus>,
+        vnode_mapping: &PbParallelUnitMapping,
+    ) -> PbWorkerSlotMapping {
+        let parallel_unit_to_worker = actor_status
+            .values()
+            .map(|actor_status| {
+                let parallel_unit = actor_status.get_parallel_unit().unwrap();
+                (parallel_unit.id, parallel_unit.worker_node_id)
+            })
+            .collect();
+
+        ParallelUnitMapping::from_protobuf(vnode_mapping)
+            .to_worker_slot(&parallel_unit_to_worker)
+            .to_protobuf()
+    }
+
     pub async fn table_node_actors(
         &self,
         table_ids: &HashSet<TableId>,
diff --git a/src/meta/src/manager/catalog/mod.rs b/src/meta/src/manager/catalog/mod.rs
index 5a316696e6195..3aeab64bef128 100644
--- a/src/meta/src/manager/catalog/mod.rs
+++ b/src/meta/src/manager/catalog/mod.rs
@@ -19,7 +19,6 @@ mod utils;
 
 use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
 use std::iter;
-use std::option::Option::Some;
 use std::sync::Arc;
 
 use anyhow::{anyhow, Context};
@@ -27,16 +26,17 @@ pub use database::*;
 pub use fragment::*;
 use itertools::Itertools;
 use risingwave_common::catalog::{
-    is_subscription_internal_table, valid_table_name, TableId as StreamingJobId, TableOption,
-    DEFAULT_DATABASE_NAME, DEFAULT_SCHEMA_NAME, DEFAULT_SUPER_USER, DEFAULT_SUPER_USER_FOR_PG,
+    valid_table_name, TableId as StreamingJobId, TableOption, DEFAULT_DATABASE_NAME,
+    DEFAULT_SCHEMA_NAME, DEFAULT_SUPER_USER, DEFAULT_SUPER_USER_FOR_PG,
     DEFAULT_SUPER_USER_FOR_PG_ID, DEFAULT_SUPER_USER_ID, SYSTEM_SCHEMAS,
 };
-use risingwave_common::{bail, ensure};
+use risingwave_common::{bail, current_cluster_version, ensure};
 use risingwave_connector::source::{should_copy_to_format_encode_options, UPSTREAM_SOURCE_KEY};
+use risingwave_pb::catalog::subscription::PbSubscriptionState;
 use risingwave_pb::catalog::table::{OptionalAssociatedSourceId, TableType};
 use risingwave_pb::catalog::{
     Comment, Connection, CreateType, Database, Function, Index, PbSource, PbStreamJobStatus,
-    Schema, Sink, Source, StreamJobStatus, Subscription, Table, View,
+    Schema, Secret, Sink, Source, StreamJobStatus, Subscription, Table, View,
 };
 use risingwave_pb::ddl_service::{alter_owner_request, alter_set_schema_request};
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
@@ -49,7 +49,7 @@ use user::*;
 use crate::manager::{
     IdCategory, MetaSrvEnv, NotificationVersion, StreamingJob, IGNORED_NOTIFICATION_VERSION,
 };
-use crate::model::{BTreeMapTransaction, MetadataModel, TableFragments, ValTransaction};
+use crate::model::{BTreeMapTransaction, MetadataModel, TableFragments};
 use crate::storage::Transaction;
 use crate::{MetaError, MetaResult};
 
@@ -63,6 +63,7 @@ pub type RelationId = u32;
 pub type IndexId = u32;
 pub type ViewId = u32;
 pub type FunctionId = u32;
+pub type SecretId = u32;
 
 pub type UserId = u32;
 pub type ConnectionId = u32;
@@ -83,6 +84,7 @@ macro_rules! commit_meta_with_trx {
         {
             use tracing::Instrument;
             use $crate::storage::meta_store::MetaStore;
+            use $crate::model::{InMemValTransaction, ValTransaction};
             async {
                 // Apply the change in `ValTransaction` to trx
                 $(
@@ -185,6 +187,7 @@ impl CatalogManager {
     /// We identify a 'legacy' source based on two conditions:
     /// 1. The `format_encode_options` in `source_info` is empty.
     /// 2. Keys with certain prefixes belonging to `format_encode_options` exist in `with_properties` instead.
+    ///
     /// And if the source is identified as 'legacy', we copy the misplaced keys from `with_properties` to `format_encode_options`.
     async fn source_backward_compat_check(&self) -> MetaResult<()> {
         let core = &mut *self.core.lock().await;
@@ -327,6 +330,7 @@ impl CatalogManager {
         let mut users = BTreeMapTransaction::new(&mut user_core.user_info);
         let mut functions = BTreeMapTransaction::new(&mut database_core.functions);
         let mut connections = BTreeMapTransaction::new(&mut database_core.connections);
+        let mut secrets = BTreeMapTransaction::new(&mut database_core.secrets);
 
         /// `drop_by_database_id` provides a wrapper for dropping relations by database id, it will
         /// return the relation ids that dropped.
@@ -359,6 +363,7 @@ impl CatalogManager {
             let views_to_drop = drop_by_database_id!(views, database_id);
             let functions_to_drop = drop_by_database_id!(functions, database_id);
             let connections_to_drop = drop_by_database_id!(connections, database_id);
+            let secrets_to_drop = drop_by_database_id!(secrets, database_id);
             connections_dropped = connections_to_drop.clone();
 
             let objects = std::iter::once(Object::DatabaseId(database_id))
@@ -420,6 +425,7 @@ impl CatalogManager {
                         .iter()
                         .map(|connection| connection.owner),
                 )
+                .chain(secrets_to_drop.iter().map(|secret| secret.owner))
                 .for_each(|owner_id| user_core.decrease_ref(owner_id));
 
             // Update relation ref count.
@@ -432,9 +438,8 @@ impl CatalogManager {
             for view in &views_to_drop {
                 database_core.relation_ref_count.remove(&view.id);
             }
-            // TODO(weili): wait for yezizp to refactor ref cnt
             for connection in &connections_to_drop {
-                database_core.relation_ref_count.remove(&connection.id);
+                database_core.connection_ref_count.remove(&connection.id);
             }
             for user in users_need_update {
                 self.notify_frontend(Operation::Update, Info::User(user))
@@ -477,6 +482,57 @@ impl CatalogManager {
         }
     }
 
+    pub async fn create_secret(&self, secret: Secret) -> MetaResult<NotificationVersion> {
+        let core = &mut *self.core.lock().await;
+        let database_core = &mut core.database;
+        let user_core = &mut core.user;
+        database_core.ensure_database_id(secret.database_id)?;
+        database_core.ensure_schema_id(secret.schema_id)?;
+        #[cfg(not(test))]
+        user_core.ensure_user_id(secret.owner)?;
+        let key = (
+            secret.database_id as DatabaseId,
+            secret.schema_id as SchemaId,
+            secret.name.clone(),
+        );
+        database_core.check_secret_name_duplicated(&key)?;
+
+        let secret_id = secret.id;
+        let mut secret_entry = BTreeMapTransaction::new(&mut database_core.secrets);
+        secret_entry.insert(secret_id, secret.to_owned());
+        commit_meta!(self, secret_entry)?;
+
+        user_core.increase_ref(secret.owner);
+
+        let version = self
+            .notify_frontend(Operation::Add, Info::Secret(secret))
+            .await;
+        Ok(version)
+    }
+
+    pub async fn drop_secret(&self, secret_id: SecretId) -> MetaResult<NotificationVersion> {
+        let core = &mut *self.core.lock().await;
+        let database_core = &mut core.database;
+        let user_core = &mut core.user;
+        let mut secrets = BTreeMapTransaction::new(&mut database_core.secrets);
+
+        // todo: impl a ref count check for secret
+        // if secret is used by other relations, not found in the catalog or do not have the privilege to drop, return error
+        // else: commit the change and notify frontend
+
+        let secret = secrets
+            .remove(secret_id)
+            .ok_or_else(|| anyhow!("secret not found"))?;
+
+        commit_meta!(self, secrets)?;
+        user_core.decrease_ref(secret.owner);
+
+        let version = self
+            .notify_frontend(Operation::Delete, Info::Secret(secret))
+            .await;
+        Ok(version)
+    }
+
     pub async fn create_connection(
         &self,
         connection: Connection,
@@ -520,12 +576,11 @@ impl CatalogManager {
         let user_core = &mut core.user;
         let mut connections = BTreeMapTransaction::new(&mut database_core.connections);
 
-        // TODO(weili): wait for yezizp to refactor ref cnt
-        match database_core.relation_ref_count.get(&conn_id) {
+        match database_core.connection_ref_count.get(&conn_id) {
             Some(ref_count) => {
                 let connection_name = connections
                     .get(&conn_id)
-                    .ok_or_else(|| anyhow!("connection not found"))?
+                    .ok_or_else(|| MetaError::catalog_id_not_found("connection", conn_id))?
                     .name
                     .clone();
                 Err(MetaError::permission_denied(format!(
@@ -536,7 +591,7 @@ impl CatalogManager {
             None => {
                 let connection = connections
                     .remove(conn_id)
-                    .ok_or_else(|| anyhow!("connection not found"))?;
+                    .ok_or_else(|| MetaError::catalog_id_not_found("connection", conn_id))?;
 
                 commit_meta!(self, connections)?;
                 user_core.decrease_ref(connection.owner);
@@ -632,7 +687,7 @@ impl CatalogManager {
         user_core.increase_ref(view.owner);
 
         for &dependent_relation_id in &view.dependent_relations {
-            database_core.increase_ref_count(dependent_relation_id);
+            database_core.increase_relation_ref_count(dependent_relation_id);
         }
 
         let version = self
@@ -681,7 +736,7 @@ impl CatalogManager {
 
         let function = functions
             .remove(function_id)
-            .ok_or_else(|| anyhow!("function not found"))?;
+            .ok_or_else(|| MetaError::catalog_id_not_found("function", function_id))?;
 
         let objects = &[Object::FunctionId(function_id)];
         let users_need_update = Self::update_user_privileges(&mut users, objects);
@@ -715,9 +770,6 @@ impl CatalogManager {
                     .await
             }
             StreamingJob::Sink(sink, _) => self.start_create_sink_procedure(sink).await,
-            StreamingJob::Subscription(subscription) => {
-                self.start_create_subscription_procedure(subscription).await
-            }
             StreamingJob::Index(index, index_table) => {
                 self.start_create_index_procedure(index, index_table).await
             }
@@ -792,7 +844,6 @@ impl CatalogManager {
         database_core.ensure_database_id(table.database_id)?;
         database_core.ensure_schema_id(table.schema_id)?;
         for dependent_id in &table.dependent_relations {
-            // TODO(zehua): refactor when using SourceId.
             database_core.ensure_table_view_or_source_id(dependent_id)?;
         }
         #[cfg(not(test))]
@@ -813,7 +864,7 @@ impl CatalogManager {
         commit_meta!(self, tables)?;
 
         for &dependent_relation_id in &table.dependent_relations {
-            database_core.increase_ref_count(dependent_relation_id);
+            database_core.increase_relation_ref_count(dependent_relation_id);
         }
         user_core.increase_ref(table.owner);
         Ok(())
@@ -847,6 +898,7 @@ impl CatalogManager {
     ///    with:
     ///    1. `stream_job_status` = CREATING
     ///    2. Not belonging to a background stream job.
+    ///
     ///    Clean up these hanging tables by the id.
     pub async fn clean_dirty_tables(&self, fragment_manager: FragmentManagerRef) -> MetaResult<()> {
         let core = &mut *self.core.lock().await;
@@ -1011,7 +1063,7 @@ impl CatalogManager {
             if table.table_type != TableType::Internal as i32 {
                 // Recovered when init database manager.
                 for relation_id in &table.dependent_relations {
-                    database_core.decrease_ref_count(*relation_id);
+                    database_core.decrease_relation_ref_count(*relation_id);
                 }
                 // Recovered when init user manager.
                 tracing::debug!("decrease ref for {}", table.id);
@@ -1122,7 +1174,7 @@ impl CatalogManager {
             {
                 let database_core = &mut core.database;
                 for &dependent_relation_id in &table.dependent_relations {
-                    database_core.decrease_ref_count(dependent_relation_id);
+                    database_core.decrease_relation_ref_count(dependent_relation_id);
                 }
             }
         }
@@ -1202,7 +1254,7 @@ impl CatalogManager {
                 .tree_ref()
                 .iter()
                 .filter_map(|(_, subscription)| {
-                    if subscription.dependent_relations.contains(&relation_id) {
+                    if subscription.dependent_table_id == relation_id {
                         Some(RelationInfo::Subscription(subscription.clone()))
                     } else {
                         None
@@ -1549,11 +1601,6 @@ impl CatalogManager {
                     if !all_subscription_ids.insert(subscription.id) {
                         continue;
                     }
-                    let table_fragments = fragment_manager
-                        .select_table_fragments_by_table_id(&subscription.id.into())
-                        .await?;
-
-                    all_internal_table_ids.extend(table_fragments.internal_table_ids());
 
                     if let Some(ref_count) = database_core
                         .relation_ref_count
@@ -1714,30 +1761,25 @@ impl CatalogManager {
         // decrease dependent relations
         for table in &tables_removed {
             for dependent_relation_id in &table.dependent_relations {
-                database_core.decrease_ref_count(*dependent_relation_id);
+                database_core.decrease_relation_ref_count(*dependent_relation_id);
             }
         }
 
         for view in &views_removed {
             for dependent_relation_id in &view.dependent_relations {
-                database_core.decrease_ref_count(*dependent_relation_id);
+                database_core.decrease_relation_ref_count(*dependent_relation_id);
             }
         }
 
         for sink in &sinks_removed {
-            if let Some(connection_id) = sink.connection_id {
-                // TODO(siyuan): wait for yezizp to refactor ref cnt
-                database_core.decrease_ref_count(connection_id);
-            }
+            refcnt_dec_connection(database_core, sink.connection_id);
             for dependent_relation_id in &sink.dependent_relations {
-                database_core.decrease_ref_count(*dependent_relation_id);
+                database_core.decrease_relation_ref_count(*dependent_relation_id);
             }
         }
 
         for subscription in &subscriptions_removed {
-            for dependent_relation_id in &subscription.dependent_relations {
-                database_core.decrease_ref_count(*dependent_relation_id);
-            }
+            database_core.decrease_relation_ref_count(subscription.dependent_table_id);
         }
 
         let version = self
@@ -1780,7 +1822,6 @@ impl CatalogManager {
             .into_iter()
             .map(|id| id.into())
             .chain(all_sink_ids.into_iter().map(|id| id.into()))
-            .chain(all_subscription_ids.into_iter().map(|id| id.into()))
             .chain(all_streaming_job_source_ids.into_iter().map(|id| id.into()))
             .collect_vec();
 
@@ -1873,7 +1914,7 @@ impl CatalogManager {
         }
 
         for subscription in database_mgr.subscriptions.values() {
-            if subscription.dependent_relations.contains(&relation_id) {
+            if subscription.dependent_table_id == relation_id {
                 let mut subscription = subscription.clone();
                 subscription.definition =
                     alter_relation_rename_refs(&subscription.definition, from, to);
@@ -2395,7 +2436,6 @@ impl CatalogManager {
             alter_owner_request::Object::SubscriptionId(subscription_id) => {
                 database_core.ensure_subscription_id(subscription_id)?;
                 let mut subscriptions = BTreeMapTransaction::new(&mut database_core.subscriptions);
-                let mut tables = BTreeMapTransaction::new(&mut database_core.tables);
                 let mut subscription = subscriptions.get_mut(subscription_id).unwrap();
                 let old_owner_id = subscription.owner;
                 if old_owner_id == owner_id {
@@ -2403,26 +2443,12 @@ impl CatalogManager {
                 }
                 subscription.owner = owner_id;
 
-                let mut relations = vec![Relation {
+                let relations = vec![Relation {
                     relation_info: Some(RelationInfo::Subscription(subscription.clone())),
                 }];
 
-                // internal tables
-                let internal_table_ids = fragment_manager
-                    .select_table_fragments_by_table_id(&(subscription_id.into()))
-                    .await?
-                    .internal_table_ids();
-                for id in internal_table_ids {
-                    let mut table = tables.get_mut(id).unwrap();
-                    assert_eq!(old_owner_id, table.owner);
-                    table.owner = owner_id;
-                    relations.push(Relation {
-                        relation_info: Some(RelationInfo::Table(table.clone())),
-                    });
-                }
-
                 relation_info = Info::RelationGroup(RelationGroup { relations });
-                commit_meta!(self, subscriptions, tables)?;
+                commit_meta!(self, subscriptions)?;
                 user_core.increase_ref(owner_id);
                 user_core.decrease_ref(old_owner_id);
             }
@@ -2659,14 +2685,6 @@ impl CatalogManager {
                     return Ok(IGNORED_NOTIFICATION_VERSION);
                 }
 
-                // internal tables.
-                let to_update_internal_table_ids = Vec::from_iter(
-                    fragment_manager
-                        .select_table_fragments_by_table_id(&(subscription_id.into()))
-                        .await?
-                        .internal_table_ids(),
-                );
-
                 database_core.check_relation_name_duplicated(&(
                     database_id,
                     new_schema_id,
@@ -2676,15 +2694,7 @@ impl CatalogManager {
                 let mut subscription = subscriptions.get_mut(subscription_id).unwrap();
                 subscription.schema_id = new_schema_id;
                 relation_infos.push(Some(RelationInfo::Subscription(subscription.clone())));
-
-                let mut tables = BTreeMapTransaction::new(&mut database_core.tables);
-                for table_id in to_update_internal_table_ids {
-                    let mut table = tables.get_mut(table_id).unwrap();
-                    table.schema_id = new_schema_id;
-                    relation_infos.push(Some(RelationInfo::Table(table.clone())));
-                }
-
-                commit_meta!(self, subscriptions, tables)?;
+                commit_meta!(self, subscriptions)?;
             }
         }
 
@@ -2784,7 +2794,7 @@ impl CatalogManager {
         database_core
             .get_connection(connection_id)
             .cloned()
-            .ok_or_else(|| anyhow!(format!("could not find connection {}", connection_id)).into())
+            .ok_or_else(|| MetaError::catalog_id_not_found("connection", connection_id))
     }
 
     pub async fn finish_create_source_procedure(
@@ -2995,7 +3005,7 @@ impl CatalogManager {
             database_core.mark_creating(&key);
             database_core.mark_creating_streaming_job(index_table.id, key);
             for &dependent_relation_id in &index_table.dependent_relations {
-                database_core.increase_ref_count(dependent_relation_id);
+                database_core.increase_relation_ref_count(dependent_relation_id);
             }
             // index table and index.
             user_core.increase_ref_count(index.owner, 2);
@@ -3016,7 +3026,7 @@ impl CatalogManager {
         database_core.unmark_creating(&key);
         database_core.unmark_creating_streaming_job(index_table.id);
         for &dependent_relation_id in &index_table.dependent_relations {
-            database_core.decrease_ref_count(dependent_relation_id);
+            database_core.decrease_relation_ref_count(dependent_relation_id);
         }
         // index table and index.
         user_core.decrease_ref_count(index.owner, 2);
@@ -3087,7 +3097,6 @@ impl CatalogManager {
         database_core.ensure_database_id(sink.database_id)?;
         database_core.ensure_schema_id(sink.schema_id)?;
         for dependent_id in &sink.dependent_relations {
-            // TODO(zehua): refactor when using SourceId.
             database_core.ensure_table_view_or_source_id(dependent_id)?;
         }
         let key = (sink.database_id, sink.schema_id, sink.name.clone());
@@ -3101,7 +3110,7 @@ impl CatalogManager {
             database_core.mark_creating(&key);
             database_core.mark_creating_streaming_job(sink.id, key);
             for &dependent_relation_id in &sink.dependent_relations {
-                database_core.increase_ref_count(dependent_relation_id);
+                database_core.increase_relation_ref_count(dependent_relation_id);
             }
             user_core.increase_ref(sink.owner);
             // We have validate the status of connection before starting the procedure.
@@ -3175,7 +3184,7 @@ impl CatalogManager {
         database_core.unmark_creating(&key);
         database_core.unmark_creating_streaming_job(sink.id);
         for &dependent_relation_id in &sink.dependent_relations {
-            database_core.decrease_ref_count(dependent_relation_id);
+            database_core.decrease_relation_ref_count(dependent_relation_id);
         }
         user_core.decrease_ref(sink.owner);
         refcnt_dec_connection(database_core, sink.connection_id);
@@ -3194,9 +3203,8 @@ impl CatalogManager {
         let user_core = &mut core.user;
         database_core.ensure_database_id(subscription.database_id)?;
         database_core.ensure_schema_id(subscription.schema_id)?;
-        for dependent_id in &subscription.dependent_relations {
-            database_core.ensure_table_view_or_source_id(dependent_id)?;
-        }
+        database_core
+            .ensure_table_view_or_source_id(&TableId::from(subscription.dependent_table_id))?;
         let key = (
             subscription.database_id,
             subscription.schema_id,
@@ -3211,39 +3219,36 @@ impl CatalogManager {
         } else {
             database_core.mark_creating(&key);
             database_core.mark_creating_streaming_job(subscription.id, key);
-            for &dependent_relation_id in &subscription.dependent_relations {
-                database_core.increase_ref_count(dependent_relation_id);
-            }
+            database_core.increase_relation_ref_count(subscription.dependent_table_id);
             user_core.increase_ref(subscription.owner);
+            let mut subscriptions = BTreeMapTransaction::new(&mut database_core.subscriptions);
+            subscriptions.insert(subscription.id, subscription.clone());
+            commit_meta!(self, subscriptions)?;
             Ok(())
         }
     }
 
     pub async fn finish_create_subscription_procedure(
         &self,
-        mut internal_tables: Vec<Table>,
-        mut subscription: Subscription,
-    ) -> MetaResult<NotificationVersion> {
+        subscription_id: SubscriptionId,
+    ) -> MetaResult<()> {
         let core = &mut *self.core.lock().await;
         let database_core = &mut core.database;
+        let mut subscriptions = BTreeMapTransaction::new(&mut database_core.subscriptions);
+        let mut subscription = subscriptions
+            .get(&subscription_id)
+            .ok_or_else(|| MetaError::catalog_id_not_found("subscription", subscription_id))?
+            .clone();
+        subscription.created_at_cluster_version = Some(current_cluster_version());
+        subscription.created_at_epoch = Some(Epoch::now().0);
         let key = (
             subscription.database_id,
             subscription.schema_id,
             subscription.name.clone(),
         );
-        let log_store_names: Vec<_> = internal_tables
-            .iter()
-            .filter(|a| is_subscription_internal_table(&subscription.name, a.get_name()))
-            .map(|a| a.get_name())
-            .collect();
-        if log_store_names.len() != 1 {
-            bail!("A subscription can only have one log_store_name");
-        }
-        subscription.subscription_internal_table_name = log_store_names.get(0).cloned().cloned();
-        let mut tables = BTreeMapTransaction::new(&mut database_core.tables);
-        let mut subscriptions = BTreeMapTransaction::new(&mut database_core.subscriptions);
+
         assert!(
-            !subscriptions.contains_key(&subscription.id)
+            subscription.subscription_state == Into::<i32>::into(PbSubscriptionState::Init)
                 && database_core.in_progress_creation_tracker.contains(&key),
             "subscription must be in creating procedure"
         );
@@ -3253,13 +3258,28 @@ impl CatalogManager {
             .in_progress_creation_streaming_job
             .remove(&subscription.id);
 
-        subscription.stream_job_status = PbStreamJobStatus::Created.into();
+        subscription.subscription_state = PbSubscriptionState::Created.into();
         subscriptions.insert(subscription.id, subscription.clone());
-        for table in &mut internal_tables {
-            table.stream_job_status = PbStreamJobStatus::Created.into();
-            tables.insert(table.id, table.clone());
-        }
-        commit_meta!(self, subscriptions, tables)?;
+        commit_meta!(self, subscriptions)?;
+        Ok(())
+    }
+
+    pub async fn notify_create_subscription(
+        &self,
+        subscription_id: SubscriptionId,
+    ) -> MetaResult<NotificationVersion> {
+        let core = &mut *self.core.lock().await;
+        let database_core = &mut core.database;
+        let subscriptions = BTreeMapTransaction::new(&mut database_core.subscriptions);
+        let subscription = subscriptions
+            .get(&subscription_id)
+            .ok_or_else(|| MetaError::catalog_id_not_found("subscription", subscription_id))?
+            .clone();
+        assert_eq!(
+            subscription.subscription_state,
+            Into::<i32>::into(PbSubscriptionState::Created)
+        );
+        commit_meta!(self, subscriptions)?;
 
         let version = self
             .notify_frontend(
@@ -3267,39 +3287,45 @@ impl CatalogManager {
                 Info::RelationGroup(RelationGroup {
                     relations: vec![Relation {
                         relation_info: RelationInfo::Subscription(subscription.to_owned()).into(),
-                    }]
-                    .into_iter()
-                    .chain(internal_tables.into_iter().map(|internal_table| Relation {
-                        relation_info: RelationInfo::Table(internal_table).into(),
-                    }))
-                    .collect_vec(),
+                    }],
                 }),
             )
             .await;
-
         Ok(version)
     }
 
-    pub async fn cancel_create_subscription_procedure(&self, subscription: &Subscription) {
+    pub async fn clean_dirty_subscription(&self) -> MetaResult<()> {
         let core = &mut *self.core.lock().await;
         let database_core = &mut core.database;
+        let mut subscriptions = BTreeMapTransaction::new(&mut database_core.subscriptions);
+        let remove_subscriptions = subscriptions
+            .tree_ref()
+            .iter()
+            .filter(|(_, s)| s.subscription_state == Into::<i32>::into(PbSubscriptionState::Init))
+            .map(|(_, s)| s.clone())
+            .collect_vec();
         let user_core = &mut core.user;
-        let key = (
-            subscription.database_id,
-            subscription.schema_id,
-            subscription.name.clone(),
-        );
-        assert!(
-            !database_core.subscriptions.contains_key(&subscription.id),
-            "subscription must be in creating procedure"
-        );
+        for s in &remove_subscriptions {
+            subscriptions.remove(s.id);
+        }
+        commit_meta!(self, subscriptions)?;
+        for subscription in remove_subscriptions {
+            let key = (
+                subscription.database_id,
+                subscription.schema_id,
+                subscription.name.clone(),
+            );
+            assert!(
+                !database_core.subscriptions.contains_key(&subscription.id),
+                "subscription must be in creating procedure"
+            );
 
-        database_core.unmark_creating(&key);
-        database_core.unmark_creating_streaming_job(subscription.id);
-        for &dependent_relation_id in &subscription.dependent_relations {
-            database_core.decrease_ref_count(dependent_relation_id);
+            database_core.unmark_creating(&key);
+            database_core.unmark_creating_streaming_job(subscription.id);
+            database_core.decrease_relation_ref_count(subscription.dependent_table_id);
+            user_core.decrease_ref(subscription.owner);
         }
-        user_core.decrease_ref(subscription.owner);
+        Ok(())
     }
 
     /// This is used for `ALTER TABLE ADD/DROP COLUMN`.
@@ -3539,6 +3565,10 @@ impl CatalogManager {
         self.core.lock().await.database.list_databases()
     }
 
+    pub async fn list_schemas(&self) -> Vec<Schema> {
+        self.core.lock().await.database.list_schemas()
+    }
+
     pub async fn list_tables(&self) -> Vec<Table> {
         self.core.lock().await.database.list_tables()
     }
@@ -3628,6 +3658,10 @@ impl CatalogManager {
             .list_dml_table_ids(schema_id)
     }
 
+    pub async fn list_view_ids(&self, schema_id: SchemaId) -> Vec<TableId> {
+        self.core.lock().await.database.list_view_ids(schema_id)
+    }
+
     pub async fn list_sources(&self) -> Vec<Source> {
         self.core.lock().await.database.list_sources()
     }
@@ -3718,6 +3752,13 @@ impl CatalogManager {
             }
         }
 
+        for subscription in core.subscriptions.values() {
+            dependencies.push(PbObjectDependencies {
+                object_id: subscription.id,
+                referenced_object_id: subscription.dependent_table_id,
+            });
+        }
+
         dependencies
     }
 
@@ -3761,6 +3802,19 @@ impl CatalogManager {
         tables
     }
 
+    pub async fn get_subscription_by_id(
+        &self,
+        subscription_id: SubscriptionId,
+    ) -> MetaResult<Subscription> {
+        let guard = self.core.lock().await;
+        let subscription = guard
+            .database
+            .subscriptions
+            .get(&subscription_id)
+            .ok_or_else(|| MetaError::catalog_id_not_found("subscription", subscription_id))?;
+        Ok(subscription.clone())
+    }
+
     pub async fn get_created_table_ids(&self) -> Vec<u32> {
         let guard = self.core.lock().await;
         guard
@@ -3877,7 +3931,6 @@ impl CatalogManager {
         Ok(())
     }
 
-    #[cfg(test)]
     pub async fn list_users(&self) -> Vec<UserInfo> {
         self.core.lock().await.user.list_users()
     }
@@ -3947,7 +4000,7 @@ impl CatalogManager {
         core.user_info
             .get(&id)
             .cloned()
-            .ok_or_else(|| anyhow!("User {} not found", id).into())
+            .ok_or_else(|| MetaError::catalog_id_not_found("user", id))
     }
 
     pub async fn drop_user(&self, id: UserId) -> MetaResult<NotificationVersion> {
@@ -3966,7 +4019,7 @@ impl CatalogManager {
                 id
             )));
         }
-        if user_core.catalog_create_ref_count.get(&id).is_some() {
+        if user_core.catalog_create_ref_count.contains_key(&id) {
             return Err(MetaError::permission_denied(format!(
                 "User {} cannot be dropped because some objects depend on it",
                 user.name
@@ -4077,11 +4130,11 @@ impl CatalogManager {
         let grantor_info = users
             .get(&grantor)
             .cloned()
-            .ok_or_else(|| anyhow!("User {} does not exist", &grantor))?;
+            .ok_or_else(|| MetaError::catalog_id_not_found("user", grantor))?;
         for user_id in user_ids {
             let mut user = users
                 .get_mut(*user_id)
-                .ok_or_else(|| anyhow!("User {} does not exist", user_id))?;
+                .ok_or_else(|| MetaError::catalog_id_not_found("user", user_id))?;
 
             if user.is_super {
                 return Err(MetaError::permission_denied(format!(
@@ -4204,7 +4257,7 @@ impl CatalogManager {
         // check revoke permission
         let revoke_by = users
             .get(&revoke_by)
-            .ok_or_else(|| anyhow!("User {} does not exist", &revoke_by))?;
+            .ok_or_else(|| MetaError::catalog_id_not_found("user", revoke_by))?;
         let same_user = granted_by == revoke_by.id;
         if !revoke_by.is_super {
             for privilege in revoke_grant_privileges {
@@ -4239,7 +4292,7 @@ impl CatalogManager {
             let user = users
                 .get(user_id)
                 .cloned()
-                .ok_or_else(|| anyhow!("User {} does not exist", user_id))?;
+                .ok_or_else(|| MetaError::catalog_id_not_found("user", user_id))?;
             if user.is_super {
                 return Err(MetaError::permission_denied(format!(
                     "Cannot revoke privilege from supper user {}",
diff --git a/src/meta/src/manager/catalog/user.rs b/src/meta/src/manager/catalog/user.rs
index 0da37291a1634..c4ed4c30fc2ba 100644
--- a/src/meta/src/manager/catalog/user.rs
+++ b/src/meta/src/manager/catalog/user.rs
@@ -67,6 +67,13 @@ impl UserManager {
                     .map(|table| table.owner),
             )
             .chain(database.views.values().map(|view| view.owner))
+            .chain(database.functions.values().map(|function| function.owner))
+            .chain(
+                database
+                    .connections
+                    .values()
+                    .map(|connection| connection.owner),
+            )
             .for_each(|owner_id| user_manager.increase_ref(owner_id));
 
         Ok(user_manager)
@@ -148,7 +155,7 @@ mod tests {
 
     use super::*;
     use crate::manager::{commit_meta, CatalogManager};
-    use crate::model::{BTreeMapTransaction, ValTransaction};
+    use crate::model::BTreeMapTransaction;
     use crate::storage::Transaction;
 
     fn make_test_user(id: u32, name: &str) -> UserInfo {
diff --git a/src/meta/src/manager/catalog/utils.rs b/src/meta/src/manager/catalog/utils.rs
index 6c73e6eeebb39..4d4257ec5a7e5 100644
--- a/src/meta/src/manager/catalog/utils.rs
+++ b/src/meta/src/manager/catalog/utils.rs
@@ -22,8 +22,7 @@ pub fn refcnt_inc_connection(
 ) -> anyhow::Result<()> {
     if let Some(connection_id) = connection_id {
         if let Some(_conn) = database_mgr.get_connection(connection_id) {
-            // TODO(weili): wait for yezizp to refactor ref cnt
-            database_mgr.increase_ref_count(connection_id);
+            database_mgr.increase_connection_ref_count(connection_id);
         } else {
             bail!("connection {} not found.", connection_id);
         }
@@ -36,7 +35,6 @@ pub fn refcnt_dec_connection(
     connection_id: Option<ConnectionId>,
 ) {
     if let Some(connection_id) = connection_id {
-        // TODO: wait for yezizp to refactor ref cnt
-        database_mgr.decrease_ref_count(connection_id);
+        database_mgr.decrease_connection_ref_count(connection_id);
     }
 }
diff --git a/src/meta/src/manager/cluster.rs b/src/meta/src/manager/cluster.rs
index 0aeb610b7ce91..876050c36ae6c 100644
--- a/src/meta/src/manager/cluster.rs
+++ b/src/meta/src/manager/cluster.rs
@@ -38,7 +38,9 @@ use tokio::sync::{RwLock, RwLockReadGuard};
 use tokio::task::JoinHandle;
 
 use crate::manager::{IdCategory, LocalNotification, MetaSrvEnv};
-use crate::model::{MetadataModel, ValTransaction, VarTransaction, Worker, INVALID_EXPIRE_AT};
+use crate::model::{
+    InMemValTransaction, MetadataModel, ValTransaction, VarTransaction, Worker, INVALID_EXPIRE_AT,
+};
 use crate::storage::{MetaStore, Transaction};
 use crate::{MetaError, MetaResult};
 
@@ -566,7 +568,7 @@ impl ClusterManagerCore {
     pub const MAX_WORKER_REUSABLE_ID_COUNT: usize = 1 << Self::MAX_WORKER_REUSABLE_ID_BITS;
 
     async fn new(env: MetaSrvEnv) -> MetaResult<Self> {
-        let meta_store = env.meta_store().as_kv();
+        let meta_store = env.meta_store_ref().as_kv();
         let mut workers = Worker::list(meta_store).await?;
 
         let used_transactional_ids: HashSet<_> = workers
diff --git a/src/meta/src/manager/diagnose.rs b/src/meta/src/manager/diagnose.rs
index fbc04f9324002..06c76c47c5daa 100644
--- a/src/meta/src/manager/diagnose.rs
+++ b/src/meta/src/manager/diagnose.rs
@@ -20,6 +20,7 @@ use std::sync::Arc;
 use itertools::Itertools;
 use prometheus_http_query::response::Data::Vector;
 use risingwave_common::types::Timestamptz;
+use risingwave_common::util::StackTraceResponseExt;
 use risingwave_pb::common::WorkerType;
 use risingwave_pb::hummock::Level;
 use risingwave_pb::meta::event_log::Event;
@@ -664,53 +665,18 @@ impl DiagnoseCommand {
             return;
         };
 
-        let mut all = Default::default();
-
-        fn merge(a: &mut StackTraceResponse, b: StackTraceResponse) {
-            a.actor_traces.extend(b.actor_traces);
-            a.rpc_traces.extend(b.rpc_traces);
-            a.compaction_task_traces.extend(b.compaction_task_traces);
-            a.inflight_barrier_traces.extend(b.inflight_barrier_traces);
-        }
+        let mut all = StackTraceResponse::default();
 
         let compute_clients = ComputeClientPool::default();
         for worker_node in &worker_nodes {
             if let Ok(client) = compute_clients.get(worker_node).await
                 && let Ok(result) = client.stack_trace().await
             {
-                merge(&mut all, result);
+                all.merge_other(result);
             }
         }
 
-        if !all.actor_traces.is_empty() {
-            let _ = writeln!(s, "--- Actor Traces ---");
-            for (actor_id, trace) in &all.actor_traces {
-                let _ = writeln!(s, ">> Actor {}", *actor_id);
-                let _ = writeln!(s, "{trace}");
-            }
-        }
-        if !all.rpc_traces.is_empty() {
-            let _ = writeln!(s, "--- RPC Traces ---");
-            for (name, trace) in &all.rpc_traces {
-                let _ = writeln!(s, ">> RPC {name}");
-                let _ = writeln!(s, "{trace}");
-            }
-        }
-        if !all.compaction_task_traces.is_empty() {
-            let _ = writeln!(s, "--- Compactor Traces ---");
-            for (name, trace) in &all.compaction_task_traces {
-                let _ = writeln!(s, ">> Compaction Task {name}");
-                let _ = writeln!(s, "{trace}");
-            }
-        }
-
-        if !all.inflight_barrier_traces.is_empty() {
-            let _ = writeln!(s, "--- Inflight Barrier Traces ---");
-            for (name, trace) in &all.inflight_barrier_traces {
-                let _ = writeln!(s, ">> Barrier {name}");
-                let _ = writeln!(s, "{trace}");
-            }
-        }
+        write!(s, "{}", all.output()).unwrap();
     }
 }
 
diff --git a/src/meta/src/manager/env.rs b/src/meta/src/manager/env.rs
index ca99f5de03dea..b623e441c0c22 100644
--- a/src/meta/src/manager/env.rs
+++ b/src/meta/src/manager/env.rs
@@ -188,7 +188,11 @@ pub struct MetaOpts {
     /// Interval of reporting the number of nodes in the cluster.
     pub node_num_monitor_interval_sec: u64,
 
-    /// The Prometheus endpoint for dashboard service.
+    /// The Prometheus endpoint for Meta Dashboard Service.
+    /// The Dashboard service uses this in the following ways:
+    /// 1. Query Prometheus for relevant metrics to find Stream Graph Bottleneck, and display it.
+    /// 2. Provide cluster diagnostics, at `/api/monitor/diagnose` to troubleshoot cluster.
+    ///    These are just examples which show how the Meta Dashboard Service queries Prometheus.
     pub prometheus_endpoint: Option<String>,
 
     /// The additional selector used when querying Prometheus.
@@ -243,12 +247,12 @@ pub struct MetaOpts {
 
     /// hybird compaction group config
     ///
-    /// `hybird_partition_vnode_count` determines the granularity of vnodes in the hybrid compaction group for SST alignment.
-    /// When `hybird_partition_vnode_count` > 0, in hybrid compaction group
+    /// `hybrid_partition_vnode_count` determines the granularity of vnodes in the hybrid compaction group for SST alignment.
+    /// When `hybrid_partition_vnode_count` > 0, in hybrid compaction group
     /// - Tables with high write throughput will be split at vnode granularity
     /// - Tables with high size tables will be split by table granularity
-    /// When `hybird_partition_vnode_count` = 0,no longer be special alignment operations for the hybird compaction group
-    pub hybird_partition_vnode_count: u32,
+    ///   When `hybrid_partition_vnode_count` = 0,no longer be special alignment operations for the hybird compaction group
+    pub hybrid_partition_node_count: u32,
 
     pub event_log_enabled: bool,
     pub event_log_channel_max_size: u32,
@@ -267,6 +271,18 @@ pub struct MetaOpts {
     pub enable_check_task_level_overlap: bool,
     pub enable_dropped_column_reclaim: bool,
     pub object_store_config: ObjectStoreConfig,
+
+    /// The maximum number of trivial move tasks to be picked in a single loop
+    pub max_trivial_move_task_count_per_loop: usize,
+
+    /// The maximum number of times to probe for `PullTaskEvent`
+    pub max_get_task_probe_times: usize,
+
+    pub compact_task_table_size_partition_threshold_low: u64,
+    pub compact_task_table_size_partition_threshold_high: u64,
+
+    // The private key for the secret store, used when the secret is stored in the meta.
+    pub secret_store_private_key: Vec<u8>,
 }
 
 impl MetaOpts {
@@ -305,6 +321,8 @@ impl MetaOpts {
             periodic_split_compact_group_interval_sec: 60,
             split_group_size_limit: 5 * 1024 * 1024 * 1024,
             min_table_split_size: 2 * 1024 * 1024 * 1024,
+            compact_task_table_size_partition_threshold_low: 128 * 1024 * 1024,
+            compact_task_table_size_partition_threshold_high: 512 * 1024 * 1024,
             table_write_throughput_threshold: 128 * 1024 * 1024,
             min_table_split_write_throughput: 64 * 1024 * 1024,
             do_not_config_object_storage_lifecycle: true,
@@ -313,7 +331,7 @@ impl MetaOpts {
             compaction_task_max_progress_interval_secs: 1,
             compaction_config: None,
             cut_table_size_limit: 1024 * 1024 * 1024,
-            hybird_partition_vnode_count: 4,
+            hybrid_partition_node_count: 4,
             event_log_enabled: false,
             event_log_channel_max_size: 1,
             advertise_addr: "".to_string(),
@@ -323,6 +341,9 @@ impl MetaOpts {
             enable_check_task_level_overlap: true,
             enable_dropped_column_reclaim: false,
             object_store_config: ObjectStoreConfig::default(),
+            max_trivial_move_task_count_per_loop: 256,
+            max_get_task_probe_times: 5,
+            secret_store_private_key: "demo-secret-private-key".as_bytes().to_vec(),
         }
     }
 }
@@ -441,11 +462,11 @@ impl MetaSrvEnv {
         Ok(env)
     }
 
-    pub fn meta_store_ref(&self) -> MetaStoreImpl {
+    pub fn meta_store(&self) -> MetaStoreImpl {
         self.meta_store_impl.clone()
     }
 
-    pub fn meta_store(&self) -> &MetaStoreImpl {
+    pub fn meta_store_ref(&self) -> &MetaStoreImpl {
         &self.meta_store_impl
     }
 
diff --git a/src/meta/src/manager/id.rs b/src/meta/src/manager/id.rs
index 7fbde6d655e83..023483116fdc8 100644
--- a/src/meta/src/manager/id.rs
+++ b/src/meta/src/manager/id.rs
@@ -137,6 +137,8 @@ pub mod IdCategory {
     pub const CompactionGroup: IdCategoryType = 15;
     pub const Function: IdCategoryType = 16;
     pub const Connection: IdCategoryType = 17;
+
+    pub const Secret: IdCategoryType = 18;
 }
 
 pub type IdGeneratorManagerRef = Arc<IdGeneratorManager>;
@@ -160,6 +162,7 @@ pub struct IdGeneratorManager {
     parallel_unit: Arc<StoredIdGenerator>,
     compaction_group: Arc<StoredIdGenerator>,
     connection: Arc<StoredIdGenerator>,
+    secret: Arc<StoredIdGenerator>,
 }
 
 impl IdGeneratorManager {
@@ -209,6 +212,7 @@ impl IdGeneratorManager {
             connection: Arc::new(
                 StoredIdGenerator::new(meta_store.clone(), "connection", None).await,
             ),
+            secret: Arc::new(StoredIdGenerator::new(meta_store.clone(), "secret", None).await),
         }
     }
 
@@ -230,6 +234,7 @@ impl IdGeneratorManager {
             IdCategory::HummockCompactionTask => &self.hummock_compaction_task,
             IdCategory::CompactionGroup => &self.compaction_group,
             IdCategory::Connection => &self.connection,
+            IdCategory::Secret => &self.secret,
             _ => unreachable!(),
         }
     }
diff --git a/src/meta/src/manager/metadata.rs b/src/meta/src/manager/metadata.rs
index 876f5c3365d36..241a47941755b 100644
--- a/src/meta/src/manager/metadata.rs
+++ b/src/meta/src/manager/metadata.rs
@@ -24,7 +24,8 @@ use risingwave_pb::common::worker_node::{PbResource, State};
 use risingwave_pb::common::{HostAddress, PbWorkerNode, PbWorkerType, WorkerNode, WorkerType};
 use risingwave_pb::meta::add_worker_node_request::Property as AddNodeProperty;
 use risingwave_pb::meta::table_fragments::{ActorStatus, Fragment, PbFragment};
-use risingwave_pb::stream_plan::{PbDispatchStrategy, PbStreamActor, StreamActor};
+use risingwave_pb::stream_plan::{PbDispatchStrategy, StreamActor};
+use risingwave_pb::stream_service::BuildActorInfo;
 use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver};
 use tokio::time::sleep;
 use tracing::warn;
@@ -37,7 +38,7 @@ use crate::manager::{
     StreamingClusterInfo, WorkerId,
 };
 use crate::model::{ActorId, FragmentId, MetadataModel, TableFragments, TableParallelism};
-use crate::stream::SplitAssignment;
+use crate::stream::{to_build_actor_info, SplitAssignment};
 use crate::telemetry::MetaTelemetryJobDesc;
 use crate::MetaResult;
 
@@ -63,6 +64,7 @@ pub struct MetadataManagerV2 {
 #[derive(Debug)]
 pub(crate) enum ActiveStreamingWorkerChange {
     Add(WorkerNode),
+    #[expect(dead_code)]
     Remove(WorkerNode),
     Update(WorkerNode),
 }
@@ -692,19 +694,26 @@ impl MetadataManager {
     pub async fn all_node_actors(
         &self,
         include_inactive: bool,
-    ) -> MetaResult<HashMap<WorkerId, Vec<PbStreamActor>>> {
+    ) -> MetaResult<HashMap<WorkerId, Vec<BuildActorInfo>>> {
+        let subscriptions = self.get_mv_depended_subscriptions().await?;
         match &self {
-            MetadataManager::V1(mgr) => {
-                Ok(mgr.fragment_manager.all_node_actors(include_inactive).await)
-            }
+            MetadataManager::V1(mgr) => Ok(mgr
+                .fragment_manager
+                .all_node_actors(include_inactive, &subscriptions)
+                .await),
             MetadataManager::V2(mgr) => {
                 let table_fragments = mgr.catalog_controller.table_fragments().await?;
                 let mut actor_maps = HashMap::new();
                 for (_, fragments) in table_fragments {
                     let tf = TableFragments::from_protobuf(fragments);
-                    for (node_id, actor_ids) in tf.worker_actors(include_inactive) {
-                        let node_actor_ids = actor_maps.entry(node_id).or_insert_with(Vec::new);
-                        node_actor_ids.extend(actor_ids);
+                    let table_id = tf.table_id();
+                    for (node_id, actors) in tf.worker_actors(include_inactive) {
+                        let node_actors = actor_maps.entry(node_id).or_insert_with(Vec::new);
+                        node_actors.extend(
+                            actors
+                                .into_iter()
+                                .map(|actor| to_build_actor_info(actor, &subscriptions, table_id)),
+                        )
                     }
                 }
                 Ok(actor_maps)
@@ -812,4 +821,12 @@ impl MetadataManager {
             }
         }
     }
+
+    #[expect(clippy::unused_async)]
+    pub async fn get_mv_depended_subscriptions(
+        &self,
+    ) -> MetaResult<HashMap<TableId, HashMap<u32, u64>>> {
+        // TODO(subscription): support the correct logic when supporting L0 log store subscriptions
+        Ok(HashMap::new())
+    }
 }
diff --git a/src/meta/src/manager/notification.rs b/src/meta/src/manager/notification.rs
index 0ce47608cdfd2..dcf537ad6c7e7 100644
--- a/src/meta/src/manager/notification.rs
+++ b/src/meta/src/manager/notification.rs
@@ -101,7 +101,6 @@ impl NotificationManager {
                     info: Some(task.info),
                     version: task.version.unwrap_or_default(),
                 };
-
                 core.lock().await.notify(task.target, response);
             }
         });
diff --git a/src/meta/src/manager/session_params.rs b/src/meta/src/manager/session_params.rs
index 662eae162a92d..e1f0afb21448c 100644
--- a/src/meta/src/manager/session_params.rs
+++ b/src/meta/src/manager/session_params.rs
@@ -22,7 +22,7 @@ use thiserror_ext::AsReport;
 use tokio::sync::RwLock;
 use tracing::info;
 
-use crate::model::{ValTransaction, VarTransaction};
+use crate::model::{InMemValTransaction, ValTransaction, VarTransaction};
 use crate::storage::{MetaStore, MetaStoreRef, Snapshot, Transaction};
 use crate::MetaResult;
 
diff --git a/src/meta/src/manager/streaming_job.rs b/src/meta/src/manager/streaming_job.rs
index a29e6e923de2b..a9a159e6a4535 100644
--- a/src/meta/src/manager/streaming_job.rs
+++ b/src/meta/src/manager/streaming_job.rs
@@ -15,7 +15,7 @@
 use risingwave_common::catalog::TableVersionId;
 use risingwave_common::current_cluster_version;
 use risingwave_common::util::epoch::Epoch;
-use risingwave_pb::catalog::{CreateType, Index, PbSource, Sink, Subscription, Table};
+use risingwave_pb::catalog::{CreateType, Index, PbSource, Sink, Table};
 use risingwave_pb::ddl_service::TableJobType;
 use strum::EnumDiscriminants;
 
@@ -26,7 +26,6 @@ use crate::model::FragmentId;
 #[derive(Debug, Clone, EnumDiscriminants)]
 pub enum StreamingJob {
     MaterializedView(Table),
-    Subscription(Subscription),
     Sink(Sink, Option<(Table, Option<PbSource>)>),
     Table(Option<PbSource>, Table, TableJobType),
     Index(Index, Table),
@@ -37,7 +36,6 @@ pub enum StreamingJob {
 pub enum DdlType {
     MaterializedView,
     Sink,
-    Subscription,
     Table(TableJobType),
     Index,
     Source,
@@ -51,7 +49,6 @@ impl From<&StreamingJob> for DdlType {
             StreamingJob::Table(_, _, ty) => DdlType::Table(*ty),
             StreamingJob::Index(_, _) => DdlType::Index,
             StreamingJob::Source(_) => DdlType::Source,
-            StreamingJob::Subscription(_) => DdlType::Subscription,
         }
     }
 }
@@ -94,10 +91,6 @@ impl StreamingJob {
                 source.created_at_epoch = created_at_epoch;
                 source.created_at_cluster_version = created_at_cluster_version;
             }
-            StreamingJob::Subscription(subscription) => {
-                subscription.created_at_epoch = created_at_epoch;
-                subscription.created_at_cluster_version = created_at_cluster_version;
-            }
         }
     }
 
@@ -132,10 +125,6 @@ impl StreamingJob {
                 source.initialized_at_epoch = initialized_at_epoch;
                 source.initialized_at_cluster_version = initialized_at_cluster_version;
             }
-            StreamingJob::Subscription(subscription) => {
-                subscription.initialized_at_epoch = initialized_at_epoch;
-                subscription.initialized_at_cluster_version = initialized_at_cluster_version;
-            }
         }
     }
 }
@@ -154,9 +143,6 @@ impl StreamingJob {
             StreamingJob::Source(src) => {
                 src.id = id;
             }
-            StreamingJob::Subscription(subscription) => {
-                subscription.id = id;
-            }
         }
     }
 
@@ -166,7 +152,7 @@ impl StreamingJob {
             Self::MaterializedView(table) | Self::Index(_, table) | Self::Table(_, table, ..) => {
                 table.fragment_id = id;
             }
-            Self::Sink(_, _) | Self::Source(_) | Self::Subscription(_) => {}
+            Self::Sink(_, _) | Self::Source(_) => {}
         }
     }
 
@@ -176,10 +162,7 @@ impl StreamingJob {
             Self::Table(_, table, ..) => {
                 table.dml_fragment_id = id;
             }
-            Self::MaterializedView(_)
-            | Self::Index(_, _)
-            | Self::Sink(_, _)
-            | Self::Subscription(_) => {}
+            Self::MaterializedView(_) | Self::Index(_, _) | Self::Sink(_, _) => {}
             Self::Source(_) => {}
         }
     }
@@ -191,7 +174,6 @@ impl StreamingJob {
             Self::Table(_, table, ..) => table.id,
             Self::Index(index, _) => index.id,
             Self::Source(source) => source.id,
-            Self::Subscription(subscription) => subscription.id,
         }
     }
 
@@ -202,7 +184,6 @@ impl StreamingJob {
             Self::Table(_, table, ..) => Some(table.id),
             Self::Index(_, table) => Some(table.id),
             Self::Source(_) => None,
-            Self::Subscription(_) => None,
         }
     }
 
@@ -212,7 +193,7 @@ impl StreamingJob {
             Self::MaterializedView(table) | Self::Index(_, table) | Self::Table(_, table, ..) => {
                 Some(table)
             }
-            Self::Sink(_, _) | Self::Source(_) | Self::Subscription(_) => None,
+            Self::Sink(_, _) | Self::Source(_) => None,
         }
     }
 
@@ -223,7 +204,6 @@ impl StreamingJob {
             Self::Table(_, table, ..) => table.schema_id,
             Self::Index(index, _) => index.schema_id,
             Self::Source(source) => source.schema_id,
-            Self::Subscription(subscription) => subscription.schema_id,
         }
     }
 
@@ -234,7 +214,6 @@ impl StreamingJob {
             Self::Table(_, table, ..) => table.database_id,
             Self::Index(index, _) => index.database_id,
             Self::Source(source) => source.database_id,
-            Self::Subscription(subscription) => subscription.database_id,
         }
     }
 
@@ -245,7 +224,6 @@ impl StreamingJob {
             Self::Table(_, table, ..) => table.name.clone(),
             Self::Index(index, _) => index.name.clone(),
             Self::Source(source) => source.name.clone(),
-            Self::Subscription(subscription) => subscription.name.clone(),
         }
     }
 
@@ -256,7 +234,6 @@ impl StreamingJob {
             StreamingJob::Table(_, table, ..) => table.owner,
             StreamingJob::Index(index, _) => index.owner,
             StreamingJob::Source(source) => source.owner,
-            StreamingJob::Subscription(subscription) => subscription.owner,
         }
     }
 
@@ -267,7 +244,6 @@ impl StreamingJob {
             Self::Index(_, table) => table.definition.clone(),
             Self::Sink(sink, _) => sink.definition.clone(),
             Self::Source(source) => source.definition.clone(),
-            Self::Subscription(subscription) => subscription.definition.clone(),
         }
     }
 
@@ -306,7 +282,6 @@ impl StreamingJob {
                 vec![]
             }
             StreamingJob::Source(_) => vec![],
-            Self::Subscription(subscription) => subscription.dependent_relations.clone(),
         }
     }
 
diff --git a/src/meta/src/manager/system_param/mod.rs b/src/meta/src/manager/system_param/mod.rs
index bdd54d3ed7b55..0030d3c2d8f88 100644
--- a/src/meta/src/manager/system_param/mod.rs
+++ b/src/meta/src/manager/system_param/mod.rs
@@ -32,7 +32,7 @@ use tracing::info;
 
 use self::model::SystemParamsModel;
 use super::NotificationManagerRef;
-use crate::model::{ValTransaction, VarTransaction};
+use crate::model::{InMemValTransaction, ValTransaction, VarTransaction};
 use crate::storage::{MetaStore, MetaStoreRef, Transaction};
 use crate::{MetaError, MetaResult};
 
diff --git a/src/meta/src/model/catalog.rs b/src/meta/src/model/catalog.rs
index 8f762255d60b7..8d89080ae2462 100644
--- a/src/meta/src/model/catalog.rs
+++ b/src/meta/src/model/catalog.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use risingwave_pb::catalog::{
-    Connection, Database, Function, Index, Schema, Sink, Source, Subscription, Table, View,
+    Connection, Database, Function, Index, Schema, Secret, Sink, Source, Subscription, Table, View,
 };
 
 use crate::model::{MetadataModel, MetadataModelResult};
@@ -38,6 +38,8 @@ const CATALOG_SCHEMA_CF_NAME: &str = "cf/catalog_schema";
 const CATALOG_DATABASE_CF_NAME: &str = "cf/catalog_database";
 /// Column family name for database catalog.
 const CATALOG_SUBSCRIPTION_CF_NAME: &str = "cf/catalog_subscription";
+/// Column family name for secret catalog.
+const CATALOG_SECRET_CF_NAME: &str = "cf/catalog_secret";
 
 macro_rules! impl_model_for_catalog {
     ($name:ident, $cf:ident, $key_ty:ty, $key_fn:ident) => {
@@ -74,6 +76,7 @@ impl_model_for_catalog!(Table, CATALOG_TABLE_CF_NAME, u32, get_id);
 impl_model_for_catalog!(Schema, CATALOG_SCHEMA_CF_NAME, u32, get_id);
 impl_model_for_catalog!(Database, CATALOG_DATABASE_CF_NAME, u32, get_id);
 impl_model_for_catalog!(Subscription, CATALOG_SUBSCRIPTION_CF_NAME, u32, get_id);
+impl_model_for_catalog!(Secret, CATALOG_SECRET_CF_NAME, u32, get_id);
 
 #[cfg(test)]
 mod tests {
@@ -93,7 +96,7 @@ mod tests {
     #[tokio::test]
     async fn test_database() -> MetadataModelResult<()> {
         let env = MetaSrvEnv::for_test().await;
-        let store = env.meta_store().as_kv();
+        let store = env.meta_store_ref().as_kv();
         let databases = Database::list(store).await?;
         assert!(databases.is_empty());
         assert!(Database::select(store, &0).await.unwrap().is_none());
diff --git a/src/meta/src/model/mod.rs b/src/meta/src/model/mod.rs
index 8b53d87160efa..e87251ee6d413 100644
--- a/src/meta/src/model/mod.rs
+++ b/src/meta/src/model/mod.rs
@@ -23,7 +23,6 @@ mod user;
 use std::collections::btree_map::{Entry, VacantEntry};
 use std::collections::BTreeMap;
 use std::fmt::Debug;
-use std::marker::PhantomData;
 use std::ops::{Deref, DerefMut};
 
 use async_trait::async_trait;
@@ -34,7 +33,6 @@ pub use notification::*;
 use prost::Message;
 pub use stream::*;
 
-use crate::hummock::model::ext::Transaction as TransactionV2;
 use crate::storage::{MetaStore, MetaStoreError, Snapshot, Transaction};
 
 /// A global, unique identifier of an actor
@@ -179,6 +177,7 @@ macro_rules! for_all_metadata_models {
             { risingwave_pb::user::UserInfo },
             { risingwave_pb::catalog::Function },
             { risingwave_pb::catalog::Connection },
+            { risingwave_pb::catalog::Secret },
             // These items need not be included in a meta snapshot.
             { crate::model::cluster::Worker },
             { risingwave_pb::hummock::CompactTaskAssignment },
@@ -222,20 +221,16 @@ where
     }
 }
 
-/// Trait that wraps a local memory value and applies the change to the local memory value on
-/// `commit` or leaves the local memory value untouched on `abort`.
-pub trait ValTransaction: Sized {
-    type TXN;
+pub trait InMemValTransaction: Sized {
     /// Commit the change to local memory value
     fn commit(self);
+}
 
+/// Trait that wraps a local memory value and applies the change to the local memory value on
+/// `commit` or leaves the local memory value untouched on `abort`.
+pub trait ValTransaction<TXN>: InMemValTransaction {
     /// Apply the change (upsert or delete) to `txn`
-    async fn apply_to_txn(&self, txn: &mut Self::TXN) -> MetadataModelResult<()>;
-
-    /// Abort the `VarTransaction` and leave the local memory value untouched
-    fn abort(self) {
-        drop(self);
-    }
+    async fn apply_to_txn(&self, txn: &mut TXN) -> MetadataModelResult<()>;
 }
 
 /// Transaction wrapper for a variable.
@@ -244,28 +239,27 @@ pub trait ValTransaction: Sized {
 /// When `commit` is called, the change to `new_value` will be applied to the `orig_value_ref`
 /// When `abort` is called, the `VarTransaction` is dropped and the local memory value is
 /// untouched.
-pub struct VarTransaction<'a, TXN, T: Transactional<TXN>> {
+pub struct VarTransaction<'a, T> {
     orig_value_ref: &'a mut T,
     new_value: Option<T>,
-    _phantom: PhantomData<TXN>,
 }
 
-impl<'a, TXN, T> VarTransaction<'a, TXN, T>
-where
-    T: Transactional<TXN>,
-{
+impl<'a, T> VarTransaction<'a, T> {
     /// Create a `VarTransaction` that wraps a raw variable
-    pub fn new(val_ref: &'a mut T) -> VarTransaction<'a, TXN, T> {
+    pub fn new(val_ref: &'a mut T) -> VarTransaction<'a, T> {
         VarTransaction {
             // lazy initialization
             new_value: None,
             orig_value_ref: val_ref,
-            _phantom: PhantomData,
         }
     }
+
+    pub fn has_new_value(&self) -> bool {
+        self.new_value.is_some()
+    }
 }
 
-impl<'a, TXN, T: Transactional<TXN>> Deref for VarTransaction<'a, TXN, T> {
+impl<'a, T> Deref for VarTransaction<'a, T> {
     type Target = T;
 
     fn deref(&self) -> &Self::Target {
@@ -276,10 +270,7 @@ impl<'a, TXN, T: Transactional<TXN>> Deref for VarTransaction<'a, TXN, T> {
     }
 }
 
-impl<'a, TXN, T> DerefMut for VarTransaction<'a, TXN, T>
-where
-    T: Clone + Transactional<TXN>,
-{
+impl<'a, T: Clone> DerefMut for VarTransaction<'a, T> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         if self.new_value.is_none() {
             self.new_value.replace(self.orig_value_ref.clone());
@@ -288,19 +279,22 @@ where
     }
 }
 
-impl<'a, TXN, T> ValTransaction for VarTransaction<'a, TXN, T>
+impl<'a, T> InMemValTransaction for VarTransaction<'a, T>
 where
-    T: Transactional<TXN> + PartialEq,
+    T: PartialEq,
 {
-    type TXN = TXN;
-
     fn commit(self) {
         if let Some(new_value) = self.new_value {
             *self.orig_value_ref = new_value;
         }
     }
+}
 
-    async fn apply_to_txn(&self, txn: &mut Self::TXN) -> MetadataModelResult<()> {
+impl<'a, TXN, T> ValTransaction<TXN> for VarTransaction<'a, T>
+where
+    T: Transactional<TXN> + PartialEq,
+{
+    async fn apply_to_txn(&self, txn: &mut TXN) -> MetadataModelResult<()> {
         if let Some(new_value) = &self.new_value {
             // Apply the change to `txn` only when the value is modified
             if *self.orig_value_ref != *new_value {
@@ -423,27 +417,25 @@ enum BTreeMapOp<V> {
 /// are stored in `staging`. On `commit`, it will apply the changes stored in `staging` to the in
 /// memory btree map. When serve `get` and `get_mut`, it merges the value stored in `staging` and
 /// `tree_ref`.
-pub struct BTreeMapTransaction<'a, K: Ord, V, TXN = Transaction> {
+pub struct BTreeMapTransaction<'a, K: Ord, V> {
     /// A reference to the original `BTreeMap`. All access to this field should be immutable,
     /// except when we commit the staging changes to the original map.
     tree_ref: &'a mut BTreeMap<K, V>,
     /// Store all the staging changes that will be applied to the original map on commit
     staging: BTreeMap<K, BTreeMapOp<V>>,
-    _phantom: PhantomData<TXN>,
 }
 
-impl<'a, K: Ord + Debug, V: Clone, TXN> BTreeMapTransaction<'a, K, V, TXN> {
-    pub fn new(tree_ref: &'a mut BTreeMap<K, V>) -> BTreeMapTransaction<'a, K, V, TXN> {
+impl<'a, K: Ord + Debug, V: Clone> BTreeMapTransaction<'a, K, V> {
+    pub fn new(tree_ref: &'a mut BTreeMap<K, V>) -> BTreeMapTransaction<'a, K, V> {
         Self {
             tree_ref,
             staging: BTreeMap::default(),
-            _phantom: PhantomData,
         }
     }
 
     /// Start a `BTreeMapEntryTransaction` when the `key` exists
     #[allow(dead_code)]
-    pub fn new_entry_txn(&mut self, key: K) -> Option<BTreeMapEntryTransaction<'_, K, V, TXN>> {
+    pub fn new_entry_txn(&mut self, key: K) -> Option<BTreeMapEntryTransaction<'_, K, V>> {
         BTreeMapEntryTransaction::new(self.tree_ref, key, None)
     }
 
@@ -454,17 +446,13 @@ impl<'a, K: Ord + Debug, V: Clone, TXN> BTreeMapTransaction<'a, K, V, TXN> {
         &mut self,
         key: K,
         default_val: V,
-    ) -> BTreeMapEntryTransaction<'_, K, V, TXN> {
+    ) -> BTreeMapEntryTransaction<'_, K, V> {
         BTreeMapEntryTransaction::new(self.tree_ref, key, Some(default_val))
             .expect("default value is provided and should return `Some`")
     }
 
     /// Start a `BTreeMapEntryTransaction` that inserts the `val` into `key`.
-    pub fn new_entry_insert_txn(
-        &mut self,
-        key: K,
-        val: V,
-    ) -> BTreeMapEntryTransaction<'_, K, V, TXN> {
+    pub fn new_entry_insert_txn(&mut self, key: K, val: V) -> BTreeMapEntryTransaction<'_, K, V> {
         BTreeMapEntryTransaction::new_insert(self.tree_ref, key, val)
     }
 
@@ -472,10 +460,6 @@ impl<'a, K: Ord + Debug, V: Clone, TXN> BTreeMapTransaction<'a, K, V, TXN> {
         self.tree_ref
     }
 
-    pub fn tree_mut(&mut self) -> &mut BTreeMap<K, V> {
-        self.tree_ref
-    }
-
     /// Get the value of the provided key by merging the staging value and the original value
     pub fn get(&self, key: &K) -> Option<&V> {
         self.staging
@@ -564,16 +548,16 @@ impl<'a, K: Ord + Debug, V: Clone, TXN> BTreeMapTransaction<'a, K, V, TXN> {
     }
 }
 
-impl<'a, K: Ord + Debug, V: Transactional<TXN> + Clone, TXN> ValTransaction
-    for BTreeMapTransaction<'a, K, V, TXN>
-{
-    type TXN = TXN;
-
+impl<'a, K: Ord + Debug, V: Clone> InMemValTransaction for BTreeMapTransaction<'a, K, V> {
     fn commit(self) {
         self.commit_memory();
     }
+}
 
-    async fn apply_to_txn(&self, txn: &mut Self::TXN) -> MetadataModelResult<()> {
+impl<'a, K: Ord + Debug, V: Transactional<TXN> + Clone, TXN> ValTransaction<TXN>
+    for BTreeMapTransaction<'a, K, V>
+{
+    async fn apply_to_txn(&self, txn: &mut TXN) -> MetadataModelResult<()> {
         // Add the staging operation to txn
         for (k, op) in &self.staging {
             match op {
@@ -590,26 +574,24 @@ impl<'a, K: Ord + Debug, V: Transactional<TXN> + Clone, TXN> ValTransaction
 }
 
 /// Transaction wrapper for a `BTreeMap` entry value of given `key`
-pub struct BTreeMapEntryTransaction<'a, K, V, TXN> {
+pub struct BTreeMapEntryTransaction<'a, K, V> {
     tree_ref: &'a mut BTreeMap<K, V>,
     pub key: K,
     pub new_value: V,
-    _phantom: PhantomData<TXN>,
 }
 
-impl<'a, K: Ord + Debug, V: Clone, TXN> BTreeMapEntryTransaction<'a, K, V, TXN> {
+impl<'a, K: Ord + Debug, V: Clone> BTreeMapEntryTransaction<'a, K, V> {
     /// Create a `ValTransaction` that wraps a `BTreeMap` entry of the given `key`.
     /// If the tree does not contain `key`, the `default_val` will be used as the initial value
     pub fn new_insert(
         tree_ref: &'a mut BTreeMap<K, V>,
         key: K,
         value: V,
-    ) -> BTreeMapEntryTransaction<'a, K, V, TXN> {
+    ) -> BTreeMapEntryTransaction<'a, K, V> {
         BTreeMapEntryTransaction {
             new_value: value,
             tree_ref,
             key,
-            _phantom: PhantomData,
         }
     }
 
@@ -623,7 +605,7 @@ impl<'a, K: Ord + Debug, V: Clone, TXN> BTreeMapEntryTransaction<'a, K, V, TXN>
         tree_ref: &'a mut BTreeMap<K, V>,
         key: K,
         default_val: Option<V>,
-    ) -> Option<BTreeMapEntryTransaction<'a, K, V, TXN>> {
+    ) -> Option<BTreeMapEntryTransaction<'a, K, V>> {
         tree_ref
             .get(&key)
             .cloned()
@@ -632,12 +614,11 @@ impl<'a, K: Ord + Debug, V: Clone, TXN> BTreeMapEntryTransaction<'a, K, V, TXN>
                 new_value: orig_value,
                 tree_ref,
                 key,
-                _phantom: PhantomData,
             })
     }
 }
 
-impl<'a, K, V, TXN> Deref for BTreeMapEntryTransaction<'a, K, V, TXN> {
+impl<'a, K, V> Deref for BTreeMapEntryTransaction<'a, K, V> {
     type Target = V;
 
     fn deref(&self) -> &Self::Target {
@@ -645,22 +626,22 @@ impl<'a, K, V, TXN> Deref for BTreeMapEntryTransaction<'a, K, V, TXN> {
     }
 }
 
-impl<'a, K, V, TXN> DerefMut for BTreeMapEntryTransaction<'a, K, V, TXN> {
+impl<'a, K, V> DerefMut for BTreeMapEntryTransaction<'a, K, V> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         &mut self.new_value
     }
 }
 
-impl<'a, K: Ord, V: PartialEq + Transactional<TXN>, TXN> ValTransaction
-    for BTreeMapEntryTransaction<'a, K, V, TXN>
-{
-    type TXN = TXN;
-
+impl<'a, K: Ord, V: PartialEq> InMemValTransaction for BTreeMapEntryTransaction<'a, K, V> {
     fn commit(self) {
         self.tree_ref.insert(self.key, self.new_value);
     }
+}
 
-    async fn apply_to_txn(&self, txn: &mut Self::TXN) -> MetadataModelResult<()> {
+impl<'a, K: Ord, V: PartialEq + Transactional<TXN>, TXN> ValTransaction<TXN>
+    for BTreeMapEntryTransaction<'a, K, V>
+{
+    async fn apply_to_txn(&self, txn: &mut TXN) -> MetadataModelResult<()> {
         if !self.tree_ref.contains_key(&self.key)
             || *self.tree_ref.get(&self.key).unwrap() != self.new_value
         {
@@ -670,264 +651,6 @@ impl<'a, K: Ord, V: PartialEq + Transactional<TXN>, TXN> ValTransaction
     }
 }
 
-pub enum BTreeMapTransactionWrapper<'a, K: Ord, V> {
-    V1(BTreeMapTransaction<'a, K, V, Transaction>),
-    V2(BTreeMapTransaction<'a, K, V, TransactionV2>),
-}
-
-impl<'a, K: Ord + Debug, V: Clone> BTreeMapTransactionWrapper<'a, K, V> {
-    pub fn tree_ref(&self) -> &BTreeMap<K, V> {
-        match self {
-            BTreeMapTransactionWrapper::V1(v) => v.tree_ref,
-            BTreeMapTransactionWrapper::V2(v) => v.tree_ref,
-        }
-    }
-
-    pub fn tree_mut(&mut self) -> &mut BTreeMap<K, V> {
-        match self {
-            BTreeMapTransactionWrapper::V1(v) => v.tree_ref,
-            BTreeMapTransactionWrapper::V2(v) => v.tree_ref,
-        }
-    }
-
-    /// Get the value of the provided key by merging the staging value and the original value
-    pub fn get(&self, key: &K) -> Option<&V> {
-        match self {
-            BTreeMapTransactionWrapper::V1(v) => v.get(key),
-            BTreeMapTransactionWrapper::V2(v) => v.get(key),
-        }
-    }
-
-    pub fn contains_key(&self, key: &K) -> bool {
-        match self {
-            BTreeMapTransactionWrapper::V1(v) => v.contains_key(key),
-            BTreeMapTransactionWrapper::V2(v) => v.contains_key(key),
-        }
-    }
-
-    pub fn get_mut(&mut self, key: K) -> Option<BTreeMapTransactionValueGuard<'_, K, V>> {
-        match self {
-            BTreeMapTransactionWrapper::V1(v) => v.get_mut(key),
-            BTreeMapTransactionWrapper::V2(v) => v.get_mut(key),
-        }
-    }
-
-    pub fn insert(&mut self, key: K, value: V) {
-        match self {
-            BTreeMapTransactionWrapper::V1(v) => v.insert(key, value),
-            BTreeMapTransactionWrapper::V2(v) => v.insert(key, value),
-        }
-    }
-
-    pub fn remove(&mut self, key: K) -> Option<V> {
-        match self {
-            BTreeMapTransactionWrapper::V1(v) => v.remove(key),
-            BTreeMapTransactionWrapper::V2(v) => v.remove(key),
-        }
-    }
-
-    pub fn commit_memory(self) {
-        match self {
-            BTreeMapTransactionWrapper::V1(v) => v.commit_memory(),
-            BTreeMapTransactionWrapper::V2(v) => v.commit_memory(),
-        }
-    }
-
-    pub fn new_entry_txn_or_default(
-        &mut self,
-        key: K,
-        default_val: V,
-    ) -> BTreeMapEntryTransactionWrapper<'_, K, V> {
-        match self {
-            BTreeMapTransactionWrapper::V1(v) => BTreeMapEntryTransactionWrapper::V1(
-                BTreeMapEntryTransaction::new(v.tree_ref, key, Some(default_val))
-                    .expect("default value is provided and should return `Some`"),
-            ),
-            BTreeMapTransactionWrapper::V2(v) => BTreeMapEntryTransactionWrapper::V2(
-                BTreeMapEntryTransaction::new(v.tree_ref, key, Some(default_val))
-                    .expect("default value is provided and should return `Some`"),
-            ),
-        }
-    }
-
-    pub fn new_entry_insert_txn(
-        &mut self,
-        key: K,
-        val: V,
-    ) -> BTreeMapEntryTransactionWrapper<'_, K, V> {
-        match self {
-            BTreeMapTransactionWrapper::V1(v) => BTreeMapEntryTransactionWrapper::V1(
-                BTreeMapEntryTransaction::new_insert(v.tree_ref, key, val),
-            ),
-            BTreeMapTransactionWrapper::V2(v) => BTreeMapEntryTransactionWrapper::V2(
-                BTreeMapEntryTransaction::new_insert(v.tree_ref, key, val),
-            ),
-        }
-    }
-}
-
-impl<'a, K: Ord + Debug, V: Clone> BTreeMapTransactionWrapper<'a, K, V> {
-    pub fn into_v1(self) -> BTreeMapTransaction<'a, K, V, Transaction> {
-        match self {
-            BTreeMapTransactionWrapper::V1(v) => v,
-            BTreeMapTransactionWrapper::V2(_) => panic!("expect V1, found V2"),
-        }
-    }
-
-    pub fn as_v1_ref(&self) -> &BTreeMapTransaction<'a, K, V, Transaction> {
-        match self {
-            BTreeMapTransactionWrapper::V1(v) => v,
-            BTreeMapTransactionWrapper::V2(_) => panic!("expect V1, found V2"),
-        }
-    }
-
-    pub fn into_v2(self) -> BTreeMapTransaction<'a, K, V, TransactionV2> {
-        match self {
-            BTreeMapTransactionWrapper::V1(_) => panic!("expect V2, found V1"),
-            BTreeMapTransactionWrapper::V2(v) => v,
-        }
-    }
-
-    pub fn as_v2_ref(&self) -> &BTreeMapTransaction<'a, K, V, TransactionV2> {
-        match self {
-            BTreeMapTransactionWrapper::V1(_) => panic!("expect V2, found V1"),
-            BTreeMapTransactionWrapper::V2(v) => v,
-        }
-    }
-}
-
-pub enum BTreeMapEntryTransactionWrapper<'a, K, V> {
-    V1(BTreeMapEntryTransaction<'a, K, V, Transaction>),
-    V2(BTreeMapEntryTransaction<'a, K, V, TransactionV2>),
-}
-
-impl<'a, K: Ord + Debug, V: Clone> Deref for BTreeMapEntryTransactionWrapper<'a, K, V> {
-    type Target = V;
-
-    fn deref(&self) -> &Self::Target {
-        match self {
-            BTreeMapEntryTransactionWrapper::V1(v) => v.deref(),
-            BTreeMapEntryTransactionWrapper::V2(v) => v.deref(),
-        }
-    }
-}
-
-impl<'a, K: Ord + Debug, V: Clone> DerefMut for BTreeMapEntryTransactionWrapper<'a, K, V> {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        match self {
-            BTreeMapEntryTransactionWrapper::V1(v) => v.deref_mut(),
-            BTreeMapEntryTransactionWrapper::V2(v) => v.deref_mut(),
-        }
-    }
-}
-
-impl<'a, K: Ord + Debug, V: Clone> BTreeMapEntryTransactionWrapper<'a, K, V> {
-    pub fn as_v1_ref(&self) -> &BTreeMapEntryTransaction<'a, K, V, Transaction> {
-        match self {
-            BTreeMapEntryTransactionWrapper::V1(v) => v,
-            BTreeMapEntryTransactionWrapper::V2(_) => {
-                panic!("expect V1, found V2")
-            }
-        }
-    }
-
-    pub fn into_v1(self) -> BTreeMapEntryTransaction<'a, K, V, Transaction> {
-        match self {
-            BTreeMapEntryTransactionWrapper::V1(v) => v,
-            BTreeMapEntryTransactionWrapper::V2(_) => {
-                panic!("expect V1, found V2")
-            }
-        }
-    }
-
-    pub fn as_v2_ref(&self) -> &BTreeMapEntryTransaction<'a, K, V, TransactionV2> {
-        match self {
-            BTreeMapEntryTransactionWrapper::V1(_) => {
-                panic!("expect V2, found V1")
-            }
-            BTreeMapEntryTransactionWrapper::V2(v) => v,
-        }
-    }
-
-    pub fn into_v2(self) -> BTreeMapEntryTransaction<'a, K, V, TransactionV2> {
-        match self {
-            BTreeMapEntryTransactionWrapper::V1(_) => {
-                panic!("expect V2, found V1")
-            }
-            BTreeMapEntryTransactionWrapper::V2(v) => v,
-        }
-    }
-}
-
-pub enum VarTransactionWrapper<'a, T: Transactional<Transaction> + Transactional<TransactionV2>> {
-    V1(VarTransaction<'a, Transaction, T>),
-    V2(VarTransaction<'a, TransactionV2, T>),
-}
-
-impl<'a, T: Transactional<Transaction> + Transactional<TransactionV2>>
-    VarTransactionWrapper<'a, T>
-{
-    pub fn as_v1_ref(&self) -> &VarTransaction<'a, Transaction, T> {
-        match self {
-            VarTransactionWrapper::V1(v) => v,
-            VarTransactionWrapper::V2(_) => {
-                panic!("expect V1, found V2")
-            }
-        }
-    }
-
-    pub fn into_v1(self) -> VarTransaction<'a, Transaction, T> {
-        match self {
-            VarTransactionWrapper::V1(v) => v,
-            VarTransactionWrapper::V2(_) => {
-                panic!("expect V1, found V2")
-            }
-        }
-    }
-
-    pub fn as_v2_ref(&self) -> &VarTransaction<'a, TransactionV2, T> {
-        match self {
-            VarTransactionWrapper::V1(_) => {
-                panic!("expect V2, found V1")
-            }
-            VarTransactionWrapper::V2(v) => v,
-        }
-    }
-
-    pub fn into_v2(self) -> VarTransaction<'a, TransactionV2, T> {
-        match self {
-            VarTransactionWrapper::V1(_) => {
-                panic!("expect V2, found V1")
-            }
-            VarTransactionWrapper::V2(v) => v,
-        }
-    }
-}
-
-impl<'a, T: Transactional<Transaction> + Transactional<TransactionV2>> Deref
-    for VarTransactionWrapper<'a, T>
-{
-    type Target = T;
-
-    fn deref(&self) -> &Self::Target {
-        match self {
-            VarTransactionWrapper::V1(v) => v.deref(),
-            VarTransactionWrapper::V2(v) => v.deref(),
-        }
-    }
-}
-
-impl<'a, T: Transactional<Transaction> + Transactional<TransactionV2> + Clone> DerefMut
-    for VarTransactionWrapper<'a, T>
-{
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        match self {
-            VarTransactionWrapper::V1(v) => v.deref_mut(),
-            VarTransactionWrapper::V2(v) => v.deref_mut(),
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -994,7 +717,6 @@ mod tests {
         };
         let mut num_txn = VarTransaction::new(&mut kv);
         num_txn.value = "modified";
-        num_txn.abort();
         assert_eq!("original", kv.value);
     }
 
diff --git a/src/meta/src/model/stream.rs b/src/meta/src/model/stream.rs
index 058473580a96d..4e49a602ed8a4 100644
--- a/src/meta/src/model/stream.rs
+++ b/src/meta/src/model/stream.rs
@@ -239,8 +239,8 @@ impl TableFragments {
         self.fragments.keys().cloned()
     }
 
-    pub fn fragments(&self) -> Vec<&Fragment> {
-        self.fragments.values().collect_vec()
+    pub fn fragments(&self) -> impl Iterator<Item = &Fragment> {
+        self.fragments.values()
     }
 
     /// Returns the table id.
diff --git a/src/meta/src/rpc/ddl_controller.rs b/src/meta/src/rpc/ddl_controller.rs
index c0aa62750facb..e1c6ebb0f32a8 100644
--- a/src/meta/src/rpc/ddl_controller.rs
+++ b/src/meta/src/rpc/ddl_controller.rs
@@ -18,16 +18,19 @@ use std::num::NonZeroUsize;
 use std::sync::Arc;
 use std::time::Duration;
 
+use aes_siv::aead::generic_array::GenericArray;
+use aes_siv::aead::Aead;
+use aes_siv::{Aes128SivAead, KeyInit};
 use anyhow::Context;
 use itertools::Itertools;
-use rand::Rng;
+use rand::{Rng, RngCore};
 use risingwave_common::config::DefaultParallelism;
 use risingwave_common::hash::{ParallelUnitMapping, VirtualNode};
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_common::util::epoch::Epoch;
 use risingwave_common::util::stream_graph_visitor::{
-    visit_fragment, visit_stream_node, visit_stream_node_cont,
+    visit_fragment, visit_stream_node, visit_stream_node_cont_mut,
 };
 use risingwave_common::{bail, current_cluster_version};
 use risingwave_connector::dispatch_source_prop;
@@ -45,7 +48,7 @@ use risingwave_pb::catalog::source::OptionalAssociatedTableId;
 use risingwave_pb::catalog::table::OptionalAssociatedSourceId;
 use risingwave_pb::catalog::{
     connection, Comment, Connection, CreateType, Database, Function, PbSource, PbTable, Schema,
-    Sink, Source, Table, View,
+    Secret, Sink, Source, Subscription, Table, View,
 };
 use risingwave_pb::ddl_service::alter_owner_request::Object;
 use risingwave_pb::ddl_service::{
@@ -58,6 +61,7 @@ use risingwave_pb::stream_plan::{
     Dispatcher, DispatcherType, FragmentTypeFlag, MergeNode, PbStreamFragmentGraph,
     StreamFragmentGraph as StreamFragmentGraphProto,
 };
+use serde::{Deserialize, Serialize};
 use thiserror_ext::AsReport;
 use tokio::sync::Semaphore;
 use tokio::time::sleep;
@@ -65,11 +69,13 @@ use tracing::log::warn;
 use tracing::Instrument;
 
 use crate::barrier::BarrierManagerRef;
+use crate::error::MetaErrorInner;
 use crate::manager::{
     CatalogManagerRef, ConnectionId, DatabaseId, FragmentManagerRef, FunctionId, IdCategory,
     IdCategoryType, IndexId, LocalNotification, MetaSrvEnv, MetadataManager, MetadataManagerV1,
-    NotificationVersion, RelationIdEnum, SchemaId, SinkId, SourceId, StreamingClusterInfo,
-    StreamingJob, SubscriptionId, TableId, UserId, ViewId, IGNORED_NOTIFICATION_VERSION,
+    NotificationVersion, RelationIdEnum, SchemaId, SecretId, SinkId, SourceId,
+    StreamingClusterInfo, StreamingJob, StreamingJobDiscriminants, SubscriptionId, TableId, UserId,
+    ViewId, IGNORED_NOTIFICATION_VERSION,
 };
 use crate::model::{FragmentId, StreamContext, TableFragments, TableParallelism};
 use crate::rpc::cloud_provider::AwsEc2Client;
@@ -101,7 +107,6 @@ pub enum StreamingJobId {
     Sink(SinkId),
     Table(Option<SourceId>, TableId),
     Index(IndexId),
-    Subscription(SubscriptionId),
 }
 
 impl StreamingJobId {
@@ -110,7 +115,6 @@ impl StreamingJobId {
         match self {
             StreamingJobId::MaterializedView(id)
             | StreamingJobId::Sink(id)
-            | StreamingJobId::Subscription(id)
             | StreamingJobId::Table(_, id)
             | StreamingJobId::Index(id) => *id,
         }
@@ -149,7 +153,17 @@ pub enum DdlCommand {
     AlterSetSchema(alter_set_schema_request::Object, SchemaId),
     CreateConnection(Connection),
     DropConnection(ConnectionId),
+    CreateSecret(Secret),
+    DropSecret(SecretId),
     CommentOn(Comment),
+    CreateSubscription(Subscription),
+    DropSubscription(SubscriptionId, DropMode),
+}
+
+#[derive(Deserialize, Serialize)]
+struct SecretEncryption {
+    nonce: [u8; 16],
+    ciphertext: Vec<u8>,
 }
 
 impl DdlCommand {
@@ -161,7 +175,9 @@ impl DdlCommand {
             | DdlCommand::DropFunction(_)
             | DdlCommand::DropView(_, _)
             | DdlCommand::DropStreamingJob(_, _, _)
-            | DdlCommand::DropConnection(_) => true,
+            | DdlCommand::DropConnection(_)
+            | DdlCommand::DropSecret(_) => true,
+
             // Simply ban all other commands in recovery.
             _ => false,
         }
@@ -330,8 +346,16 @@ impl DdlController {
                 DdlCommand::DropConnection(connection_id) => {
                     ctrl.drop_connection(connection_id).await
                 }
+                DdlCommand::CreateSecret(secret) => ctrl.create_secret(secret).await,
+                DdlCommand::DropSecret(secret_id) => ctrl.drop_secret(secret_id).await,
                 DdlCommand::AlterSourceColumn(source) => ctrl.alter_source_column(source).await,
                 DdlCommand::CommentOn(comment) => ctrl.comment_on(comment).await,
+                DdlCommand::CreateSubscription(subscription) => {
+                    ctrl.create_subscription(subscription).await
+                }
+                DdlCommand::DropSubscription(subscription_id, drop_mode) => {
+                    ctrl.drop_subscription(subscription_id, drop_mode).await
+                }
             }
         }
         .in_current_span();
@@ -602,6 +626,57 @@ impl DdlController {
         }
     }
 
+    async fn create_secret(&self, mut secret: Secret) -> MetaResult<NotificationVersion> {
+        // The 'secret' part of the request we receive from the frontend is in plaintext;
+        // here, we need to encrypt it before storing it in the catalog.
+
+        let encrypted_payload = {
+            let data = secret.get_value().as_slice();
+            let key = self.env.opts.secret_store_private_key.as_slice();
+            let encrypt_key = {
+                let mut k = key[..(std::cmp::min(key.len(), 32))].to_vec();
+                k.resize_with(32, || 0);
+                k
+            };
+
+            let mut rng = rand::thread_rng();
+            let mut nonce: [u8; 16] = [0; 16];
+            rng.fill_bytes(&mut nonce);
+            let nonce_array = GenericArray::from_slice(&nonce);
+            let cipher = Aes128SivAead::new(encrypt_key.as_slice().into());
+
+            let ciphertext = cipher.encrypt(nonce_array, data).map_err(|e| {
+                MetaError::from(MetaErrorInner::InvalidParameter(format!(
+                    "failed to encrypt secret {}: {:?}",
+                    secret.name, e
+                )))
+            })?;
+            bincode::serialize(&SecretEncryption { nonce, ciphertext }).map_err(|e| {
+                MetaError::from(MetaErrorInner::InvalidParameter(format!(
+                    "failed to serialize secret {}: {:?}",
+                    secret.name,
+                    e.as_report()
+                )))
+            })?
+        };
+        secret.value = encrypted_payload;
+
+        match &self.metadata_manager {
+            MetadataManager::V1(mgr) => {
+                secret.id = self.gen_unique_id::<{ IdCategory::Secret }>().await?;
+                mgr.catalog_manager.create_secret(secret).await
+            }
+            MetadataManager::V2(mgr) => mgr.catalog_controller.create_secret(secret).await,
+        }
+    }
+
+    async fn drop_secret(&self, secret_id: SecretId) -> MetaResult<NotificationVersion> {
+        match &self.metadata_manager {
+            MetadataManager::V1(mgr) => mgr.catalog_manager.drop_secret(secret_id).await,
+            MetadataManager::V2(mgr) => mgr.catalog_controller.drop_secret(secret_id as _).await,
+        }
+    }
+
     pub(crate) async fn delete_vpc_endpoint(&self, connection: &Connection) -> MetaResult<()> {
         // delete AWS vpc endpoint
         if let Some(connection::Info::PrivateLinkService(svc)) = &connection.info
@@ -634,6 +709,114 @@ impl DdlController {
         Ok(())
     }
 
+    async fn create_subscription(
+        &self,
+        mut subscription: Subscription,
+    ) -> MetaResult<NotificationVersion> {
+        tracing::debug!("create subscription");
+        let _permit = self
+            .creating_streaming_job_permits
+            .semaphore
+            .acquire()
+            .await
+            .unwrap();
+        let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await;
+        match &self.metadata_manager {
+            MetadataManager::V1(mgr) => {
+                let id = self.gen_unique_id::<{ IdCategory::Table }>().await?;
+                let initialized_at_epoch = Some(Epoch::now().0);
+                let initialized_at_cluster_version = Some(current_cluster_version());
+                subscription.initialized_at_epoch = initialized_at_epoch;
+                subscription.initialized_at_cluster_version = initialized_at_cluster_version;
+                subscription.id = id;
+
+                mgr.catalog_manager
+                    .start_create_subscription_procedure(&subscription)
+                    .await?;
+                match self.stream_manager.create_subscription(&subscription).await {
+                    Ok(_) => {
+                        let version = mgr
+                            .catalog_manager
+                            .notify_create_subscription(subscription.id)
+                            .await?;
+                        tracing::debug!("finish create subscription");
+                        Ok(version)
+                    }
+                    Err(e) => {
+                        tracing::debug!("cancel create subscription");
+                        Err(e)
+                    }
+                }
+            }
+            MetadataManager::V2(mgr) => {
+                mgr.catalog_controller
+                    .create_subscription_catalog(&mut subscription)
+                    .await?;
+                match self.stream_manager.create_subscription(&subscription).await {
+                    Ok(_) => {
+                        let version = mgr
+                            .catalog_controller
+                            .notify_create_subscription(subscription.id)
+                            .await?;
+                        tracing::debug!("finish create subscription");
+                        Ok(version)
+                    }
+                    Err(e) => {
+                        tracing::debug!("cancel create subscription");
+                        Err(e)
+                    }
+                }
+            }
+        }
+    }
+
+    async fn drop_subscription(
+        &self,
+        subscription_id: SubscriptionId,
+        drop_mode: DropMode,
+    ) -> MetaResult<NotificationVersion> {
+        tracing::debug!("preparing drop subscription");
+        let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await;
+        match &self.metadata_manager {
+            MetadataManager::V1(mgr) => {
+                let table_id = mgr
+                    .catalog_manager
+                    .get_subscription_by_id(subscription_id)
+                    .await?
+                    .dependent_table_id;
+                let (version, _) = mgr
+                    .catalog_manager
+                    .drop_relation(
+                        RelationIdEnum::Subscription(subscription_id),
+                        mgr.fragment_manager.clone(),
+                        drop_mode,
+                    )
+                    .await?;
+                self.stream_manager
+                    .drop_subscription(subscription_id, table_id)
+                    .await;
+                tracing::debug!("finish drop subscription");
+                Ok(version)
+            }
+            MetadataManager::V2(mgr) => {
+                let table_id = mgr
+                    .catalog_controller
+                    .get_subscription_by_id(subscription_id as i32)
+                    .await?
+                    .dependent_table_id;
+                let (_, version) = mgr
+                    .catalog_controller
+                    .drop_relation(ObjectType::Subscription, subscription_id as _, drop_mode)
+                    .await?;
+                self.stream_manager
+                    .drop_subscription(subscription_id, table_id)
+                    .await;
+                tracing::debug!("finish drop subscription");
+                Ok(version)
+            }
+        }
+    }
+
     async fn create_streaming_job(
         &self,
         mut stream_job: StreamingJob,
@@ -793,9 +976,9 @@ impl DdlController {
                     ctx,
                     internal_tables,
                 )
-                .await
+                    .await
             }
-            (CreateType::Background, _) => {
+            (CreateType::Background, &StreamingJob::MaterializedView(_)) => {
                 let ctrl = self.clone();
                 let mgr = mgr.clone();
                 let stream_job_id = stream_job.id();
@@ -821,6 +1004,10 @@ impl DdlController {
                 tokio::spawn(fut);
                 Ok(IGNORED_NOTIFICATION_VERSION)
             }
+            (CreateType::Background, _) => {
+                let d: StreamingJobDiscriminants = stream_job.into();
+                bail!("background_ddl not supported for: {:?}", d)
+            }
         }
     }
 
@@ -853,11 +1040,7 @@ impl DdlController {
                 actor.nodes.as_ref().unwrap().node_body
                 && let Some(ref cdc_table_desc) = stream_cdc_scan.cdc_table_desc
             {
-                let properties: HashMap<String, String> = cdc_table_desc
-                    .connect_properties
-                    .iter()
-                    .map(|(k, v)| (k.clone(), v.clone()))
-                    .collect();
+                let properties = cdc_table_desc.connect_properties.clone();
                 let mut props = ConnectorProperties::extract(properties, true)?;
                 props.init_from_pb_cdc_table_desc(cdc_table_desc);
 
@@ -1043,7 +1226,7 @@ impl DdlController {
             if let Some(node) = &mut actor.nodes {
                 let fields = node.fields.clone();
 
-                visit_stream_node_cont(node, |node| {
+                visit_stream_node_cont_mut(node, |node| {
                     if let Some(NodeBody::Union(_)) = &mut node.node_body {
                         for input in &mut node.input {
                             if let Some(NodeBody::Merge(merge_node)) = &mut input.node_body
@@ -1159,7 +1342,6 @@ impl DdlController {
                     StreamingJobId::Sink(id) => (id as _, ObjectType::Sink),
                     StreamingJobId::Table(_, id) => (id as _, ObjectType::Table),
                     StreamingJobId::Index(idx) => (idx as _, ObjectType::Index),
-                    StreamingJobId::Subscription(id) => (id as _, ObjectType::Subscription),
                 };
 
                 let version = self
@@ -1217,15 +1399,6 @@ impl DdlController {
                     )
                     .await?
             }
-            StreamingJobId::Subscription(subscription_id) => {
-                mgr.catalog_manager
-                    .drop_relation(
-                        RelationIdEnum::Subscription(subscription_id),
-                        mgr.fragment_manager.clone(),
-                        drop_mode,
-                    )
-                    .await?
-            }
         };
 
         if let Some(replace_table_info) = target_replace_info {
@@ -1543,11 +1716,6 @@ impl DdlController {
                     .cancel_create_sink_procedure(sink, target_table)
                     .await;
             }
-            StreamingJob::Subscription(subscription) => {
-                mgr.catalog_manager
-                    .cancel_create_subscription_procedure(subscription)
-                    .await;
-            }
             StreamingJob::Table(source, table, ..) => {
                 if let Some(source) = source {
                     mgr.catalog_manager
@@ -1634,11 +1802,6 @@ impl DdlController {
 
                 version
             }
-            StreamingJob::Subscription(subscription) => {
-                mgr.catalog_manager
-                    .finish_create_subscription_procedure(internal_tables, subscription)
-                    .await?
-            }
             StreamingJob::Table(source, table, ..) => {
                 creating_internal_table_ids.push(table.id);
                 if let Some(source) = source {
@@ -1832,20 +1995,20 @@ impl DdlController {
 
         // Map the column indices in the dispatchers with the given mapping.
         let downstream_fragments = self.metadata_manager.get_downstream_chain_fragments(id).await?
-                .into_iter()
-                .map(|(d, f)|
-                    if let Some(mapping) = &table_col_index_mapping {
-                        Some((mapping.rewrite_dispatch_strategy(&d)?, f))
-                    } else {
-                        Some((d, f))
-                    })
-                .collect::<Option<_>>()
-                .ok_or_else(|| {
-                    // The `rewrite` only fails if some column is dropped.
-                    MetaError::invalid_parameter(
-                        "unable to drop the column due to being referenced by downstream materialized views or sinks",
-                    )
-                })?;
+            .into_iter()
+            .map(|(d, f)|
+                if let Some(mapping) = &table_col_index_mapping {
+                    Some((mapping.rewrite_dispatch_strategy(&d)?, f))
+                } else {
+                    Some((d, f))
+                })
+            .collect::<Option<_>>()
+            .ok_or_else(|| {
+                // The `rewrite` only fails if some column is dropped.
+                MetaError::invalid_parameter(
+                    "unable to drop the column due to being referenced by downstream materialized views or sinks",
+                )
+            })?;
 
         let complete_graph = CompleteStreamFragmentGraph::with_downstreams(
             fragment_graph,
diff --git a/src/meta/src/rpc/ddl_controller_v2.rs b/src/meta/src/rpc/ddl_controller_v2.rs
index 3e948e88e2821..13ecfca13c782 100644
--- a/src/meta/src/rpc/ddl_controller_v2.rs
+++ b/src/meta/src/rpc/ddl_controller_v2.rs
@@ -214,7 +214,7 @@ impl DdlController {
                         (
                             streaming_job.clone(),
                             ctx.merge_updates.clone(),
-                            table_fragments.table_id(),
+                            table_fragments.table_id().table_id(),
                         )
                     },
                 );
@@ -223,25 +223,11 @@ impl DdlController {
                     .create_streaming_job(table_fragments, ctx)
                     .await?;
 
-                let mut version = mgr
+                let version = mgr
                     .catalog_controller
-                    .finish_streaming_job(stream_job_id as _)
+                    .finish_streaming_job(stream_job_id as _, replace_table_job_info)
                     .await?;
 
-                if let Some((streaming_job, merge_updates, table_id)) = replace_table_job_info {
-                    version = mgr
-                        .catalog_controller
-                        .finish_replace_streaming_job(
-                            table_id.table_id as _,
-                            streaming_job,
-                            merge_updates,
-                            None,
-                            Some(stream_job_id),
-                            None,
-                        )
-                        .await?;
-                }
-
                 Ok(version)
             }
             (CreateType::Background, _) => {
@@ -257,7 +243,7 @@ impl DdlController {
                     if result.is_ok() {
                         let _ = mgr
                             .catalog_controller
-                            .finish_streaming_job(stream_job_id as _)
+                            .finish_streaming_job(stream_job_id as _, None)
                             .await.inspect_err(|err| {
                                 tracing::error!(id = stream_job_id, error = ?err.as_report(), "failed to finish background streaming job");
                             });
diff --git a/src/meta/src/rpc/election/sql.rs b/src/meta/src/rpc/election/sql.rs
index 65c3ad613dde0..9ec5bd199cf76 100644
--- a/src/meta/src/rpc/election/sql.rs
+++ b/src/meta/src/rpc/election/sql.rs
@@ -64,6 +64,8 @@ pub trait SqlDriver: Send + Sync + 'static {
     async fn candidates(&self, service_name: &str) -> MetaResult<Vec<ElectionRow>>;
 
     async fn resign(&self, service_name: &str, id: &str) -> MetaResult<()>;
+
+    async fn trim_candidates(&self, service_name: &str, timeout: i64) -> MetaResult<()>;
 }
 
 pub trait SqlDriverCommon {
@@ -263,6 +265,23 @@ DO
 
         Ok(())
     }
+
+    async fn trim_candidates(&self, service_name: &str, timeout: i64) -> MetaResult<()> {
+        self.conn
+            .execute(Statement::from_sql_and_values(
+                DatabaseBackend::Sqlite,
+                format!(
+                    r#"
+                    DELETE FROM {table} WHERE service = $1 AND DATETIME({table}.last_heartbeat, '+' || $2 || ' second') < CURRENT_TIMESTAMP;
+                    "#,
+                    table = Self::member_table_name()
+                ),
+                vec![Value::from(service_name), Value::from(timeout)],
+            ))
+            .await?;
+
+        Ok(())
+    }
 }
 
 #[async_trait::async_trait]
@@ -421,6 +440,23 @@ impl SqlDriver for MySqlDriver {
 
         Ok(())
     }
+
+    async fn trim_candidates(&self, service_name: &str, timeout: i64) -> MetaResult<()> {
+        self.conn
+            .execute(Statement::from_sql_and_values(
+                DatabaseBackend::MySql,
+                format!(
+                    r#"
+                    DELETE FROM {table} WHERE service = ? AND last_heartbeat < NOW() - INTERVAL ? SECOND;
+                    "#,
+                    table = Self::member_table_name()
+                ),
+                vec![Value::from(service_name), Value::from(timeout)],
+            ))
+            .await?;
+
+        Ok(())
+    }
 }
 
 #[async_trait::async_trait]
@@ -579,6 +615,23 @@ impl SqlDriver for PostgresDriver {
 
         Ok(())
     }
+
+    async fn trim_candidates(&self, service_name: &str, timeout: i64) -> MetaResult<()> {
+        self.conn
+            .execute(Statement::from_sql_and_values(
+                DatabaseBackend::Postgres,
+                format!(
+                    r#"
+                    DELETE FROM {table} WHERE {table}.service = $1 AND {table}.last_heartbeat < NOW() - $2::INTERVAL;
+                    "#,
+                    table = Self::member_table_name()
+                ),
+                vec![Value::from(service_name), Value::from(timeout.to_string())],
+            ))
+            .await?;
+
+        Ok(())
+    }
 }
 
 #[async_trait::async_trait]
@@ -638,13 +691,15 @@ where
 
         let mut election_ticker = time::interval(Duration::from_secs(1));
 
+        let mut prev_leader = "".to_string();
+
         loop {
             tokio::select! {
-                _ = election_ticker.tick() => {
-                    let election_row = self
-                        .driver
-                        .try_campaign(META_ELECTION_KEY, self.id.as_str(), ttl)
-                        .await?;
+                    _ = election_ticker.tick() => {
+                        let election_row = self
+                            .driver
+                            .try_campaign(META_ELECTION_KEY, self.id.as_str(), ttl)
+                            .await?;
 
                         assert_eq!(election_row.service, META_ELECTION_KEY);
 
@@ -652,14 +707,25 @@ where
                             if !is_leader{
                                 self.is_leader_sender.send_replace(true);
                                 is_leader = true;
+                            } else {
+                                self.is_leader_sender.send_replace(false);
                             }
                         } else if is_leader {
                             tracing::warn!("leader has been changed to {}", election_row.id);
                             break;
+                        } else if prev_leader != election_row.id {
+                            tracing::info!("leader is {}", election_row.id);
+                            prev_leader.clone_from(&election_row.id)
                         }
 
-                    timeout_ticker.reset();
-                }
+                        timeout_ticker.reset();
+
+                        if is_leader {
+                            if let Err(e) = self.driver.trim_candidates(META_ELECTION_KEY, ttl * 2).await {
+                                tracing::warn!(error = %e.as_report(), "trim candidates failed");
+                            }
+                        }
+                    }
                 _ = timeout_ticker.tick() => {
                     tracing::error!("member {} election timeout", self.id);
                     break;
diff --git a/src/meta/src/rpc/metrics.rs b/src/meta/src/rpc/metrics.rs
index 2df847516c4a8..28520720e98fe 100644
--- a/src/meta/src/rpc/metrics.rs
+++ b/src/meta/src/rpc/metrics.rs
@@ -449,7 +449,7 @@ impl MetaMetrics {
         let hummock_manager_lock_time = register_histogram_vec_with_registry!(
             "hummock_manager_lock_time",
             "latency for hummock manager to acquire the rwlock",
-            &["method", "lock_name", "lock_type"],
+            &["lock_name", "lock_type"],
             registry
         )
         .unwrap();
diff --git a/src/meta/src/serving/mod.rs b/src/meta/src/serving/mod.rs
index 36e7b77ccf63a..69e17a978212e 100644
--- a/src/meta/src/serving/mod.rs
+++ b/src/meta/src/serving/mod.rs
@@ -16,11 +16,11 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use parking_lot::RwLock;
-use risingwave_common::hash::ParallelUnitMapping;
+use risingwave_common::hash::WorkerSlotMapping;
 use risingwave_common::vnode_mapping::vnode_placement::place_vnode;
 use risingwave_pb::common::{WorkerNode, WorkerType};
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
-use risingwave_pb::meta::{FragmentParallelUnitMapping, FragmentParallelUnitMappings};
+use risingwave_pb::meta::{FragmentWorkerSlotMapping, FragmentWorkerSlotMappings};
 use tokio::sync::oneshot::Sender;
 use tokio::task::JoinHandle;
 
@@ -31,11 +31,11 @@ pub type ServingVnodeMappingRef = Arc<ServingVnodeMapping>;
 
 #[derive(Default)]
 pub struct ServingVnodeMapping {
-    serving_vnode_mappings: RwLock<HashMap<FragmentId, ParallelUnitMapping>>,
+    serving_vnode_mappings: RwLock<HashMap<FragmentId, WorkerSlotMapping>>,
 }
 
 impl ServingVnodeMapping {
-    pub fn all(&self) -> HashMap<FragmentId, ParallelUnitMapping> {
+    pub fn all(&self) -> HashMap<FragmentId, WorkerSlotMapping> {
         self.serving_vnode_mappings.read().clone()
     }
 
@@ -45,9 +45,9 @@ impl ServingVnodeMapping {
         &self,
         streaming_parallelisms: HashMap<FragmentId, usize>,
         workers: &[WorkerNode],
-    ) -> (HashMap<FragmentId, ParallelUnitMapping>, Vec<FragmentId>) {
+    ) -> (HashMap<FragmentId, WorkerSlotMapping>, Vec<FragmentId>) {
         let mut serving_vnode_mappings = self.serving_vnode_mappings.write();
-        let mut upserted: HashMap<FragmentId, ParallelUnitMapping> = HashMap::default();
+        let mut upserted: HashMap<FragmentId, WorkerSlotMapping> = HashMap::default();
         let mut failed: Vec<FragmentId> = vec![];
         for (fragment_id, streaming_parallelism) in streaming_parallelisms {
             let new_mapping = {
@@ -81,24 +81,24 @@ impl ServingVnodeMapping {
     }
 }
 
-pub(crate) fn to_fragment_parallel_unit_mapping(
-    mappings: &HashMap<FragmentId, ParallelUnitMapping>,
-) -> Vec<FragmentParallelUnitMapping> {
+pub(crate) fn to_fragment_worker_slot_mapping(
+    mappings: &HashMap<FragmentId, WorkerSlotMapping>,
+) -> Vec<FragmentWorkerSlotMapping> {
     mappings
         .iter()
-        .map(|(fragment_id, mapping)| FragmentParallelUnitMapping {
+        .map(|(fragment_id, mapping)| FragmentWorkerSlotMapping {
             fragment_id: *fragment_id,
             mapping: Some(mapping.to_protobuf()),
         })
         .collect()
 }
 
-pub(crate) fn to_deleted_fragment_parallel_unit_mapping(
+pub(crate) fn to_deleted_fragment_worker_slot_mapping(
     fragment_ids: &[FragmentId],
-) -> Vec<FragmentParallelUnitMapping> {
+) -> Vec<FragmentWorkerSlotMapping> {
     fragment_ids
         .iter()
-        .map(|fragment_id| FragmentParallelUnitMapping {
+        .map(|fragment_id| FragmentWorkerSlotMapping {
             fragment_id: *fragment_id,
             mapping: None,
         })
@@ -120,8 +120,8 @@ pub async fn on_meta_start(
     );
     notification_manager.notify_frontend_without_version(
         Operation::Snapshot,
-        Info::ServingParallelUnitMappings(FragmentParallelUnitMappings {
-            mappings: to_fragment_parallel_unit_mapping(&mappings),
+        Info::ServingWorkerSlotMappings(FragmentWorkerSlotMappings {
+            mappings: to_fragment_worker_slot_mapping(&mappings),
         }),
     );
 }
@@ -185,7 +185,7 @@ pub async fn start_serving_vnode_mapping_worker(
                                     let (workers, streaming_parallelisms) = fetch_serving_infos(&metadata_manager).await;
                                     let (mappings, _) = serving_vnode_mapping.upsert(streaming_parallelisms, &workers);
                                     tracing::debug!("Update serving vnode mapping snapshot for fragments {:?}.", mappings.keys());
-                                    notification_manager.notify_frontend_without_version(Operation::Snapshot, Info::ServingParallelUnitMappings(FragmentParallelUnitMappings{ mappings: to_fragment_parallel_unit_mapping(&mappings) }));
+                                    notification_manager.notify_frontend_without_version(Operation::Snapshot, Info::ServingWorkerSlotMappings(FragmentWorkerSlotMappings{ mappings: to_fragment_worker_slot_mapping(&mappings) }));
                                 }
                                 LocalNotification::FragmentMappingsUpsert(fragment_ids) => {
                                     if fragment_ids.is_empty() {
@@ -195,11 +195,11 @@ pub async fn start_serving_vnode_mapping_worker(
                                     let (upserted, failed) = serving_vnode_mapping.upsert(streaming_parallelisms, &workers);
                                     if !upserted.is_empty() {
                                         tracing::debug!("Update serving vnode mapping for fragments {:?}.", upserted.keys());
-                                        notification_manager.notify_frontend_without_version(Operation::Update, Info::ServingParallelUnitMappings(FragmentParallelUnitMappings{ mappings: to_fragment_parallel_unit_mapping(&upserted) }));
+                                        notification_manager.notify_frontend_without_version(Operation::Update, Info::ServingWorkerSlotMappings(FragmentWorkerSlotMappings{ mappings: to_fragment_worker_slot_mapping(&upserted) }));
                                     }
                                     if !failed.is_empty() {
                                         tracing::debug!("Fail to update serving vnode mapping for fragments {:?}.", failed);
-                                        notification_manager.notify_frontend_without_version(Operation::Delete, Info::ServingParallelUnitMappings(FragmentParallelUnitMappings{ mappings: to_deleted_fragment_parallel_unit_mapping(&failed)}));
+                                        notification_manager.notify_frontend_without_version(Operation::Delete, Info::ServingWorkerSlotMappings(FragmentWorkerSlotMappings{ mappings: to_deleted_fragment_worker_slot_mapping(&failed)}));
                                     }
                                 }
                                 LocalNotification::FragmentMappingsDelete(fragment_ids) => {
@@ -208,7 +208,7 @@ pub async fn start_serving_vnode_mapping_worker(
                                     }
                                     tracing::debug!("Delete serving vnode mapping for fragments {:?}.", fragment_ids);
                                     serving_vnode_mapping.remove(&fragment_ids);
-                                    notification_manager.notify_frontend_without_version(Operation::Delete, Info::ServingParallelUnitMappings(FragmentParallelUnitMappings{ mappings: to_deleted_fragment_parallel_unit_mapping(&fragment_ids) }));
+                                    notification_manager.notify_frontend_without_version(Operation::Delete, Info::ServingWorkerSlotMappings(FragmentWorkerSlotMappings{ mappings: to_deleted_fragment_worker_slot_mapping(&fragment_ids) }));
                                 }
                                 _ => {}
                             }
diff --git a/src/meta/src/storage/etcd_meta_store.rs b/src/meta/src/storage/etcd_meta_store.rs
index 8c5f93f9dd9ac..1986dab6247d6 100644
--- a/src/meta/src/storage/etcd_meta_store.rs
+++ b/src/meta/src/storage/etcd_meta_store.rs
@@ -14,7 +14,6 @@
 
 use std::sync::atomic::{self, AtomicI64};
 
-use anyhow;
 use async_trait::async_trait;
 use etcd_client::{Compare, CompareOp, Error as EtcdError, GetOptions, Txn, TxnOp};
 use futures::Future;
diff --git a/src/meta/src/storage/transaction.rs b/src/meta/src/storage/transaction.rs
index 9a9df92b5f833..daac28f276049 100644
--- a/src/meta/src/storage/transaction.rs
+++ b/src/meta/src/storage/transaction.rs
@@ -17,9 +17,9 @@ use crate::storage::{ColumnFamily, Key, Value};
 /// A `Transaction` executes several writes(aka. operations) to meta store atomically with optional
 /// preconditions checked. It executes as follow:
 /// 1. If all `preconditions` are valid, all `operations` are executed; Otherwise no operation
-/// is executed.
+///    is executed.
 /// 2. Upon `commit` the transaction, the `TransactionAbort` error will be returned if
-/// any precondition was not met in previous step.
+///    any precondition was not met in previous step.
 #[derive(Default)]
 pub struct Transaction {
     preconditions: Vec<Precondition>,
diff --git a/src/meta/src/stream/mod.rs b/src/meta/src/stream/mod.rs
index 8f97f1a4634de..57ca9896257f3 100644
--- a/src/meta/src/stream/mod.rs
+++ b/src/meta/src/stream/mod.rs
@@ -21,8 +21,36 @@ mod stream_manager;
 mod test_fragmenter;
 mod test_scale;
 
+use std::collections::HashMap;
+
+use risingwave_common::catalog::TableId;
+use risingwave_pb::stream_plan::StreamActor;
+use risingwave_pb::stream_service::build_actor_info::SubscriptionIds;
+use risingwave_pb::stream_service::BuildActorInfo;
 pub use scale::*;
 pub use sink::*;
 pub use source_manager::*;
 pub use stream_graph::*;
 pub use stream_manager::*;
+
+pub(crate) fn to_build_actor_info(
+    actor: StreamActor,
+    subscriptions: &HashMap<TableId, HashMap<u32, u64>>,
+    fragment_table_id: TableId,
+) -> BuildActorInfo {
+    BuildActorInfo {
+        actor: Some(actor),
+        related_subscriptions: subscriptions
+            .get(&fragment_table_id)
+            .into_iter()
+            .map(|subscriptions| {
+                (
+                    fragment_table_id.table_id,
+                    SubscriptionIds {
+                        subscription_ids: subscriptions.keys().cloned().collect(),
+                    },
+                )
+            })
+            .collect(),
+    }
+}
diff --git a/src/meta/src/stream/scale.rs b/src/meta/src/stream/scale.rs
index a1c0aaa735fe1..0b7a711b38643 100644
--- a/src/meta/src/stream/scale.rs
+++ b/src/meta/src/stream/scale.rs
@@ -41,11 +41,13 @@ use risingwave_pb::meta::table_fragments::fragment::{
     FragmentDistributionType, PbFragmentDistributionType,
 };
 use risingwave_pb::meta::table_fragments::{self, ActorStatus, PbFragment, State};
-use risingwave_pb::meta::FragmentParallelUnitMappings;
+use risingwave_pb::meta::FragmentWorkerSlotMappings;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
 use risingwave_pb::stream_plan::{
     Dispatcher, DispatcherType, FragmentTypeFlag, PbStreamActor, StreamNode,
 };
+use risingwave_pb::stream_service::build_actor_info::SubscriptionIds;
+use risingwave_pb::stream_service::BuildActorInfo;
 use thiserror_ext::AsReport;
 use tokio::sync::oneshot::Receiver;
 use tokio::sync::{oneshot, RwLock, RwLockReadGuard, RwLockWriteGuard};
@@ -59,8 +61,7 @@ use crate::manager::{
 };
 use crate::model::{ActorId, DispatcherId, FragmentId, TableFragments, TableParallelism};
 use crate::serving::{
-    to_deleted_fragment_parallel_unit_mapping, to_fragment_parallel_unit_mapping,
-    ServingVnodeMapping,
+    to_deleted_fragment_worker_slot_mapping, to_fragment_worker_slot_mapping, ServingVnodeMapping,
 };
 use crate::storage::{MetaStore, MetaStoreError, MetaStoreRef, Transaction, DEFAULT_COLUMN_FAMILY};
 use crate::stream::{GlobalStreamManager, SourceManagerRef};
@@ -236,19 +237,19 @@ impl RescheduleContext {
 /// The specific process is as follows
 ///
 /// 1. Calculate the number of target actors, and calculate the average value and the remainder, and
-/// use the average value as expected.
+///    use the average value as expected.
 ///
 /// 2. Filter out the actor to be removed and the actor to be retained, and sort them from largest
-/// to smallest (according to the number of virtual nodes held).
+///    to smallest (according to the number of virtual nodes held).
 ///
 /// 3. Calculate their balance, 1) For the actors to be removed, the number of virtual nodes per
-/// actor is the balance. 2) For retained actors, the number of virtual nodes - expected is the
-/// balance. 3) For newly created actors, -expected is the balance (always negative).
+///    actor is the balance. 2) For retained actors, the number of virtual nodes - expected is the
+///    balance. 3) For newly created actors, -expected is the balance (always negative).
 ///
 /// 4. Allocate the remainder, high priority to newly created nodes.
 ///
 /// 5. After that, merge removed, retained and created into a queue, with the head of the queue
-/// being the source, and move the virtual nodes to the destination at the end of the queue.
+///    being the source, and move the virtual nodes to the destination at the end of the queue.
 ///
 /// This can handle scale in, scale out, migration, and simultaneous scaling with as much affinity
 /// as possible.
@@ -824,7 +825,7 @@ impl ScaleController {
         &self,
         worker_nodes: &HashMap<WorkerId, WorkerNode>,
         actor_infos_to_broadcast: BTreeMap<ActorId, ActorInfo>,
-        node_actors_to_create: HashMap<WorkerId, Vec<PbStreamActor>>,
+        node_actors_to_create: HashMap<WorkerId, Vec<BuildActorInfo>>,
         broadcast_worker_ids: HashSet<WorkerId>,
     ) -> MetaResult<()> {
         self.stream_rpc_manager
@@ -846,7 +847,7 @@ impl ScaleController {
                             *node_id,
                             stream_actors
                                 .iter()
-                                .map(|stream_actor| stream_actor.actor_id)
+                                .map(|stream_actor| stream_actor.actor.as_ref().unwrap().actor_id)
                                 .collect_vec(),
                         )
                     }),
@@ -1124,9 +1125,24 @@ impl ScaleController {
             // After modification, for newly created actors, both upstream and downstream actor ids
             // have been modified
             let mut actor_infos_to_broadcast = BTreeMap::new();
-            let mut node_actors_to_create: HashMap<WorkerId, Vec<_>> = HashMap::new();
+            let mut node_actors_to_create: HashMap<WorkerId, Vec<BuildActorInfo>> = HashMap::new();
             let mut broadcast_worker_ids = HashSet::new();
 
+            let subscriptions: HashMap<_, SubscriptionIds> = self
+                .metadata_manager
+                .get_mv_depended_subscriptions()
+                .await?
+                .iter()
+                .map(|(table_id, subscriptions)| {
+                    (
+                        table_id.table_id,
+                        SubscriptionIds {
+                            subscription_ids: subscriptions.keys().cloned().collect(),
+                        },
+                    )
+                })
+                .collect();
+
             for actors_to_create in fragment_actors_to_create.values() {
                 for (new_actor_id, new_parallel_unit_id) in actors_to_create {
                     let new_actor = new_created_actors.get(new_actor_id).unwrap();
@@ -1188,7 +1204,12 @@ impl ScaleController {
                     node_actors_to_create
                         .entry(worker.id)
                         .or_default()
-                        .push(new_actor.clone());
+                        .push(BuildActorInfo {
+                            actor: Some(new_actor.clone()),
+                            // TODO: may include only the subscriptions related to the table fragment
+                            // of the actor.
+                            related_subscriptions: subscriptions.clone(),
+                        });
 
                     broadcast_worker_ids.insert(worker.id);
 
@@ -1703,8 +1724,8 @@ impl ScaleController {
                     .notification_manager()
                     .notify_frontend_without_version(
                         Operation::Update,
-                        Info::ServingParallelUnitMappings(FragmentParallelUnitMappings {
-                            mappings: to_fragment_parallel_unit_mapping(&upserted),
+                        Info::ServingWorkerSlotMappings(FragmentWorkerSlotMappings {
+                            mappings: to_fragment_worker_slot_mapping(&upserted),
                         }),
                     );
             }
@@ -1717,8 +1738,8 @@ impl ScaleController {
                     .notification_manager()
                     .notify_frontend_without_version(
                         Operation::Delete,
-                        Info::ServingParallelUnitMappings(FragmentParallelUnitMappings {
-                            mappings: to_deleted_fragment_parallel_unit_mapping(&failed),
+                        Info::ServingWorkerSlotMappings(FragmentWorkerSlotMappings {
+                            mappings: to_deleted_fragment_worker_slot_mapping(&failed),
                         }),
                     );
             }
@@ -2504,9 +2525,7 @@ impl ScaleController {
         // We trace the upstreams of each downstream under the hierarchy until we reach the top
         // for every no_shuffle relation.
         while let Some(fragment_id) = queue.pop_front() {
-            if !no_shuffle_target_fragment_ids.contains(&fragment_id)
-                && !no_shuffle_source_fragment_ids.contains(&fragment_id)
-            {
+            if !no_shuffle_target_fragment_ids.contains(&fragment_id) {
                 continue;
             }
 
@@ -2573,9 +2592,7 @@ impl ScaleController {
         // We trace the upstreams of each downstream under the hierarchy until we reach the top
         // for every no_shuffle relation.
         while let Some(fragment_id) = queue.pop_front() {
-            if !no_shuffle_target_fragment_ids.contains(&fragment_id)
-                && !no_shuffle_source_fragment_ids.contains(&fragment_id)
-            {
+            if !no_shuffle_target_fragment_ids.contains(&fragment_id) {
                 continue;
             }
 
diff --git a/src/meta/src/stream/stream_graph/actor.rs b/src/meta/src/stream/stream_graph/actor.rs
index 80004aa48eaa7..7b02920ac3c4d 100644
--- a/src/meta/src/stream/stream_graph/actor.rs
+++ b/src/meta/src/stream/stream_graph/actor.rs
@@ -121,7 +121,7 @@ impl ActorBuilder {
     ///
     /// During this process, the following things will be done:
     /// 1. Replace the logical `Exchange` in node's input with `Merge`, which can be executed on the
-    /// compute nodes.
+    ///    compute nodes.
     /// 2. Fill the upstream mview info of the `Merge` node under the other "leaf" nodes.
     fn rewrite(&self) -> MetaResult<StreamNode> {
         self.rewrite_inner(&self.nodes, 0)
diff --git a/src/meta/src/stream/stream_graph/fragment.rs b/src/meta/src/stream/stream_graph/fragment.rs
index af9e233fb2984..e347dd0287f36 100644
--- a/src/meta/src/stream/stream_graph/fragment.rs
+++ b/src/meta/src/stream/stream_graph/fragment.rs
@@ -148,11 +148,6 @@ impl BuildingFragment {
 
                 has_table = true;
             }
-            NodeBody::Subscription(subscription_node) => {
-                subscription_node.subscription_catalog.as_mut().unwrap().id = table_id;
-
-                has_table = true;
-            }
             NodeBody::Dml(dml_node) => {
                 dml_node.table_id = table_id;
                 dml_node.table_version_id = job.table_version_id().unwrap();
@@ -666,10 +661,7 @@ impl CompleteStreamFragmentGraph {
 
                             (source_job_id, edge)
                         }
-                        DdlType::MaterializedView
-                        | DdlType::Sink
-                        | DdlType::Index
-                        | DdlType::Subscription => {
+                        DdlType::MaterializedView | DdlType::Sink | DdlType::Index => {
                             // handle MV on MV/Source
 
                             // Build the extra edges between the upstream `Materialize` and the downstream `StreamScan`
diff --git a/src/meta/src/stream/stream_manager.rs b/src/meta/src/stream/stream_manager.rs
index 6a801932785c7..d582095367fdc 100644
--- a/src/meta/src/stream/stream_manager.rs
+++ b/src/meta/src/stream/stream_manager.rs
@@ -19,7 +19,7 @@ use futures::future::join_all;
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
 use risingwave_meta_model_v2::ObjectId;
-use risingwave_pb::catalog::{CreateType, Table};
+use risingwave_pb::catalog::{CreateType, Subscription, Table};
 use risingwave_pb::stream_plan::update_mutation::MergeUpdate;
 use risingwave_pb::stream_plan::Dispatcher;
 use thiserror_ext::AsReport;
@@ -31,7 +31,7 @@ use super::{Locations, RescheduleOptions, ScaleControllerRef, TableResizePolicy}
 use crate::barrier::{BarrierScheduler, Command, ReplaceTablePlan, StreamRpcManager};
 use crate::manager::{DdlType, MetaSrvEnv, MetadataManager, StreamingJob};
 use crate::model::{ActorId, MetadataModel, TableFragments, TableParallelism};
-use crate::stream::SourceManagerRef;
+use crate::stream::{to_build_actor_info, SourceManagerRef};
 use crate::{MetaError, MetaResult};
 
 pub type GlobalStreamManagerRef = Arc<GlobalStreamManager>;
@@ -219,9 +219,9 @@ impl GlobalStreamManager {
     /// Create streaming job, it works as follows:
     ///
     /// 1. Broadcast the actor info based on the scheduling result in the context, build the hanging
-    /// channels in upstream worker nodes.
+    ///    channels in upstream worker nodes.
     /// 2. (optional) Get the split information of the `StreamSource` via source manager and patch
-    /// actors.
+    ///    actors.
     /// 3. Notify related worker nodes to update and build the actors.
     /// 4. Store related meta data.
     pub async fn create_streaming_job(
@@ -346,6 +346,10 @@ impl GlobalStreamManager {
             .chain(existing_locations.actor_infos());
 
         let building_worker_actors = building_locations.worker_actors();
+        let subscriptions = self
+            .metadata_manager
+            .get_mv_depended_subscriptions()
+            .await?;
 
         // We send RPC request in two stages.
         // The first stage does 2 things: broadcast actor info, and send local actor ids to
@@ -359,7 +363,13 @@ impl GlobalStreamManager {
                 building_worker_actors.iter().map(|(worker_id, actors)| {
                     let stream_actors = actors
                         .iter()
-                        .map(|actor_id| actor_map[actor_id].clone())
+                        .map(|actor_id| {
+                            to_build_actor_info(
+                                actor_map[actor_id].clone(),
+                                &subscriptions,
+                                table_fragments.table_id(),
+                            )
+                        })
                         .collect::<Vec<_>>();
                     (*worker_id, stream_actors)
                 }),
@@ -731,17 +741,49 @@ impl GlobalStreamManager {
 
         Ok(())
     }
+
+    // Dont need add actor, just send a command
+    pub async fn create_subscription(
+        self: &Arc<Self>,
+        subscription: &Subscription,
+    ) -> MetaResult<()> {
+        let command = Command::CreateSubscription {
+            subscription_id: subscription.id,
+            upstream_mv_table_id: TableId::new(subscription.dependent_table_id),
+            retention_second: subscription.retention_seconds,
+        };
+
+        tracing::debug!("sending Command::CreateSubscription");
+        self.barrier_scheduler.run_command(command).await?;
+        Ok(())
+    }
+
+    // Dont need add actor, just send a command
+    pub async fn drop_subscription(self: &Arc<Self>, subscription_id: u32, table_id: u32) {
+        let command = Command::DropSubscription {
+            subscription_id,
+            upstream_mv_table_id: TableId::new(table_id),
+        };
+
+        tracing::debug!("sending Command::DropSubscription");
+        let _ = self
+            .barrier_scheduler
+            .run_command(command)
+            .await
+            .inspect_err(|err| {
+                tracing::error!(error = ?err.as_report(), "failed to run drop command");
+            });
+    }
 }
 
 #[cfg(test)]
 mod tests {
-    use std::collections::{BTreeMap, HashMap, HashSet};
+    use std::collections::BTreeMap;
     use std::net::SocketAddr;
-    use std::sync::{Arc, Mutex};
+    use std::sync::Mutex;
     use std::time::Duration;
 
     use futures::{Stream, TryStreamExt};
-    use risingwave_common::catalog::TableId;
     use risingwave_common::hash::ParallelUnitMapping;
     use risingwave_common::system_param::reader::SystemParamsRead;
     use risingwave_pb::common::{HostAddress, WorkerType};
@@ -754,10 +796,7 @@ mod tests {
         StreamService, StreamServiceServer,
     };
     use risingwave_pb::stream_service::streaming_control_stream_response::InitResponse;
-    use risingwave_pb::stream_service::{
-        BroadcastActorInfoTableResponse, BuildActorsResponse, DropActorsRequest,
-        DropActorsResponse, UpdateActorsResponse, *,
-    };
+    use risingwave_pb::stream_service::*;
     use tokio::spawn;
     use tokio::sync::mpsc::unbounded_channel;
     use tokio::sync::oneshot::Sender;
@@ -769,14 +808,14 @@ mod tests {
     use tonic::{Request, Response, Status, Streaming};
 
     use super::*;
-    use crate::barrier::{GlobalBarrierManager, StreamRpcManager};
+    use crate::barrier::GlobalBarrierManager;
     use crate::hummock::{CompactorManager, HummockManager};
     use crate::manager::sink_coordination::SinkCoordinatorManager;
     use crate::manager::{
         CatalogManager, CatalogManagerRef, ClusterManager, FragmentManager, FragmentManagerRef,
-        MetaSrvEnv, RelationIdEnum, StreamingClusterInfo,
+        RelationIdEnum, StreamingClusterInfo,
     };
-    use crate::model::{ActorId, FragmentId};
+    use crate::model::FragmentId;
     use crate::rpc::ddl_controller::DropMode;
     use crate::rpc::metrics::MetaMetrics;
     use crate::stream::{ScaleController, SourceManager};
@@ -804,6 +843,7 @@ mod tests {
             let req = request.into_inner();
             let mut guard = self.inner.actor_streams.lock().unwrap();
             for actor in req.get_actors() {
+                let actor = actor.actor.as_ref().unwrap();
                 guard.insert(actor.get_actor_id(), actor.clone());
             }
 
diff --git a/src/meta/src/stream/test_fragmenter.rs b/src/meta/src/stream/test_fragmenter.rs
index c0fc385ad96fb..f2d9a43241729 100644
--- a/src/meta/src/stream/test_fragmenter.rs
+++ b/src/meta/src/stream/test_fragmenter.rs
@@ -74,6 +74,7 @@ fn make_sum_aggcall(idx: u32) -> AggCall {
         order_by: vec![],
         filter: None,
         direct_args: vec![],
+        udf: None,
     }
 }
 
@@ -344,9 +345,7 @@ fn make_stream_fragments() -> Vec<StreamFragment> {
                 make_inputref(0),
                 make_inputref(1),
             ],
-            watermark_input_cols: vec![],
-            watermark_output_cols: vec![],
-            nondecreasing_exprs: vec![],
+            ..Default::default()
         })),
         fields: vec![], // TODO: fill this later
         input: vec![simple_agg_node_1],
diff --git a/src/object_store/Cargo.toml b/src/object_store/Cargo.toml
index e4d573e8f4d2d..43a812081a36c 100644
--- a/src/object_store/Cargo.toml
+++ b/src/object_store/Cargo.toml
@@ -8,6 +8,9 @@ license = { workspace = true }
 repository = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[lints]
+workspace = true
+
 [dependencies]
 async-trait = "0.1"
 await-tree = { workspace = true }
@@ -23,14 +26,14 @@ crc32fast = "1"
 either = "1"
 fail = "0.5"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
-hyper = { version = "0.14", features = ["tcp", "client"] } # required by aws sdk
+hyper = { version = "0.14", features = ["tcp", "client"] }                    # required by aws sdk
 hyper-rustls = { version = "0.24.2", features = ["webpki-roots"] }
 hyper-tls = "0.5.0"
 itertools = { workspace = true }
-madsim = "0.2.22"
+madsim = "0.2.27"
 opendal = "0.45.1"
 prometheus = { version = "0.13", features = ["process"] }
-reqwest = "0.11" # required by opendal
+reqwest = "0.11"                                                              # required by opendal
 risingwave_common = { workspace = true }
 rustls = "0.23.5"
 spin = "0.9"
diff --git a/src/object_store/src/object/error.rs b/src/object_store/src/object/error.rs
index d4e8e0bebe155..aa79f53c4c065 100644
--- a/src/object_store/src/object/error.rs
+++ b/src/object_store/src/object/error.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::io;
-use std::marker::{Send, Sync};
 
 use aws_sdk_s3::operation::get_object::GetObjectError;
 use aws_sdk_s3::operation::head_object::HeadObjectError;
@@ -27,8 +26,13 @@ use tokio::sync::oneshot::error::RecvError;
 #[derive(Error, thiserror_ext::ReportDebug, thiserror_ext::Box, thiserror_ext::Construct)]
 #[thiserror_ext(newtype(name = ObjectError, backtrace))]
 pub enum ObjectErrorInner {
-    #[error("s3 error: {0}")]
-    S3(#[source] BoxedError),
+    #[error("s3 error: {inner}")]
+    S3 {
+        // TODO: remove this after switch s3 backend to opendal
+        should_retry: bool,
+        #[source]
+        inner: BoxedError,
+    },
     #[error("disk error: {msg}")]
     Disk {
         msg: String,
@@ -45,6 +49,9 @@ pub enum ObjectErrorInner {
     #[cfg(madsim)]
     #[error(transparent)]
     Sim(#[from] crate::object::sim::SimError),
+
+    #[error("Timeout error: {0}")]
+    Timeout(String),
 }
 
 impl ObjectError {
@@ -55,17 +62,20 @@ impl ObjectError {
     /// Tells whether the error indicates the target object is not found.
     pub fn is_object_not_found_error(&self) -> bool {
         match self.inner() {
-            ObjectErrorInner::S3(e) => {
-                if let Some(aws_smithy_runtime_api::client::result::SdkError::ServiceError(err)) = e
-                    .downcast_ref::<aws_smithy_runtime_api::client::result::SdkError<
+            ObjectErrorInner::S3 {
+                inner,
+                should_retry: _,
+            } => {
+                if let Some(aws_smithy_runtime_api::client::result::SdkError::ServiceError(err)) =
+                    inner.downcast_ref::<aws_smithy_runtime_api::client::result::SdkError<
                         GetObjectError,
                         aws_smithy_runtime_api::http::Response<SdkBody>,
                     >>()
                 {
                     return matches!(err.err(), GetObjectError::NoSuchKey(_));
                 }
-                if let Some(aws_smithy_runtime_api::client::result::SdkError::ServiceError(err)) = e
-                    .downcast_ref::<aws_smithy_runtime_api::client::result::SdkError<
+                if let Some(aws_smithy_runtime_api::client::result::SdkError::ServiceError(err)) =
+                    inner.downcast_ref::<aws_smithy_runtime_api::client::result::SdkError<
                         HeadObjectError,
                         aws_smithy_runtime_api::http::Response<SdkBody>,
                     >>()
@@ -90,6 +100,21 @@ impl ObjectError {
         };
         false
     }
+
+    pub fn should_retry(&self) -> bool {
+        match self.inner() {
+            ObjectErrorInner::S3 {
+                inner: _,
+                should_retry,
+            } => *should_retry,
+
+            ObjectErrorInner::Opendal(e) => e.is_temporary(),
+
+            ObjectErrorInner::Timeout(_) => true,
+
+            _ => false,
+        }
+    }
 }
 
 impl<E, R> From<aws_smithy_runtime_api::client::result::SdkError<E, R>> for ObjectError
@@ -98,7 +123,11 @@ where
     R: Send + Sync + 'static + std::fmt::Debug,
 {
     fn from(e: aws_smithy_runtime_api::client::result::SdkError<E, R>) -> Self {
-        ObjectErrorInner::S3(e.into()).into()
+        ObjectErrorInner::S3 {
+            inner: e.into(),
+            should_retry: false,
+        }
+        .into()
     }
 }
 
@@ -110,7 +139,11 @@ impl From<RecvError> for ObjectError {
 
 impl From<ByteStreamError> for ObjectError {
     fn from(e: ByteStreamError) -> Self {
-        ObjectErrorInner::Internal(e.to_report_string()).into()
+        ObjectErrorInner::S3 {
+            inner: e.into(),
+            should_retry: true,
+        }
+        .into()
     }
 }
 
diff --git a/src/object_store/src/object/mem.rs b/src/object_store/src/object/mem.rs
index 3a1a7ed655e81..6bc78ae8b198d 100644
--- a/src/object_store/src/object/mem.rs
+++ b/src/object_store/src/object/mem.rs
@@ -28,8 +28,7 @@ use thiserror::Error;
 use tokio::sync::Mutex;
 
 use super::{
-    BoxedStreamingUploader, ObjectError, ObjectMetadata, ObjectRangeBounds, ObjectResult,
-    ObjectStore, StreamingUploader,
+    ObjectError, ObjectMetadata, ObjectRangeBounds, ObjectResult, ObjectStore, StreamingUploader,
 };
 use crate::object::{ObjectDataStream, ObjectMetadataIter};
 
@@ -64,7 +63,6 @@ pub struct InMemStreamingUploader {
     objects: Arc<Mutex<HashMap<String, (ObjectMetadata, Bytes)>>>,
 }
 
-#[async_trait::async_trait]
 impl StreamingUploader for InMemStreamingUploader {
     async fn write_bytes(&mut self, data: Bytes) -> ObjectResult<()> {
         fail_point!("mem_write_bytes_err", |_| Err(ObjectError::internal(
@@ -74,7 +72,7 @@ impl StreamingUploader for InMemStreamingUploader {
         Ok(())
     }
 
-    async fn finish(self: Box<Self>) -> ObjectResult<()> {
+    async fn finish(self) -> ObjectResult<()> {
         fail_point!("mem_finish_streaming_upload_err", |_| Err(
             ObjectError::internal("mem finish streaming upload error")
         ));
@@ -101,6 +99,8 @@ pub struct InMemObjectStore {
 
 #[async_trait::async_trait]
 impl ObjectStore for InMemObjectStore {
+    type StreamingUploader = InMemStreamingUploader;
+
     fn get_object_prefix(&self, _obj_id: u64) -> String {
         String::default()
     }
@@ -121,12 +121,12 @@ impl ObjectStore for InMemObjectStore {
         }
     }
 
-    async fn streaming_upload(&self, path: &str) -> ObjectResult<BoxedStreamingUploader> {
-        Ok(Box::new(InMemStreamingUploader {
+    async fn streaming_upload(&self, path: &str) -> ObjectResult<Self::StreamingUploader> {
+        Ok(InMemStreamingUploader {
             path: path.to_string(),
             buf: BytesMut::new(),
             objects: self.objects.clone(),
-        }))
+        })
     }
 
     async fn read(&self, path: &str, range: impl ObjectRangeBounds) -> ObjectResult<Bytes> {
@@ -307,7 +307,6 @@ impl Stream for InMemObjectIter {
 
 #[cfg(test)]
 mod tests {
-    use bytes::Bytes;
     use futures::TryStreamExt;
     use itertools::enumerate;
 
diff --git a/src/object_store/src/object/mod.rs b/src/object_store/src/object/mod.rs
index a623e4b116fd2..5369d914a7514 100644
--- a/src/object_store/src/object/mod.rs
+++ b/src/object_store/src/object/mod.rs
@@ -12,14 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(madsim)]
 pub mod sim;
 use std::ops::{Range, RangeBounds};
 use std::sync::Arc;
 use std::time::Duration;
 
 use bytes::Bytes;
-use prometheus::HistogramTimer;
 
 pub mod mem;
 pub use mem::*;
@@ -30,7 +28,7 @@ pub use opendal_engine::*;
 pub mod s3;
 use await_tree::InstrumentAwait;
 use futures::stream::BoxStream;
-use futures::StreamExt;
+use futures::{Future, StreamExt};
 pub use risingwave_common::config::ObjectStoreConfig;
 pub use s3::*;
 
@@ -42,31 +40,16 @@ mod prefix;
 pub use error::*;
 use object_metrics::ObjectStoreMetrics;
 use thiserror_ext::AsReport;
+use tokio_retry::strategy::{jitter, ExponentialBackoff};
 
 #[cfg(madsim)]
 use self::sim::SimObjectStore;
 
 pub type ObjectStoreRef = Arc<ObjectStoreImpl>;
-pub type ObjectStreamingUploader = MonitoredStreamingUploader;
-
-type BoxedStreamingUploader = Box<dyn StreamingUploader>;
+pub type ObjectStreamingUploader = StreamingUploaderImpl;
 
 pub trait ObjectRangeBounds = RangeBounds<usize> + Clone + Send + Sync + std::fmt::Debug + 'static;
 
-/// Partitions a set of given paths into two vectors. The first vector contains all local paths, and
-/// the second contains all remote paths.
-fn partition_object_store_paths(paths: &[String]) -> Vec<String> {
-    // ToDo: Currently the result is a copy of the input. Would it be worth it to use an in-place
-    //       partition instead?
-    let mut vec_rem = vec![];
-
-    for path in paths {
-        vec_rem.push(path.to_string());
-    }
-
-    vec_rem
-}
-
 #[derive(Debug, Clone, PartialEq)]
 pub struct ObjectMetadata {
     // Full path
@@ -76,11 +59,12 @@ pub struct ObjectMetadata {
     pub total_size: usize,
 }
 
-#[async_trait::async_trait]
 pub trait StreamingUploader: Send {
+    #[expect(async_fn_in_trait)]
     async fn write_bytes(&mut self, data: Bytes) -> ObjectResult<()>;
 
-    async fn finish(self: Box<Self>) -> ObjectResult<()>;
+    #[expect(async_fn_in_trait)]
+    async fn finish(self) -> ObjectResult<()>;
 
     fn get_memory_usage(&self) -> u64;
 }
@@ -88,13 +72,14 @@ pub trait StreamingUploader: Send {
 /// The implementation must be thread-safe.
 #[async_trait::async_trait]
 pub trait ObjectStore: Send + Sync {
+    type StreamingUploader: StreamingUploader;
     /// Get the key prefix for object
     fn get_object_prefix(&self, obj_id: u64) -> String;
 
     /// Uploads the object to `ObjectStore`.
     async fn upload(&self, path: &str, obj: Bytes) -> ObjectResult<()>;
 
-    async fn streaming_upload(&self, path: &str) -> ObjectResult<BoxedStreamingUploader>;
+    async fn streaming_upload(&self, path: &str) -> ObjectResult<Self::StreamingUploader>;
 
     /// If objects are PUT using a multipart upload, it's a good practice to GET them in the same
     /// part sizes (or at least aligned to part boundaries) for best performance.
@@ -123,7 +108,7 @@ pub trait ObjectStore: Send + Sync {
     fn monitored(
         self,
         metrics: Arc<ObjectStoreMetrics>,
-        config: ObjectStoreConfig,
+        config: Arc<ObjectStoreConfig>,
     ) -> MonitoredObjectStore<Self>
     where
         Self: Sized,
@@ -134,94 +119,182 @@ pub trait ObjectStore: Send + Sync {
     async fn list(&self, prefix: &str) -> ObjectResult<ObjectMetadataIter>;
 
     fn store_media_type(&self) -> &'static str;
+
+    fn support_streaming_upload(&self) -> bool {
+        true
+    }
 }
 
-pub enum ObjectStoreImpl {
-    InMem(MonitoredObjectStore<InMemObjectStore>),
-    Opendal(MonitoredObjectStore<OpendalObjectStore>),
-    S3(MonitoredObjectStore<S3ObjectStore>),
-    #[cfg(madsim)]
-    Sim(MonitoredObjectStore<SimObjectStore>),
+#[cfg(not(madsim))]
+macro_rules! for_all_object_store {
+    ($macro:ident $($args:tt)*) => {
+        $macro! {
+            {
+                { InMem, InMemObjectStore },
+                { Opendal, OpendalObjectStore },
+                { S3, S3ObjectStore }
+            }
+            $($args)*
+        }
+    }
 }
 
-macro_rules! dispatch_async {
-    ($object_store:expr, $method_name:ident $(, $args:expr)*) => {
-        $object_store.$method_name($($args, )*).await
+#[cfg(madsim)]
+macro_rules! for_all_object_store {
+    ($macro:ident $($args:tt)*) => {
+        $macro! {
+            {
+                { InMem, InMemObjectStore },
+                { Opendal, OpendalObjectStore },
+                { S3, S3ObjectStore },
+                { Sim, SimObjectStore }
+            }
+            $($args)*
+        }
     }
 }
 
-/// This macro routes the object store operation to the real implementation by the `ObjectStoreImpl`
-/// enum type and the `path`.
-///
-/// Except for `InMem`,the operation should be performed on remote object store.
-macro_rules! object_store_impl_method_body {
-    ($object_store:expr, $method_name:ident, $dispatch_macro:ident, $path:expr $(, $args:expr)*) => {
+macro_rules! enum_map {
+    (
         {
-            let path = $path;
-            match $object_store {
-                ObjectStoreImpl::InMem(in_mem) => {
-                    $dispatch_macro!(in_mem, $method_name, path $(, $args)*)
-                },
-                ObjectStoreImpl::Opendal(opendal) => {
-                    $dispatch_macro!(opendal, $method_name, path $(, $args)*)
-                },
-                ObjectStoreImpl::S3(s3) => {
-                    $dispatch_macro!(s3, $method_name, path $(, $args)*)
-                },
-                #[cfg(madsim)]
-                ObjectStoreImpl::Sim(in_mem) => {
-                    $dispatch_macro!(in_mem, $method_name, path $(, $args)*)
+            $(
+                {$variant:ident, $_type_name:ty}
+            ),*
+        },
+        $object_store:expr,
+        $var_name:ident,
+        $func:expr
+    ) => {
+        match $object_store {
+            $(
+                ObjectStoreEnum::$variant($var_name) => ObjectStoreEnum::$variant({
+                    $func
+                }),
+            )*
+        }
+    };
+    ($object_store:expr, |$var_name:ident| $func:expr) => {
+        for_all_object_store! {
+            enum_map, $object_store, $var_name, $func
+        }
+    };
+}
+
+macro_rules! dispatch_object_store_enum {
+    (
+        {
+            $(
+                {$variant:ident, $_type_name:ty}
+            ),*
+        },
+        $object_store:expr,
+        $var_name:ident,
+        $func:expr
+    ) => {
+        match $object_store {
+            $(
+                ObjectStoreEnum::$variant($var_name) => {
+                    $func
                 },
-            }
+            )*
+        }
+    };
+    ($object_store:expr, |$var_name:ident| $func:expr) => {
+        for_all_object_store! {
+            dispatch_object_store_enum, $object_store, $var_name, $func
+        }
+    };
+}
 
+macro_rules! define_object_store_impl {
+    () => {
+        for_all_object_store! {
+            define_object_store_impl
         }
     };
+    (
+        {$(
+            {$variant:ident, $type_name:ty}
+        ),*}
+    ) => {
+        pub enum ObjectStoreEnum<
+            $($variant),*
+        > {
+            $(
+                $variant($variant),
+            )*
+        }
+
+        pub type ObjectStoreImpl = ObjectStoreEnum<
+            $(
+                MonitoredObjectStore<$type_name>,
+            )*
+        >;
+
+        pub type StreamingUploaderImpl = ObjectStoreEnum<
+            $(
+                MonitoredStreamingUploader<<$type_name as ObjectStore>::StreamingUploader>
+            ),*
+        >;
+    };
 }
 
+define_object_store_impl!();
+
 /// This macro routes the object store operation to the real implementation by the `ObjectStoreImpl`
-/// enum type and the `paths`. It is a modification of the macro above to work with a slice of
-/// strings instead of just a single one.
+/// enum type and the `path`.
 ///
-/// Except for `InMem`, the operation should be performed on remote object store.
-macro_rules! object_store_impl_method_body_slice {
-    ($object_store:expr, $method_name:ident, $dispatch_macro:ident, $paths:expr $(, $args:expr)*) => {
+/// Except for `InMem`,the operation should be performed on remote object store.
+macro_rules! object_store_impl_method_body {
+    // with await
+    ($object_store:expr, $method_name:ident ($($args:expr),*).await) => {
         {
-            let paths_rem = partition_object_store_paths($paths);
-
-            match $object_store {
-                ObjectStoreImpl::InMem(in_mem) => {
-                    $dispatch_macro!(in_mem, $method_name, &paths_rem $(, $args)*)
-                },
-                ObjectStoreImpl::Opendal(opendal) => {
-                    $dispatch_macro!(opendal, $method_name, &paths_rem $(, $args)*)
-                },
-                ObjectStoreImpl::S3(s3) => {
-                    $dispatch_macro!(s3, $method_name, &paths_rem $(, $args)*)
-                },
-                #[cfg(madsim)]
-                ObjectStoreImpl::Sim(in_mem) => {
-                    $dispatch_macro!(in_mem, $method_name, &paths_rem $(, $args)*)
-                },
-            }
+            dispatch_object_store_enum! {$object_store, |os| {
+                os.$method_name($($args),*).await
+            }}
+        }
+    };
+    // no await
+    ($object_store:expr, $method_name:ident ($(, $args:expr)*)) => {
+        {
+            dispatch_object_store_enum! {$object_store, |os| {
+                os.$method_name($($args),*)
+            }}
         }
     };
 }
 
+impl StreamingUploaderImpl {
+    pub async fn write_bytes(&mut self, data: Bytes) -> ObjectResult<()> {
+        object_store_impl_method_body!(self, write_bytes(data).await)
+    }
+
+    pub async fn finish(self) -> ObjectResult<()> {
+        object_store_impl_method_body!(self, finish().await)
+    }
+
+    pub fn get_memory_usage(&self) -> u64 {
+        object_store_impl_method_body!(self, get_memory_usage())
+    }
+}
+
 impl ObjectStoreImpl {
     pub async fn upload(&self, path: &str, obj: Bytes) -> ObjectResult<()> {
-        object_store_impl_method_body!(self, upload, dispatch_async, path, obj)
+        object_store_impl_method_body!(self, upload(path, obj).await)
     }
 
-    pub async fn streaming_upload(&self, path: &str) -> ObjectResult<MonitoredStreamingUploader> {
-        object_store_impl_method_body!(self, streaming_upload, dispatch_async, path)
+    pub async fn streaming_upload(&self, path: &str) -> ObjectResult<ObjectStreamingUploader> {
+        Ok(enum_map!(self, |store| {
+            store.streaming_upload(path).await?
+        }))
     }
 
     pub async fn read(&self, path: &str, range: impl ObjectRangeBounds) -> ObjectResult<Bytes> {
-        object_store_impl_method_body!(self, read, dispatch_async, path, range)
+        object_store_impl_method_body!(self, read(path, range).await)
     }
 
     pub async fn metadata(&self, path: &str) -> ObjectResult<ObjectMetadata> {
-        object_store_impl_method_body!(self, metadata, dispatch_async, path)
+        object_store_impl_method_body!(self, metadata(path).await)
     }
 
     /// Returns a stream reading the object specified in `path`. If given, the stream starts at the
@@ -232,11 +305,11 @@ impl ObjectStoreImpl {
         path: &str,
         start_loc: Range<usize>,
     ) -> ObjectResult<MonitoredStreamingReader> {
-        object_store_impl_method_body!(self, streaming_read, dispatch_async, path, start_loc)
+        object_store_impl_method_body!(self, streaming_read(path, start_loc).await)
     }
 
     pub async fn delete(&self, path: &str) -> ObjectResult<()> {
-        object_store_impl_method_body!(self, delete, dispatch_async, path)
+        object_store_impl_method_body!(self, delete(path).await)
     }
 
     /// Deletes the objects with the given paths permanently from the storage. If an object
@@ -245,36 +318,19 @@ impl ObjectStoreImpl {
     /// If a hybrid storage is used, the method will first attempt to delete objects in local
     /// storage. Only if that is successful, it will remove objects from remote storage.
     pub async fn delete_objects(&self, paths: &[String]) -> ObjectResult<()> {
-        object_store_impl_method_body_slice!(self, delete_objects, dispatch_async, paths)
+        object_store_impl_method_body!(self, delete_objects(paths).await)
     }
 
     pub async fn list(&self, prefix: &str) -> ObjectResult<ObjectMetadataIter> {
-        object_store_impl_method_body!(self, list, dispatch_async, prefix)
+        object_store_impl_method_body!(self, list(prefix).await)
     }
 
     pub fn get_object_prefix(&self, obj_id: u64) -> String {
-        // FIXME: ObjectStoreImpl lacks flexibility for adding new interface to ObjectStore
-        // trait. Macro object_store_impl_method_body routes to local or remote only depending on
-        // the path
-        match self {
-            ObjectStoreImpl::InMem(store) => store.inner.get_object_prefix(obj_id),
-            ObjectStoreImpl::Opendal(store) => store.inner.get_object_prefix(obj_id),
-            ObjectStoreImpl::S3(store) => store.inner.get_object_prefix(obj_id),
-            #[cfg(madsim)]
-            ObjectStoreImpl::Sim(store) => store.inner.get_object_prefix(obj_id),
-        }
+        dispatch_object_store_enum!(self, |store| store.inner.get_object_prefix(obj_id))
     }
 
     pub fn support_streaming_upload(&self) -> bool {
-        match self {
-            ObjectStoreImpl::InMem(_) => true,
-            ObjectStoreImpl::Opendal(store) => {
-                store.inner.op.info().native_capability().write_can_multi
-            }
-            ObjectStoreImpl::S3(_) => true,
-            #[cfg(madsim)]
-            ObjectStoreImpl::Sim(_) => true,
-        }
+        dispatch_object_store_enum!(self, |store| store.inner.support_streaming_upload())
     }
 }
 
@@ -303,101 +359,102 @@ fn try_update_failure_metric<T>(
 ///   - `streaming_upload_finish`: The time spent calling `finish`.
 /// - `failure_count`: `streaming_upload_start`, `streaming_upload_write_bytes`,
 ///   `streaming_upload_finish`
-pub struct MonitoredStreamingUploader {
-    inner: BoxedStreamingUploader,
+pub struct MonitoredStreamingUploader<U: StreamingUploader> {
+    inner: U,
     object_store_metrics: Arc<ObjectStoreMetrics>,
     /// Length of data uploaded with this uploader.
     operation_size: usize,
     media_type: &'static str,
-    streaming_upload_timeout: Option<Duration>,
 }
 
-impl MonitoredStreamingUploader {
+impl<U: StreamingUploader> MonitoredStreamingUploader<U> {
     pub fn new(
         media_type: &'static str,
-        handle: BoxedStreamingUploader,
+        handle: U,
         object_store_metrics: Arc<ObjectStoreMetrics>,
-        streaming_upload_timeout: Option<Duration>,
     ) -> Self {
         Self {
             inner: handle,
             object_store_metrics,
             operation_size: 0,
             media_type,
-            streaming_upload_timeout,
         }
     }
 }
 
-impl MonitoredStreamingUploader {
-    pub async fn write_bytes(&mut self, data: Bytes) -> ObjectResult<()> {
-        let operation_type = "streaming_upload_write_bytes";
+/// NOTICE: after #16231, streaming uploader implemented via aws-sdk-s3 will maintain metrics internally in s3.rs
+/// so `MonitoredStreamingUploader` will only be used when the inner object store is opendal.
+impl<U: StreamingUploader> MonitoredStreamingUploader<U> {
+    async fn write_bytes(&mut self, data: Bytes) -> ObjectResult<()> {
+        let operation_type = OperationType::StreamingUpload;
+        let operation_type_str = operation_type.as_str();
         let data_len = data.len();
-        self.object_store_metrics
-            .write_bytes
-            .inc_by(data.len() as u64);
-        self.object_store_metrics
-            .operation_size
-            .with_label_values(&[operation_type])
-            .observe(data_len as f64);
-        let _timer = self
-            .object_store_metrics
-            .operation_latency
-            .with_label_values(&[self.media_type, operation_type])
-            .start_timer();
-        self.operation_size += data_len;
 
-        let future = async {
+        let res = if self.media_type == "s3" {
+            // TODO: we should avoid this special case after fully migrating to opeandal for s3.
             self.inner
                 .write_bytes(data)
-                .verbose_instrument_await("object_store_streaming_upload_write_bytes")
+                .verbose_instrument_await(operation_type_str)
                 .await
-        };
-        let res = match self.streaming_upload_timeout.as_ref() {
-            None => future.await,
-            Some(timeout) => tokio::time::timeout(*timeout, future)
+        } else {
+            let _timer = self
+                .object_store_metrics
+                .operation_latency
+                .with_label_values(&[self.media_type, operation_type_str])
+                .start_timer();
+
+            self.inner
+                .write_bytes(data)
+                .verbose_instrument_await(operation_type_str)
                 .await
-                .unwrap_or_else(|_| {
-                    Err(ObjectError::internal(
-                        "streaming_upload write_bytes timeout",
-                    ))
-                }),
         };
 
-        try_update_failure_metric(&self.object_store_metrics, &res, operation_type);
-        res
-    }
+        try_update_failure_metric(&self.object_store_metrics, &res, operation_type_str);
 
-    pub async fn finish(self) -> ObjectResult<()> {
-        let operation_type = "streaming_upload_finish";
+        self.object_store_metrics
+            .write_bytes
+            .inc_by(data_len as u64);
         self.object_store_metrics
             .operation_size
-            .with_label_values(&["streaming_upload"])
-            .observe(self.operation_size as f64);
-        let _timer = self
-            .object_store_metrics
-            .operation_latency
-            .with_label_values(&[self.media_type, operation_type])
-            .start_timer();
+            .with_label_values(&[operation_type_str])
+            .observe(data_len as f64);
+        self.operation_size += data_len;
 
-        let future = async {
+        res
+    }
+
+    async fn finish(self) -> ObjectResult<()> {
+        let operation_type = OperationType::StreamingUploadFinish;
+        let operation_type_str = operation_type.as_str();
+
+        let res = if self.media_type == "s3" {
+            // TODO: we should avoid this special case after fully migrating to opeandal for s3.
             self.inner
                 .finish()
-                .verbose_instrument_await("object_store_streaming_upload_finish")
+                .verbose_instrument_await(operation_type_str)
                 .await
-        };
-        let res = match self.streaming_upload_timeout.as_ref() {
-            None => future.await,
-            Some(timeout) => tokio::time::timeout(*timeout, future)
+        } else {
+            let _timer = self
+                .object_store_metrics
+                .operation_latency
+                .with_label_values(&[self.media_type, operation_type_str])
+                .start_timer();
+
+            self.inner
+                .finish()
+                .verbose_instrument_await(operation_type_str)
                 .await
-                .unwrap_or_else(|_| Err(ObjectError::internal("streaming_upload finish timeout"))),
         };
 
-        try_update_failure_metric(&self.object_store_metrics, &res, operation_type);
+        try_update_failure_metric(&self.object_store_metrics, &res, operation_type_str);
+        self.object_store_metrics
+            .operation_size
+            .with_label_values(&[operation_type_str])
+            .observe(self.operation_size as f64);
         res
     }
 
-    pub fn get_memory_usage(&self) -> u64 {
+    fn get_memory_usage(&self) -> u64 {
         self.inner.get_memory_usage()
     }
 }
@@ -407,8 +464,8 @@ pub struct MonitoredStreamingReader {
     object_store_metrics: Arc<ObjectStoreMetrics>,
     operation_size: usize,
     media_type: &'static str,
-    timer: Option<HistogramTimer>,
     streaming_read_timeout: Option<Duration>,
+    operation_type_str: &'static str,
 }
 
 impl MonitoredStreamingReader {
@@ -418,54 +475,49 @@ impl MonitoredStreamingReader {
         object_store_metrics: Arc<ObjectStoreMetrics>,
         streaming_read_timeout: Option<Duration>,
     ) -> Self {
-        let operation_type = "streaming_read";
-        let timer = object_store_metrics
-            .operation_latency
-            .with_label_values(&[media_type, operation_type])
-            .start_timer();
         Self {
             inner: handle,
             object_store_metrics,
             operation_size: 0,
             media_type,
-            timer: Some(timer),
             streaming_read_timeout,
+            operation_type_str: OperationType::StreamingRead.as_str(),
         }
     }
 
     pub async fn read_bytes(&mut self) -> Option<ObjectResult<Bytes>> {
-        let operation_type = "streaming_read_read_bytes";
         let _timer = self
             .object_store_metrics
             .operation_latency
-            .with_label_values(&[self.media_type, operation_type])
+            .with_label_values(&[self.media_type, self.operation_type_str])
             .start_timer();
         let future = async {
             self.inner
                 .next()
-                .verbose_instrument_await("object_store_streaming_read_read_bytes")
+                .verbose_instrument_await(self.operation_type_str)
                 .await
         };
         let res = match self.streaming_read_timeout.as_ref() {
             None => future.await,
-            Some(timeout) => tokio::time::timeout(*timeout, future)
+            Some(timeout_duration) => tokio::time::timeout(*timeout_duration, future)
                 .await
                 .unwrap_or_else(|_| {
-                    Some(Err(ObjectError::internal(
-                        "streaming_read read_bytes timeout",
-                    )))
+                    Some(Err(ObjectError::timeout(format!(
+                        "Retry attempts exhausted for {}. Please modify {}_attempt_timeout_ms (current={:?}) under [storage.object_store.retry] in the config accordingly if needed.",
+                        self.operation_type_str, self.operation_type_str, timeout_duration.as_millis()
+                    ))))
                 }),
         };
 
         if let Some(ret) = &res {
-            try_update_failure_metric(&self.object_store_metrics, ret, operation_type);
+            try_update_failure_metric(&self.object_store_metrics, ret, self.operation_type_str);
         }
         if let Some(Ok(data)) = &res {
             let data_len = data.len();
             self.object_store_metrics.read_bytes.inc_by(data_len as u64);
             self.object_store_metrics
                 .operation_size
-                .with_label_values(&[operation_type])
+                .with_label_values(&[self.operation_type_str])
                 .observe(data_len as f64);
             self.operation_size += data_len;
         }
@@ -475,22 +527,17 @@ impl MonitoredStreamingReader {
 
 impl Drop for MonitoredStreamingReader {
     fn drop(&mut self) {
-        let operation_type = "streaming_read";
         self.object_store_metrics
             .operation_size
-            .with_label_values(&[operation_type])
+            .with_label_values(&[self.operation_type_str])
             .observe(self.operation_size as f64);
-        self.timer.take().unwrap().observe_duration();
     }
 }
 
 pub struct MonitoredObjectStore<OS: ObjectStore> {
     inner: OS,
     object_store_metrics: Arc<ObjectStoreMetrics>,
-    streaming_read_timeout: Option<Duration>,
-    streaming_upload_timeout: Option<Duration>,
-    read_timeout: Option<Duration>,
-    upload_timeout: Option<Duration>,
+    config: Arc<ObjectStoreConfig>,
 }
 
 /// Manually dispatch trait methods.
@@ -513,19 +560,12 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> {
     pub fn new(
         store: OS,
         object_store_metrics: Arc<ObjectStoreMetrics>,
-        config: ObjectStoreConfig,
+        config: Arc<ObjectStoreConfig>,
     ) -> Self {
         Self {
             object_store_metrics,
-            streaming_read_timeout: Some(Duration::from_millis(
-                config.object_store_streaming_read_timeout_ms,
-            )),
-            streaming_upload_timeout: Some(Duration::from_millis(
-                config.object_store_streaming_upload_timeout_ms,
-            )),
-            read_timeout: Some(Duration::from_millis(config.object_store_read_timeout_ms)),
-            upload_timeout: Some(Duration::from_millis(config.object_store_upload_timeout_ms)),
             inner: store,
+            config,
         }
     }
 
@@ -542,86 +582,93 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> {
     }
 
     pub async fn upload(&self, path: &str, obj: Bytes) -> ObjectResult<()> {
-        let operation_type = "upload";
+        let operation_type = OperationType::Upload;
+        let operation_type_str = operation_type.as_str();
+
         self.object_store_metrics
             .write_bytes
             .inc_by(obj.len() as u64);
         self.object_store_metrics
             .operation_size
-            .with_label_values(&[operation_type])
+            .with_label_values(&[operation_type_str])
             .observe(obj.len() as f64);
         let _timer = self
             .object_store_metrics
             .operation_latency
-            .with_label_values(&[self.media_type(), operation_type])
+            .with_label_values(&[self.media_type(), operation_type_str])
             .start_timer();
-        let future = async {
+
+        let builder = || async {
             self.inner
-                .upload(path, obj)
-                .verbose_instrument_await("object_store_upload")
-                .await
-        };
-        let res = match self.upload_timeout.as_ref() {
-            None => future.await,
-            Some(timeout) => tokio::time::timeout(*timeout, future)
+                .upload(path, obj.clone())
+                .verbose_instrument_await(operation_type_str)
                 .await
-                .unwrap_or_else(|_| Err(ObjectError::internal("upload timeout"))),
         };
 
-        try_update_failure_metric(&self.object_store_metrics, &res, operation_type);
+        let res = retry_request(
+            builder,
+            &self.config,
+            operation_type,
+            self.object_store_metrics.clone(),
+        )
+        .await;
+
+        try_update_failure_metric(&self.object_store_metrics, &res, operation_type_str);
         res
     }
 
-    pub async fn streaming_upload(&self, path: &str) -> ObjectResult<MonitoredStreamingUploader> {
-        let operation_type = "streaming_upload_start";
+    pub async fn streaming_upload(
+        &self,
+        path: &str,
+    ) -> ObjectResult<MonitoredStreamingUploader<OS::StreamingUploader>> {
+        let operation_type = OperationType::StreamingUploadInit;
+        let operation_type_str = operation_type.as_str();
         let media_type = self.media_type();
         let _timer = self
             .object_store_metrics
             .operation_latency
-            .with_label_values(&[media_type, operation_type])
+            .with_label_values(&[media_type, operation_type_str])
             .start_timer();
-        let future = async {
-            self.inner
-                .streaming_upload(path)
-                .verbose_instrument_await("object_store_streaming_upload")
-                .await
-        };
-        let res = match self.streaming_upload_timeout.as_ref() {
-            None => future.await,
-            Some(timeout) => tokio::time::timeout(*timeout, future)
-                .await
-                .unwrap_or_else(|_| Err(ObjectError::internal("streaming_upload init timeout"))),
-        };
 
-        try_update_failure_metric(&self.object_store_metrics, &res, operation_type);
+        let res = self
+            .inner
+            .streaming_upload(path)
+            .verbose_instrument_await(operation_type_str)
+            .await;
+
+        try_update_failure_metric(&self.object_store_metrics, &res, operation_type_str);
+
         Ok(MonitoredStreamingUploader::new(
             media_type,
             res?,
             self.object_store_metrics.clone(),
-            self.streaming_upload_timeout,
         ))
     }
 
     pub async fn read(&self, path: &str, range: impl ObjectRangeBounds) -> ObjectResult<Bytes> {
-        let operation_type = "read";
+        let operation_type = OperationType::Read;
+        let operation_type_str = operation_type.as_str();
         let _timer = self
             .object_store_metrics
             .operation_latency
-            .with_label_values(&[self.media_type(), operation_type])
+            .with_label_values(&[self.media_type(), operation_type_str])
             .start_timer();
-        let future = async {
+
+        let builder = || async {
             self.inner
-                .read(path, range)
-                .verbose_instrument_await("object_store_read")
-                .await
-        };
-        let res = match self.read_timeout.as_ref() {
-            None => future.await,
-            Some(read_timeout) => tokio::time::timeout(*read_timeout, future)
+                .read(path, range.clone())
+                .verbose_instrument_await(operation_type_str)
                 .await
-                .unwrap_or_else(|_| Err(ObjectError::internal("read timeout"))),
         };
 
+        let res = retry_request(
+            builder,
+            &self.config,
+            operation_type,
+            self.object_store_metrics.clone(),
+        )
+        .await;
+
         if let Err(e) = &res
             && e.is_object_not_found_error()
             && !path.ends_with(".data")
@@ -629,7 +676,7 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> {
             // Some not_found_error is expected, e.g. metadata backup's manifest.json.
             // This is a quick fix that'll only log error in `try_update_failure_metric` in state store usage.
         } else {
-            try_update_failure_metric(&self.object_store_metrics, &res, operation_type);
+            try_update_failure_metric(&self.object_store_metrics, &res, operation_type_str);
         }
 
         let data = res?;
@@ -638,7 +685,7 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> {
             .inc_by(data.len() as u64);
         self.object_store_metrics
             .operation_size
-            .with_label_values(&[operation_type])
+            .with_label_values(&[operation_type_str])
             .observe(data.len() as f64);
         Ok(data)
     }
@@ -651,130 +698,152 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> {
         path: &str,
         range: Range<usize>,
     ) -> ObjectResult<MonitoredStreamingReader> {
-        let operation_type = "streaming_read_start";
+        let operation_type = OperationType::StreamingReadInit;
+        let operation_type_str = operation_type.as_str();
         let media_type = self.media_type();
         let _timer = self
             .object_store_metrics
             .operation_latency
-            .with_label_values(&[media_type, operation_type])
+            .with_label_values(&[media_type, operation_type_str])
             .start_timer();
-        let future = self
-            .inner
-            .streaming_read(path, range)
-            .verbose_instrument_await("object_store_streaming_read");
-        let res = match self.streaming_read_timeout.as_ref() {
-            None => future.await,
-            Some(timeout) => tokio::time::timeout(*timeout, future)
+
+        let builder = || async {
+            self.inner
+                .streaming_read(path, range.clone())
+                .verbose_instrument_await(operation_type_str)
                 .await
-                .unwrap_or_else(|_| Err(ObjectError::internal("streaming_read init timeout"))),
         };
 
-        try_update_failure_metric(&self.object_store_metrics, &res, operation_type);
+        let res = retry_request(
+            builder,
+            &self.config,
+            operation_type,
+            self.object_store_metrics.clone(),
+        )
+        .await;
+
+        try_update_failure_metric(&self.object_store_metrics, &res, operation_type_str);
+
         Ok(MonitoredStreamingReader::new(
             media_type,
             res?,
             self.object_store_metrics.clone(),
-            self.streaming_read_timeout,
+            Some(Duration::from_millis(
+                self.config.retry.streaming_read_attempt_timeout_ms,
+            )),
         ))
     }
 
     pub async fn metadata(&self, path: &str) -> ObjectResult<ObjectMetadata> {
-        let operation_type = "metadata";
+        let operation_type = OperationType::Metadata;
+        let operation_type_str = operation_type.as_str();
         let _timer = self
             .object_store_metrics
             .operation_latency
-            .with_label_values(&[self.media_type(), operation_type])
+            .with_label_values(&[self.media_type(), operation_type_str])
             .start_timer();
 
-        let future = async {
+        let builder = || async {
             self.inner
                 .metadata(path)
-                .verbose_instrument_await("object_store_metadata")
-                .await
-        };
-        let res = match self.read_timeout.as_ref() {
-            None => future.await,
-            Some(timeout) => tokio::time::timeout(*timeout, future)
+                .verbose_instrument_await(operation_type_str)
                 .await
-                .unwrap_or_else(|_| Err(ObjectError::internal("metadata timeout"))),
         };
 
-        try_update_failure_metric(&self.object_store_metrics, &res, operation_type);
+        let res = retry_request(
+            builder,
+            &self.config,
+            operation_type,
+            self.object_store_metrics.clone(),
+        )
+        .await;
+
+        try_update_failure_metric(&self.object_store_metrics, &res, operation_type_str);
         res
     }
 
     pub async fn delete(&self, path: &str) -> ObjectResult<()> {
-        let operation_type = "delete";
+        let operation_type = OperationType::Delete;
+        let operation_type_str = operation_type.as_str();
         let _timer = self
             .object_store_metrics
             .operation_latency
-            .with_label_values(&[self.media_type(), operation_type])
+            .with_label_values(&[self.media_type(), operation_type_str])
             .start_timer();
 
-        let future = async {
+        let builder = || async {
             self.inner
                 .delete(path)
-                .verbose_instrument_await("object_store_delete")
+                .verbose_instrument_await(operation_type_str)
                 .await
         };
-        let res = match self.read_timeout.as_ref() {
-            None => future.await,
-            Some(timeout) => tokio::time::timeout(*timeout, future)
-                .await
-                .unwrap_or_else(|_| Err(ObjectError::internal("delete timeout"))),
-        };
 
-        try_update_failure_metric(&self.object_store_metrics, &res, operation_type);
+        let res = retry_request(
+            builder,
+            &self.config,
+            operation_type,
+            self.object_store_metrics.clone(),
+        )
+        .await;
+
+        try_update_failure_metric(&self.object_store_metrics, &res, operation_type_str);
         res
     }
 
     async fn delete_objects(&self, paths: &[String]) -> ObjectResult<()> {
-        let operation_type = "delete_objects";
+        let operation_type = OperationType::DeleteObjects;
+        let operation_type_str = operation_type.as_str();
         let _timer = self
             .object_store_metrics
             .operation_latency
-            .with_label_values(&[self.media_type(), operation_type])
+            .with_label_values(&[self.media_type(), operation_type_str])
             .start_timer();
 
-        let future = async {
+        let builder = || async {
             self.inner
                 .delete_objects(paths)
-                .verbose_instrument_await("object_store_delete_objects")
-                .await
-        };
-        let res = match self.read_timeout.as_ref() {
-            None => future.await,
-            Some(timeout) => tokio::time::timeout(*timeout, future)
+                .verbose_instrument_await(operation_type_str)
                 .await
-                .unwrap_or_else(|_| Err(ObjectError::internal("delete_objects timeout"))),
         };
 
-        try_update_failure_metric(&self.object_store_metrics, &res, operation_type);
+        let res = retry_request(
+            builder,
+            &self.config,
+            operation_type,
+            self.object_store_metrics.clone(),
+        )
+        .await;
+
+        try_update_failure_metric(&self.object_store_metrics, &res, operation_type_str);
         res
     }
 
     pub async fn list(&self, prefix: &str) -> ObjectResult<ObjectMetadataIter> {
-        let operation_type = "list";
+        let operation_type = OperationType::List;
+        let operation_type_str = operation_type.as_str();
+
         let _timer = self
             .object_store_metrics
             .operation_latency
-            .with_label_values(&[self.media_type(), operation_type])
+            .with_label_values(&[self.media_type(), operation_type_str])
             .start_timer();
 
-        let future = async {
+        let builder = || async {
             self.inner
                 .list(prefix)
-                .verbose_instrument_await("object_store_list")
+                .verbose_instrument_await(operation_type_str)
                 .await
         };
-        let res = match self.read_timeout.as_ref() {
-            None => future.await,
-            Some(timeout) => tokio::time::timeout(*timeout, future)
-                .await
-                .unwrap_or_else(|_| Err(ObjectError::internal("list timeout"))),
-        };
 
-        try_update_failure_metric(&self.object_store_metrics, &res, operation_type);
+        let res = retry_request(
+            builder,
+            &self.config,
+            operation_type,
+            self.object_store_metrics.clone(),
+        )
+        .await;
+
+        try_update_failure_metric(&self.object_store_metrics, &res, operation_type_str);
         res
     }
 }
@@ -788,7 +857,7 @@ pub async fn build_remote_object_store(
     url: &str,
     metrics: Arc<ObjectStoreMetrics>,
     ident: &str,
-    config: ObjectStoreConfig,
+    config: Arc<ObjectStoreConfig>,
 ) -> ObjectStoreImpl {
     tracing::debug!(config=?config, "object store {ident}");
     match url {
@@ -831,18 +900,26 @@ pub async fn build_remote_object_store(
             let gcs = gcs.strip_prefix("gcs://").unwrap();
             let (bucket, root) = gcs.split_once('@').unwrap_or((gcs, ""));
             ObjectStoreImpl::Opendal(
-                OpendalObjectStore::new_gcs_engine(bucket.to_string(), root.to_string())
-                    .unwrap()
-                    .monitored(metrics, config),
+                OpendalObjectStore::new_gcs_engine(
+                    bucket.to_string(),
+                    root.to_string(),
+                    config.clone(),
+                )
+                .unwrap()
+                .monitored(metrics, config),
             )
         }
         obs if obs.starts_with("obs://") => {
             let obs = obs.strip_prefix("obs://").unwrap();
             let (bucket, root) = obs.split_once('@').unwrap_or((obs, ""));
             ObjectStoreImpl::Opendal(
-                OpendalObjectStore::new_obs_engine(bucket.to_string(), root.to_string())
-                    .unwrap()
-                    .monitored(metrics, config),
+                OpendalObjectStore::new_obs_engine(
+                    bucket.to_string(),
+                    root.to_string(),
+                    config.clone(),
+                )
+                .unwrap()
+                .monitored(metrics, config),
             )
         }
 
@@ -850,27 +927,39 @@ pub async fn build_remote_object_store(
             let oss = oss.strip_prefix("oss://").unwrap();
             let (bucket, root) = oss.split_once('@').unwrap_or((oss, ""));
             ObjectStoreImpl::Opendal(
-                OpendalObjectStore::new_oss_engine(bucket.to_string(), root.to_string())
-                    .unwrap()
-                    .monitored(metrics, config),
+                OpendalObjectStore::new_oss_engine(
+                    bucket.to_string(),
+                    root.to_string(),
+                    config.clone(),
+                )
+                .unwrap()
+                .monitored(metrics, config),
             )
         }
         webhdfs if webhdfs.starts_with("webhdfs://") => {
             let webhdfs = webhdfs.strip_prefix("webhdfs://").unwrap();
             let (namenode, root) = webhdfs.split_once('@').unwrap_or((webhdfs, ""));
             ObjectStoreImpl::Opendal(
-                OpendalObjectStore::new_webhdfs_engine(namenode.to_string(), root.to_string())
-                    .unwrap()
-                    .monitored(metrics, config),
+                OpendalObjectStore::new_webhdfs_engine(
+                    namenode.to_string(),
+                    root.to_string(),
+                    config.clone(),
+                )
+                .unwrap()
+                .monitored(metrics, config),
             )
         }
         azblob if azblob.starts_with("azblob://") => {
             let azblob = azblob.strip_prefix("azblob://").unwrap();
             let (container_name, root) = azblob.split_once('@').unwrap_or((azblob, ""));
             ObjectStoreImpl::Opendal(
-                OpendalObjectStore::new_azblob_engine(container_name.to_string(), root.to_string())
-                    .unwrap()
-                    .monitored(metrics, config),
+                OpendalObjectStore::new_azblob_engine(
+                    container_name.to_string(),
+                    root.to_string(),
+                    config.clone(),
+                )
+                .unwrap()
+                .monitored(metrics, config),
             )
         }
         fs if fs.starts_with("fs://") => {
@@ -892,13 +981,13 @@ pub async fn build_remote_object_store(
             if config.s3.developer.use_opendal {
                 tracing::info!("Using OpenDAL to access minio.");
                 ObjectStoreImpl::Opendal(
-                    OpendalObjectStore::with_minio(minio, config.clone())
+                    OpendalObjectStore::new_minio_engine(minio, config.clone())
                         .unwrap()
                         .monitored(metrics, config),
                 )
             } else {
                 ObjectStoreImpl::S3(
-                    S3ObjectStore::with_minio(minio, metrics.clone(), config.clone())
+                    S3ObjectStore::new_minio_engine(minio, metrics.clone(), config.clone())
                         .await
                         .monitored(metrics, config),
                 )
@@ -933,5 +1022,164 @@ pub async fn build_remote_object_store(
     }
 }
 
+#[inline(always)]
+fn get_retry_strategy(
+    config: &ObjectStoreConfig,
+    operation_type: OperationType,
+) -> impl Iterator<Item = Duration> {
+    let attempts = get_retry_attempts_by_type(config, operation_type);
+    ExponentialBackoff::from_millis(config.retry.req_backoff_interval_ms)
+        .max_delay(Duration::from_millis(config.retry.req_backoff_max_delay_ms))
+        .factor(config.retry.req_backoff_factor)
+        .take(attempts)
+        .map(jitter)
+}
+
 pub type ObjectMetadataIter = BoxStream<'static, ObjectResult<ObjectMetadata>>;
 pub type ObjectDataStream = BoxStream<'static, ObjectResult<Bytes>>;
+
+#[derive(Debug, Clone, Copy)]
+enum OperationType {
+    Upload,
+    StreamingUploadInit,
+    StreamingUpload,
+    StreamingUploadFinish,
+    Read,
+    StreamingReadInit,
+    StreamingRead,
+    Metadata,
+    Delete,
+    DeleteObjects,
+    List,
+}
+
+impl OperationType {
+    fn as_str(&self) -> &'static str {
+        match self {
+            Self::Upload => "upload",
+            Self::StreamingUploadInit => "streaming_upload_init",
+            Self::StreamingUpload => "streaming_upload",
+            Self::StreamingUploadFinish => "streaming_upload_finish",
+            Self::Read => "read",
+            Self::StreamingReadInit => "streaming_read_init",
+            Self::StreamingRead => "streaming_read",
+            Self::Metadata => "metadata",
+            Self::Delete => "delete",
+            Self::DeleteObjects => "delete_objects",
+            Self::List => "list",
+        }
+    }
+}
+
+fn get_retry_attempts_by_type(config: &ObjectStoreConfig, operation_type: OperationType) -> usize {
+    match operation_type {
+        OperationType::Upload => config.retry.upload_retry_attempts,
+        OperationType::StreamingUploadInit
+        | OperationType::StreamingUpload
+        | OperationType::StreamingUploadFinish => config.retry.streaming_upload_retry_attempts,
+        OperationType::Read => config.retry.read_retry_attempts,
+        OperationType::StreamingReadInit | OperationType::StreamingRead => {
+            config.retry.streaming_read_retry_attempts
+        }
+        OperationType::Metadata => config.retry.metadata_retry_attempts,
+        OperationType::Delete => config.retry.delete_retry_attempts,
+        OperationType::DeleteObjects => config.retry.delete_objects_retry_attempts,
+        OperationType::List => config.retry.list_retry_attempts,
+    }
+}
+
+fn get_attempt_timeout_by_type(config: &ObjectStoreConfig, operation_type: OperationType) -> u64 {
+    match operation_type {
+        OperationType::Upload => config.retry.upload_attempt_timeout_ms,
+        OperationType::StreamingUploadInit
+        | OperationType::StreamingUpload
+        | OperationType::StreamingUploadFinish => config.retry.streaming_upload_attempt_timeout_ms,
+        OperationType::Read => config.retry.read_attempt_timeout_ms,
+        OperationType::StreamingReadInit | OperationType::StreamingRead => {
+            config.retry.streaming_read_attempt_timeout_ms
+        }
+        OperationType::Metadata => config.retry.metadata_attempt_timeout_ms,
+        OperationType::Delete => config.retry.delete_attempt_timeout_ms,
+        OperationType::DeleteObjects => config.retry.delete_objects_attempt_timeout_ms,
+        OperationType::List => config.retry.list_attempt_timeout_ms,
+    }
+}
+
+struct RetryCondition {
+    operation_type: OperationType,
+    retry_count: usize,
+    metrics: Arc<ObjectStoreMetrics>,
+}
+
+impl RetryCondition {
+    fn new(operation_type: OperationType, metrics: Arc<ObjectStoreMetrics>) -> Self {
+        Self {
+            operation_type,
+            retry_count: 0,
+            metrics,
+        }
+    }
+
+    #[inline(always)]
+    fn should_retry_inner(&mut self, err: &ObjectError) -> bool {
+        let should_retry = err.should_retry();
+        if should_retry {
+            self.retry_count += 1;
+        }
+
+        should_retry
+    }
+}
+
+impl tokio_retry::Condition<ObjectError> for RetryCondition {
+    fn should_retry(&mut self, err: &ObjectError) -> bool {
+        self.should_retry_inner(err)
+    }
+}
+
+impl Drop for RetryCondition {
+    fn drop(&mut self) {
+        if self.retry_count > 0 {
+            self.metrics
+                .request_retry_count
+                .with_label_values(&[self.operation_type.as_str()])
+                .inc_by(self.retry_count as _);
+        }
+    }
+}
+
+async fn retry_request<F, T, B>(
+    builder: B,
+    config: &ObjectStoreConfig,
+    operation_type: OperationType,
+    object_store_metrics: Arc<ObjectStoreMetrics>,
+) -> ObjectResult<T>
+where
+    B: Fn() -> F,
+    F: Future<Output = ObjectResult<T>>,
+{
+    let backoff = get_retry_strategy(config, operation_type);
+    let timeout_duration =
+        Duration::from_millis(get_attempt_timeout_by_type(config, operation_type));
+    let operation_type_str = operation_type.as_str();
+
+    let retry_condition = RetryCondition::new(operation_type, object_store_metrics);
+
+    let f = || async {
+        let future = builder();
+        if timeout_duration.is_zero() {
+            future.await
+        } else {
+            tokio::time::timeout(timeout_duration, future)
+                .await
+                .unwrap_or_else(|_| {
+                    Err(ObjectError::timeout(format!(
+                        "Retry attempts exhausted for {}. Please modify {}_attempt_timeout_ms (current={:?}) and {}_retry_attempts (current={}) under [storage.object_store.retry] in the config accordingly if needed.",
+                        operation_type_str, operation_type_str, timeout_duration.as_millis(), operation_type_str, get_retry_attempts_by_type(config, operation_type)
+                    )))
+                })
+        }
+    };
+
+    tokio_retry::RetryIf::spawn(backoff, f, retry_condition).await
+}
diff --git a/src/object_store/src/object/object_metrics.rs b/src/object_store/src/object/object_metrics.rs
index aadf5f35cdb3b..893f914f7f4c4 100644
--- a/src/object_store/src/object/object_metrics.rs
+++ b/src/object_store/src/object/object_metrics.rs
@@ -90,7 +90,7 @@ impl ObjectStoreMetrics {
         .unwrap();
 
         let request_retry_count = register_int_counter_vec_with_registry!(
-            "s3_read_request_retry_count",
+            "object_store_request_retry_count",
             "The number of retry times of object store request",
             &["type"],
             registry
diff --git a/src/object_store/src/object/opendal_engine/azblob.rs b/src/object_store/src/object/opendal_engine/azblob.rs
index 8ac8caf09fda3..590859eaaa706 100644
--- a/src/object_store/src/object/opendal_engine/azblob.rs
+++ b/src/object_store/src/object/opendal_engine/azblob.rs
@@ -12,9 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use opendal::layers::{LoggingLayer, RetryLayer};
+use std::sync::Arc;
+
+use opendal::layers::LoggingLayer;
 use opendal::services::Azblob;
 use opendal::Operator;
+use risingwave_common::config::ObjectStoreConfig;
 
 use super::{EngineType, OpendalObjectStore};
 use crate::object::ObjectResult;
@@ -22,7 +25,11 @@ use crate::object::ObjectResult;
 const AZBLOB_ENDPOINT: &str = "AZBLOB_ENDPOINT";
 impl OpendalObjectStore {
     /// create opendal azblob engine.
-    pub fn new_azblob_engine(container_name: String, root: String) -> ObjectResult<Self> {
+    pub fn new_azblob_engine(
+        container_name: String,
+        root: String,
+        config: Arc<ObjectStoreConfig>,
+    ) -> ObjectResult<Self> {
         // Create azblob backend builder.
         let mut builder = Azblob::default();
         builder.root(&root);
@@ -35,11 +42,11 @@ impl OpendalObjectStore {
 
         let op: Operator = Operator::new(builder)?
             .layer(LoggingLayer::default())
-            .layer(RetryLayer::default())
             .finish();
         Ok(Self {
             op,
             engine_type: EngineType::Azblob,
+            config,
         })
     }
 }
diff --git a/src/object_store/src/object/opendal_engine/fs.rs b/src/object_store/src/object/opendal_engine/fs.rs
index f2e8211e93664..4f2715d6ccfb8 100644
--- a/src/object_store/src/object/opendal_engine/fs.rs
+++ b/src/object_store/src/object/opendal_engine/fs.rs
@@ -12,7 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use opendal::layers::RetryLayer;
+use std::sync::Arc;
+
+use opendal::layers::LoggingLayer;
 use opendal::services::Fs;
 use opendal::Operator;
 use risingwave_common::config::ObjectStoreConfig;
@@ -23,7 +25,7 @@ use crate::object::ObjectResult;
 
 impl OpendalObjectStore {
     /// create opendal fs engine.
-    pub fn new_fs_engine(root: String, config: ObjectStoreConfig) -> ObjectResult<Self> {
+    pub fn new_fs_engine(root: String, config: Arc<ObjectStoreConfig>) -> ObjectResult<Self> {
         // Create fs backend builder.
         let mut builder = Fs::default();
         builder.root(&root);
@@ -33,11 +35,12 @@ impl OpendalObjectStore {
         }
 
         let op: Operator = Operator::new(builder)?
-            .layer(RetryLayer::default())
+            .layer(LoggingLayer::default())
             .finish();
         Ok(Self {
             op,
             engine_type: EngineType::Fs,
+            config,
         })
     }
 }
diff --git a/src/object_store/src/object/opendal_engine/gcs.rs b/src/object_store/src/object/opendal_engine/gcs.rs
index 0577288005fca..c55de2377202e 100644
--- a/src/object_store/src/object/opendal_engine/gcs.rs
+++ b/src/object_store/src/object/opendal_engine/gcs.rs
@@ -12,16 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use opendal::layers::{LoggingLayer, RetryLayer};
+use std::sync::Arc;
+
+use opendal::layers::LoggingLayer;
 use opendal::services::Gcs;
 use opendal::Operator;
+use risingwave_common::config::ObjectStoreConfig;
 
 use super::{EngineType, OpendalObjectStore};
 use crate::object::ObjectResult;
 
 impl OpendalObjectStore {
     /// create opendal gcs engine.
-    pub fn new_gcs_engine(bucket: String, root: String) -> ObjectResult<Self> {
+    pub fn new_gcs_engine(
+        bucket: String,
+        root: String,
+        config: Arc<ObjectStoreConfig>,
+    ) -> ObjectResult<Self> {
         // Create gcs backend builder.
         let mut builder = Gcs::default();
 
@@ -37,11 +44,11 @@ impl OpendalObjectStore {
 
         let op: Operator = Operator::new(builder)?
             .layer(LoggingLayer::default())
-            .layer(RetryLayer::default())
             .finish();
         Ok(Self {
             op,
             engine_type: EngineType::Gcs,
+            config,
         })
     }
 }
diff --git a/src/object_store/src/object/opendal_engine/hdfs.rs b/src/object_store/src/object/opendal_engine/hdfs.rs
index c539cb43cb49d..8c1e16eda1f57 100644
--- a/src/object_store/src/object/opendal_engine/hdfs.rs
+++ b/src/object_store/src/object/opendal_engine/hdfs.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use opendal::layers::{LoggingLayer, RetryLayer};
+use opendal::layers::LoggingLayer;
 use opendal::services::Hdfs;
 use opendal::Operator;
 use risingwave_common::config::ObjectStoreConfig;
@@ -33,13 +33,12 @@ impl OpendalObjectStore {
         // Set the name node for hdfs.
         builder.name_node(&namenode);
         builder.root(&root);
-        if config.object_store_set_atomic_write_dir {
+        if config.retry.set_atomic_write_dir {
             let atomic_write_dir = format!("{}/{}", root, ATOMIC_WRITE_DIR);
             builder.atomic_write_dir(&atomic_write_dir);
         }
         let op: Operator = Operator::new(builder)?
             .layer(LoggingLayer::default())
-            .layer(RetryLayer::default())
             .finish();
         Ok(Self {
             op,
diff --git a/src/object_store/src/object/opendal_engine/obs.rs b/src/object_store/src/object/opendal_engine/obs.rs
index 4ddf9579685f1..77178ca9ae7bc 100644
--- a/src/object_store/src/object/opendal_engine/obs.rs
+++ b/src/object_store/src/object/opendal_engine/obs.rs
@@ -12,16 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use opendal::layers::{LoggingLayer, RetryLayer};
+use std::sync::Arc;
+
+use opendal::layers::LoggingLayer;
 use opendal::services::Obs;
 use opendal::Operator;
+use risingwave_common::config::ObjectStoreConfig;
 
 use super::{EngineType, OpendalObjectStore};
 use crate::object::ObjectResult;
 
 impl OpendalObjectStore {
     /// create opendal obs engine.
-    pub fn new_obs_engine(bucket: String, root: String) -> ObjectResult<Self> {
+    pub fn new_obs_engine(
+        bucket: String,
+        root: String,
+        config: Arc<ObjectStoreConfig>,
+    ) -> ObjectResult<Self> {
         // Create obs backend builder.
         let mut builder = Obs::default();
 
@@ -43,11 +50,11 @@ impl OpendalObjectStore {
 
         let op: Operator = Operator::new(builder)?
             .layer(LoggingLayer::default())
-            .layer(RetryLayer::default())
             .finish();
         Ok(Self {
             op,
             engine_type: EngineType::Obs,
+            config,
         })
     }
 }
diff --git a/src/object_store/src/object/opendal_engine/opendal_object_store.rs b/src/object_store/src/object/opendal_engine/opendal_object_store.rs
index d7ba829dda632..47ca4f362702a 100644
--- a/src/object_store/src/object/opendal_engine/opendal_object_store.rs
+++ b/src/object_store/src/object/opendal_engine/opendal_object_store.rs
@@ -13,18 +13,22 @@
 // limitations under the License.
 
 use std::ops::Range;
+use std::sync::Arc;
+use std::time::Duration;
 
 use bytes::Bytes;
 use fail::fail_point;
 use futures::{stream, StreamExt, TryStreamExt};
+use opendal::layers::{RetryLayer, TimeoutLayer};
 use opendal::services::Memory;
 use opendal::{Metakey, Operator, Writer};
+use risingwave_common::config::ObjectStoreConfig;
 use risingwave_common::range::RangeBoundsExt;
 use thiserror_ext::AsReport;
 
 use crate::object::{
-    prefix, BoxedStreamingUploader, ObjectDataStream, ObjectError, ObjectMetadata,
-    ObjectMetadataIter, ObjectRangeBounds, ObjectResult, ObjectStore, StreamingUploader,
+    prefix, ObjectDataStream, ObjectError, ObjectMetadata, ObjectMetadataIter, ObjectRangeBounds,
+    ObjectResult, ObjectStore, StreamingUploader,
 };
 
 /// Opendal object storage.
@@ -32,6 +36,8 @@ use crate::object::{
 pub struct OpendalObjectStore {
     pub(crate) op: Operator,
     pub(crate) engine_type: EngineType,
+
+    pub(crate) config: Arc<ObjectStoreConfig>,
 }
 
 #[derive(Clone)]
@@ -50,19 +56,22 @@ pub enum EngineType {
 
 impl OpendalObjectStore {
     /// create opendal memory engine, used for unit tests.
-    pub fn new_memory_engine() -> ObjectResult<Self> {
+    pub fn test_new_memory_engine() -> ObjectResult<Self> {
         // Create memory backend builder.
         let builder = Memory::default();
         let op: Operator = Operator::new(builder)?.finish();
         Ok(Self {
             op,
             engine_type: EngineType::Memory,
+            config: Arc::new(ObjectStoreConfig::default()),
         })
     }
 }
 
 #[async_trait::async_trait]
 impl ObjectStore for OpendalObjectStore {
+    type StreamingUploader = OpendalStreamingUploader;
+
     fn get_object_prefix(&self, obj_id: u64) -> String {
         match self.engine_type {
             EngineType::S3 => prefix::s3::get_object_prefix(obj_id),
@@ -87,10 +96,11 @@ impl ObjectStore for OpendalObjectStore {
         }
     }
 
-    async fn streaming_upload(&self, path: &str) -> ObjectResult<BoxedStreamingUploader> {
-        Ok(Box::new(
-            OpendalStreamingUploader::new(self.op.clone(), path.to_string()).await?,
-        ))
+    async fn streaming_upload(&self, path: &str) -> ObjectResult<Self::StreamingUploader> {
+        Ok(
+            OpendalStreamingUploader::new(self.op.clone(), path.to_string(), self.config.clone())
+                .await?,
+        )
     }
 
     async fn read(&self, path: &str, range: impl ObjectRangeBounds) -> ObjectResult<Bytes> {
@@ -130,9 +140,31 @@ impl ObjectStore for OpendalObjectStore {
             ObjectError::internal("opendal streaming read error")
         ));
         let range: Range<u64> = (range.start as u64)..(range.end as u64);
-        let reader = self.op.reader_with(path).range(range).await?;
+        let reader = self
+            .op
+            .clone()
+            .layer(
+                RetryLayer::new()
+                    .with_min_delay(Duration::from_millis(
+                        self.config.retry.req_backoff_interval_ms,
+                    ))
+                    .with_max_delay(Duration::from_millis(
+                        self.config.retry.req_backoff_max_delay_ms,
+                    ))
+                    .with_max_times(self.config.retry.streaming_read_retry_attempts)
+                    .with_factor(self.config.retry.req_backoff_factor as f32)
+                    .with_jitter(),
+            )
+            .layer(TimeoutLayer::new().with_io_timeout(Duration::from_millis(
+                self.config.retry.streaming_read_attempt_timeout_ms,
+            )))
+            .reader_with(path)
+            .range(range)
+            .await?;
         let stream = reader.into_stream().map(|item| {
-            item.map_err(|e| ObjectError::internal(format!("OpendalError: {}", e.as_report())))
+            item.map_err(|e| {
+                ObjectError::internal(format!("reader into_stream fail {}", e.as_report()))
+            })
         });
 
         Ok(Box::pin(stream))
@@ -214,6 +246,10 @@ impl ObjectStore for OpendalObjectStore {
             EngineType::Fs => "Fs",
         }
     }
+
+    fn support_streaming_upload(&self) -> bool {
+        self.op.info().native_capability().write_can_multi
+    }
 }
 
 /// Store multiple parts in a map, and concatenate them on finish.
@@ -222,8 +258,24 @@ pub struct OpendalStreamingUploader {
 }
 
 impl OpendalStreamingUploader {
-    pub async fn new(op: Operator, path: String) -> ObjectResult<Self> {
+    pub async fn new(
+        op: Operator,
+        path: String,
+        config: Arc<ObjectStoreConfig>,
+    ) -> ObjectResult<Self> {
         let writer = op
+            .clone()
+            .layer(
+                RetryLayer::new()
+                    .with_min_delay(Duration::from_millis(config.retry.req_backoff_interval_ms))
+                    .with_max_delay(Duration::from_millis(config.retry.req_backoff_max_delay_ms))
+                    .with_max_times(config.retry.streaming_upload_retry_attempts)
+                    .with_factor(config.retry.req_backoff_factor as f32)
+                    .with_jitter(),
+            )
+            .layer(TimeoutLayer::new().with_io_timeout(Duration::from_millis(
+                config.retry.streaming_upload_attempt_timeout_ms,
+            )))
             .writer_with(&path)
             .concurrent(8)
             .buffer(OPENDAL_BUFFER_SIZE)
@@ -234,18 +286,21 @@ impl OpendalStreamingUploader {
 
 const OPENDAL_BUFFER_SIZE: usize = 16 * 1024 * 1024;
 
-#[async_trait::async_trait]
 impl StreamingUploader for OpendalStreamingUploader {
     async fn write_bytes(&mut self, data: Bytes) -> ObjectResult<()> {
         self.writer.write(data).await?;
+
         Ok(())
     }
 
-    async fn finish(mut self: Box<Self>) -> ObjectResult<()> {
+    async fn finish(mut self) -> ObjectResult<()> {
         match self.writer.close().await {
             Ok(_) => (),
             Err(err) => {
-                self.writer.abort().await?;
+                // Due to a bug in OpenDAL, calling `abort()` here may trigger unreachable code and cause panic.
+                // refer to https://github.com/apache/opendal/issues/4651
+                // This will be fixed after the next bump in the opendal version.
+                // self.writer.abort().await?;
                 return Err(err.into());
             }
         };
@@ -260,8 +315,6 @@ impl StreamingUploader for OpendalStreamingUploader {
 
 #[cfg(test)]
 mod tests {
-    use bytes::Bytes;
-
     use super::*;
 
     async fn list_all(prefix: &str, store: &OpendalObjectStore) -> Vec<ObjectMetadata> {
@@ -277,7 +330,7 @@ mod tests {
     #[tokio::test]
     async fn test_memory_upload() {
         let block = Bytes::from("123456");
-        let store = OpendalObjectStore::new_memory_engine().unwrap();
+        let store = OpendalObjectStore::test_new_memory_engine().unwrap();
         store.upload("/abc", block).await.unwrap();
 
         // No such object.
@@ -299,7 +352,7 @@ mod tests {
     async fn test_memory_metadata() {
         let block = Bytes::from("123456");
         let path = "/abc".to_string();
-        let obj_store = OpendalObjectStore::new_memory_engine().unwrap();
+        let obj_store = OpendalObjectStore::test_new_memory_engine().unwrap();
         obj_store.upload("/abc", block).await.unwrap();
 
         let err = obj_store.metadata("/not_exist").await.unwrap_err();
@@ -313,7 +366,7 @@ mod tests {
     async fn test_memory_delete_objects_and_list_object() {
         let block1 = Bytes::from("123456");
         let block2 = Bytes::from("987654");
-        let store = OpendalObjectStore::new_memory_engine().unwrap();
+        let store = OpendalObjectStore::test_new_memory_engine().unwrap();
         store.upload("abc", Bytes::from("123456")).await.unwrap();
         store.upload("prefix/abc", block1).await.unwrap();
 
diff --git a/src/object_store/src/object/opendal_engine/opendal_s3.rs b/src/object_store/src/object/opendal_engine/opendal_s3.rs
index c83498305d8e5..7a51cbb36955f 100644
--- a/src/object_store/src/object/opendal_engine/opendal_s3.rs
+++ b/src/object_store/src/object/opendal_engine/opendal_s3.rs
@@ -12,9 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
 use std::time::Duration;
 
-use opendal::layers::{LoggingLayer, RetryLayer};
+use opendal::layers::LoggingLayer;
 use opendal::raw::HttpClient;
 use opendal::services::S3;
 use opendal::Operator;
@@ -25,10 +26,7 @@ use crate::object::ObjectResult;
 
 impl OpendalObjectStore {
     /// create opendal s3 engine.
-    pub fn new_s3_engine(
-        bucket: String,
-        object_store_config: ObjectStoreConfig,
-    ) -> ObjectResult<Self> {
+    pub fn new_s3_engine(bucket: String, config: Arc<ObjectStoreConfig>) -> ObjectResult<Self> {
         // Create s3 builder.
         let mut builder = S3::default();
         builder.bucket(&bucket);
@@ -41,33 +39,22 @@ impl OpendalObjectStore {
             builder.enable_virtual_host_style();
         }
 
-        let http_client = Self::new_http_client(&object_store_config)?;
+        let http_client = Self::new_http_client(&config)?;
         builder.http_client(http_client);
 
         let op: Operator = Operator::new(builder)?
             .layer(LoggingLayer::default())
-            .layer(
-                RetryLayer::new()
-                    .with_min_delay(Duration::from_millis(
-                        object_store_config.s3.object_store_req_retry_interval_ms,
-                    ))
-                    .with_max_delay(Duration::from_millis(
-                        object_store_config.s3.object_store_req_retry_max_delay_ms,
-                    ))
-                    .with_max_times(object_store_config.s3.object_store_req_retry_max_attempts)
-                    .with_factor(1.0)
-                    .with_jitter(),
-            )
             .finish();
 
         Ok(Self {
             op,
             engine_type: EngineType::S3,
+            config,
         })
     }
 
     /// Creates a minio client. The server should be like `minio://key:secret@address:port/bucket`.
-    pub fn with_minio(server: &str, object_store_config: ObjectStoreConfig) -> ObjectResult<Self> {
+    pub fn new_minio_engine(server: &str, config: Arc<ObjectStoreConfig>) -> ObjectResult<Self> {
         let server = server.strip_prefix("minio://").unwrap();
         let (access_key_id, rest) = server.split_once(':').unwrap();
         let (secret_access_key, mut rest) = rest.split_once('@').unwrap();
@@ -92,27 +79,16 @@ impl OpendalObjectStore {
             .endpoint(&format!("{}{}", endpoint_prefix, address));
 
         builder.disable_config_load();
-        let http_client = Self::new_http_client(&object_store_config)?;
+        let http_client = Self::new_http_client(&config)?;
         builder.http_client(http_client);
         let op: Operator = Operator::new(builder)?
             .layer(LoggingLayer::default())
-            .layer(
-                RetryLayer::new()
-                    .with_min_delay(Duration::from_millis(
-                        object_store_config.s3.object_store_req_retry_interval_ms,
-                    ))
-                    .with_max_delay(Duration::from_millis(
-                        object_store_config.s3.object_store_req_retry_max_delay_ms,
-                    ))
-                    .with_max_times(object_store_config.s3.object_store_req_retry_max_attempts)
-                    .with_factor(1.0)
-                    .with_jitter(),
-            )
             .finish();
 
         Ok(Self {
             op,
             engine_type: EngineType::Minio,
+            config,
         })
     }
 
@@ -134,7 +110,7 @@ impl OpendalObjectStore {
     /// especially when sinking to the intermediate s3 bucket.
     pub fn new_s3_engine_with_credentials(
         bucket: &str,
-        object_store_config: ObjectStoreConfig,
+        config: Arc<ObjectStoreConfig>,
         aws_access_key_id: &str,
         aws_secret_access_key: &str,
         aws_region: &str,
@@ -148,28 +124,17 @@ impl OpendalObjectStore {
         builder.secret_access_key(aws_secret_access_key);
         builder.region(aws_region);
 
-        let http_client = Self::new_http_client(&object_store_config)?;
+        let http_client = Self::new_http_client(config.as_ref())?;
         builder.http_client(http_client);
 
         let op: Operator = Operator::new(builder)?
             .layer(LoggingLayer::default())
-            .layer(
-                RetryLayer::new()
-                    .with_min_delay(Duration::from_millis(
-                        object_store_config.s3.object_store_req_retry_interval_ms,
-                    ))
-                    .with_max_delay(Duration::from_millis(
-                        object_store_config.s3.object_store_req_retry_max_delay_ms,
-                    ))
-                    .with_max_times(object_store_config.s3.object_store_req_retry_max_attempts)
-                    .with_factor(1.1)
-                    .with_jitter(),
-            )
             .finish();
 
         Ok(Self {
             op,
             engine_type: EngineType::S3,
+            config,
         })
     }
 }
diff --git a/src/object_store/src/object/opendal_engine/oss.rs b/src/object_store/src/object/opendal_engine/oss.rs
index e215b6f93d31e..70fd6628f29b0 100644
--- a/src/object_store/src/object/opendal_engine/oss.rs
+++ b/src/object_store/src/object/opendal_engine/oss.rs
@@ -12,16 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use opendal::layers::{LoggingLayer, RetryLayer};
+use std::sync::Arc;
+
+use opendal::layers::LoggingLayer;
 use opendal::services::Oss;
 use opendal::Operator;
+use risingwave_common::config::ObjectStoreConfig;
 
 use super::{EngineType, OpendalObjectStore};
 use crate::object::ObjectResult;
 
 impl OpendalObjectStore {
     /// create opendal oss engine.
-    pub fn new_oss_engine(bucket: String, root: String) -> ObjectResult<Self> {
+    pub fn new_oss_engine(
+        bucket: String,
+        root: String,
+        config: Arc<ObjectStoreConfig>,
+    ) -> ObjectResult<Self> {
         // Create oss backend builder.
         let mut builder = Oss::default();
 
@@ -43,11 +50,11 @@ impl OpendalObjectStore {
 
         let op: Operator = Operator::new(builder)?
             .layer(LoggingLayer::default())
-            .layer(RetryLayer::default())
             .finish();
         Ok(Self {
             op,
             engine_type: EngineType::Oss,
+            config,
         })
     }
 }
diff --git a/src/object_store/src/object/opendal_engine/webhdfs.rs b/src/object_store/src/object/opendal_engine/webhdfs.rs
index 1f6b87b44fd5e..cb8a2ad1753b3 100644
--- a/src/object_store/src/object/opendal_engine/webhdfs.rs
+++ b/src/object_store/src/object/opendal_engine/webhdfs.rs
@@ -12,9 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use opendal::layers::{LoggingLayer, RetryLayer};
+use std::sync::Arc;
+
+use opendal::layers::LoggingLayer;
 use opendal::services::Webhdfs;
 use opendal::Operator;
+use risingwave_common::config::ObjectStoreConfig;
 
 use super::{EngineType, OpendalObjectStore};
 use crate::object::opendal_engine::ATOMIC_WRITE_DIR;
@@ -22,7 +25,11 @@ use crate::object::ObjectResult;
 
 impl OpendalObjectStore {
     /// create opendal webhdfs engine.
-    pub fn new_webhdfs_engine(endpoint: String, root: String) -> ObjectResult<Self> {
+    pub fn new_webhdfs_engine(
+        endpoint: String,
+        root: String,
+        config: Arc<ObjectStoreConfig>,
+    ) -> ObjectResult<Self> {
         // Create webhdfs backend builder.
         let mut builder = Webhdfs::default();
         // Set the name node for webhdfs.
@@ -35,11 +42,11 @@ impl OpendalObjectStore {
         builder.atomic_write_dir(&atomic_write_dir);
         let op: Operator = Operator::new(builder)?
             .layer(LoggingLayer::default())
-            .layer(RetryLayer::default())
             .finish();
         Ok(Self {
             op,
             engine_type: EngineType::Webhdfs,
+            config,
         })
     }
 }
diff --git a/src/object_store/src/object/s3.rs b/src/object_store/src/object/s3.rs
index 9d48f1175976b..6c72ced36563d 100644
--- a/src/object_store/src/object/s3.rs
+++ b/src/object_store/src/object/s3.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::borrow::BorrowMut;
 use std::cmp;
 use std::collections::VecDeque;
 use std::ops::Range;
@@ -20,12 +21,21 @@ use std::sync::Arc;
 use std::task::{ready, Context, Poll};
 use std::time::Duration;
 
+use await_tree::InstrumentAwait;
 use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::error::BoxError;
+use aws_sdk_s3::operation::abort_multipart_upload::AbortMultipartUploadError;
+use aws_sdk_s3::operation::complete_multipart_upload::CompleteMultipartUploadError;
+use aws_sdk_s3::operation::create_multipart_upload::CreateMultipartUploadError;
+use aws_sdk_s3::operation::delete_object::DeleteObjectError;
+use aws_sdk_s3::operation::delete_objects::DeleteObjectsError;
 use aws_sdk_s3::operation::get_object::builders::GetObjectFluentBuilder;
 use aws_sdk_s3::operation::get_object::GetObjectError;
+use aws_sdk_s3::operation::head_object::HeadObjectError;
 use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Error;
+use aws_sdk_s3::operation::put_object::PutObjectError;
 use aws_sdk_s3::operation::upload_part::UploadPartOutput;
-use aws_sdk_s3::primitives::{ByteStream, ByteStreamError};
+use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
     AbortIncompleteMultipartUpload, BucketLifecycleConfiguration, CompletedMultipartUpload,
     CompletedPart, Delete, ExpirationStatus, LifecycleRule, LifecycleRuleFilter, ObjectIdentifier,
@@ -37,26 +47,25 @@ use aws_smithy_runtime_api::client::http::HttpClient;
 use aws_smithy_runtime_api::client::result::SdkError;
 use aws_smithy_types::body::SdkBody;
 use aws_smithy_types::error::metadata::ProvideErrorMetadata;
-use aws_smithy_types::retry::RetryConfig;
-use either::Either;
 use fail::fail_point;
 use futures::future::{try_join_all, BoxFuture, FutureExt};
 use futures::{stream, Stream, StreamExt, TryStreamExt};
 use hyper::Body;
 use itertools::Itertools;
-use risingwave_common::config::{ObjectStoreConfig, S3ObjectStoreConfig};
+use risingwave_common::config::ObjectStoreConfig;
 use risingwave_common::monitor::connection::monitor_connector;
 use risingwave_common::range::RangeBoundsExt;
 use thiserror_ext::AsReport;
 use tokio::task::JoinHandle;
-use tokio_retry::strategy::{jitter, ExponentialBackoff};
 
 use super::object_metrics::ObjectStoreMetrics;
 use super::{
-    prefix, BoxedStreamingUploader, Bytes, ObjectError, ObjectMetadata, ObjectRangeBounds,
+    prefix, retry_request, Bytes, ObjectError, ObjectErrorInner, ObjectMetadata, ObjectRangeBounds,
     ObjectResult, ObjectStore, StreamingUploader,
 };
-use crate::object::{try_update_failure_metric, ObjectDataStream, ObjectMetadataIter};
+use crate::object::{
+    try_update_failure_metric, ObjectDataStream, ObjectMetadataIter, OperationType,
+};
 
 type PartId = i32;
 
@@ -96,6 +105,8 @@ pub struct S3StreamingUploader {
     not_uploaded_len: usize,
     /// To record metrics for uploading part.
     metrics: Arc<ObjectStoreMetrics>,
+
+    config: Arc<ObjectStoreConfig>,
 }
 
 impl S3StreamingUploader {
@@ -105,6 +116,7 @@ impl S3StreamingUploader {
         part_size: usize,
         key: String,
         metrics: Arc<ObjectStoreMetrics>,
+        config: Arc<ObjectStoreConfig>,
     ) -> S3StreamingUploader {
         Self {
             client,
@@ -117,22 +129,46 @@ impl S3StreamingUploader {
             buf: Default::default(),
             not_uploaded_len: 0,
             metrics,
+            config,
         }
     }
 
     async fn upload_next_part(&mut self) -> ObjectResult<()> {
-        let operation_type = "s3_upload_part";
+        let operation_type = OperationType::StreamingUpload;
+        let operation_type_str = operation_type.as_str();
 
         // Lazily create multipart upload.
         if self.upload_id.is_none() {
-            let resp = self
-                .client
-                .create_multipart_upload()
-                .bucket(&self.bucket)
-                .key(&self.key)
-                .send()
-                .await?;
-            self.upload_id = Some(resp.upload_id.unwrap());
+            let builder = || async {
+                self.client
+                    .create_multipart_upload()
+                    .bucket(&self.bucket)
+                    .key(&self.key)
+                    .send()
+                    .await
+                    .map_err(|err| {
+                        set_error_should_retry::<CreateMultipartUploadError>(
+                            self.config.clone(),
+                            err.into(),
+                        )
+                    })
+            };
+
+            let resp = retry_request(
+                builder,
+                &self.config,
+                OperationType::StreamingUploadInit,
+                self.metrics.clone(),
+            )
+            .await;
+
+            try_update_failure_metric(
+                &self.metrics,
+                &resp,
+                OperationType::StreamingUploadInit.as_str(),
+            );
+
+            self.upload_id = Some(resp?.upload_id.unwrap());
         }
 
         // Get the data to upload for the next part.
@@ -156,33 +192,46 @@ impl S3StreamingUploader {
         let metrics = self.metrics.clone();
         metrics
             .operation_size
-            .with_label_values(&[operation_type])
+            .with_label_values(&[operation_type_str])
             .observe(len as f64);
+        let config = self.config.clone();
 
         self.join_handles.push(tokio::spawn(async move {
             let _timer = metrics
                 .operation_latency
-                .with_label_values(&["s3", operation_type])
+                .with_label_values(&["s3", operation_type_str])
                 .start_timer();
-            let upload_output_res = client_cloned
-                .upload_part()
-                .bucket(bucket)
-                .key(key)
-                .upload_id(upload_id)
-                .part_number(part_id)
-                .body(get_upload_body(data))
-                .content_length(len as i64)
-                .send()
-                .await
-                .map_err(Into::into);
-            try_update_failure_metric(&metrics, &upload_output_res, operation_type);
-            Ok((part_id, upload_output_res?))
+
+            let builder = || async {
+                client_cloned
+                    .upload_part()
+                    .bucket(bucket.clone())
+                    .key(key.clone())
+                    .upload_id(upload_id.clone())
+                    .part_number(part_id)
+                    .body(get_upload_body(data.clone()))
+                    .content_length(len as i64)
+                    .send()
+                    .await
+                    .map_err(|err| {
+                        set_error_should_retry::<CreateMultipartUploadError>(
+                            config.clone(),
+                            err.into(),
+                        )
+                    })
+            };
+
+            let res = retry_request(builder, &config, operation_type, metrics.clone()).await;
+            try_update_failure_metric(&metrics, &res, operation_type_str);
+            Ok((part_id, res?))
         }));
 
         Ok(())
     }
 
     async fn flush_multipart_and_complete(&mut self) -> ObjectResult<()> {
+        let operation_type = OperationType::StreamingUploadFinish;
+
         if !self.buf.is_empty() {
             self.upload_next_part().await?;
         }
@@ -210,18 +259,30 @@ impl S3StreamingUploader {
                 .collect_vec(),
         );
 
-        self.client
-            .complete_multipart_upload()
-            .bucket(&self.bucket)
-            .key(&self.key)
-            .upload_id(self.upload_id.as_ref().unwrap())
-            .multipart_upload(
-                CompletedMultipartUpload::builder()
-                    .set_parts(completed_parts)
-                    .build(),
-            )
-            .send()
-            .await?;
+        let builder = || async {
+            self.client
+                .complete_multipart_upload()
+                .bucket(&self.bucket)
+                .key(&self.key)
+                .upload_id(self.upload_id.as_ref().unwrap())
+                .multipart_upload(
+                    CompletedMultipartUpload::builder()
+                        .set_parts(completed_parts.clone())
+                        .build(),
+                )
+                .send()
+                .await
+                .map_err(|err| {
+                    set_error_should_retry::<CompleteMultipartUploadError>(
+                        self.config.clone(),
+                        err.into(),
+                    )
+                })
+        };
+
+        let res = retry_request(builder, &self.config, operation_type, self.metrics.clone()).await;
+        try_update_failure_metric(&self.metrics, &res, operation_type.as_str());
+        let _res = res?;
 
         Ok(())
     }
@@ -233,12 +294,14 @@ impl S3StreamingUploader {
             .key(&self.key)
             .upload_id(self.upload_id.as_ref().unwrap())
             .send()
-            .await?;
+            .await
+            .map_err(|err| {
+                set_error_should_retry::<AbortMultipartUploadError>(self.config.clone(), err.into())
+            })?;
         Ok(())
     }
 }
 
-#[async_trait::async_trait]
 impl StreamingUploader for S3StreamingUploader {
     async fn write_bytes(&mut self, data: Bytes) -> ObjectResult<()> {
         fail_point!("s3_write_bytes_err", |_| Err(ObjectError::internal(
@@ -249,7 +312,9 @@ impl StreamingUploader for S3StreamingUploader {
         self.buf.push(data);
 
         if self.not_uploaded_len >= self.part_size {
-            self.upload_next_part().await?;
+            self.upload_next_part()
+                .verbose_instrument_await("s3_upload_next_part")
+                .await?;
             self.not_uploaded_len = 0;
         }
         Ok(())
@@ -258,7 +323,7 @@ impl StreamingUploader for S3StreamingUploader {
     /// If the multipart upload has not been initiated, we can use `PutObject` instead to save the
     /// `CreateMultipartUpload` and `CompleteMultipartUpload` requests. Otherwise flush the
     /// remaining data of the buffer to S3 as a new part.
-    async fn finish(mut self: Box<Self>) -> ObjectResult<()> {
+    async fn finish(mut self) -> ObjectResult<()> {
         fail_point!("s3_finish_streaming_upload_err", |_| Err(
             ObjectError::internal("s3 finish streaming upload error")
         ));
@@ -269,17 +334,37 @@ impl StreamingUploader for S3StreamingUploader {
                 debug_assert_eq!(self.not_uploaded_len, 0);
                 Err(ObjectError::internal("upload empty object"))
             } else {
-                self.client
-                    .put_object()
-                    .bucket(&self.bucket)
-                    .body(get_upload_body(self.buf))
-                    .content_length(self.not_uploaded_len as i64)
-                    .key(&self.key)
-                    .send()
-                    .await?;
+                let operation_type = OperationType::Upload;
+                let builder = || async {
+                    self.client
+                        .put_object()
+                        .bucket(&self.bucket)
+                        .body(get_upload_body(self.buf.clone()))
+                        .content_length(self.not_uploaded_len as i64)
+                        .key(&self.key)
+                        .send()
+                        .verbose_instrument_await("s3_put_object")
+                        .await
+                        .map_err(|err| {
+                            set_error_should_retry::<PutObjectError>(
+                                self.config.clone(),
+                                err.into(),
+                            )
+                        })
+                };
+
+                let res =
+                    retry_request(builder, &self.config, operation_type, self.metrics.clone())
+                        .await;
+                try_update_failure_metric(&self.metrics, &res, operation_type.as_str());
+                res?;
                 Ok(())
             }
-        } else if let Err(e) = self.flush_multipart_and_complete().await {
+        } else if let Err(e) = self
+            .flush_multipart_and_complete()
+            .verbose_instrument_await("s3_flush_multipart_and_complete")
+            .await
+        {
             tracing::warn!(key = self.key, error = %e.as_report(), "Failed to upload object");
             self.abort_multipart_upload().await?;
             Err(e)
@@ -302,6 +387,7 @@ fn get_upload_body(data: Vec<Bytes>) -> ByteStream {
 
 /// Object store with S3 backend
 /// The full path to a file on S3 would be `s3://bucket/<data_directory>/prefix/file`
+#[derive(Clone)]
 pub struct S3ObjectStore {
     client: Client,
     bucket: String,
@@ -309,11 +395,13 @@ pub struct S3ObjectStore {
     /// For S3 specific metrics.
     metrics: Arc<ObjectStoreMetrics>,
 
-    config: ObjectStoreConfig,
+    config: Arc<ObjectStoreConfig>,
 }
 
 #[async_trait::async_trait]
 impl ObjectStore for S3ObjectStore {
+    type StreamingUploader = S3StreamingUploader;
+
     fn get_object_prefix(&self, obj_id: u64) -> String {
         // Delegate to static method to avoid creating an `S3ObjectStore` in unit test.
         prefix::s3::get_object_prefix(obj_id)
@@ -332,22 +420,26 @@ impl ObjectStore for S3ObjectStore {
                 .body(ByteStream::from(obj))
                 .key(path)
                 .send()
-                .await?;
+                .await
+                .map_err(|err| {
+                    set_error_should_retry::<PutObjectError>(self.config.clone(), err.into())
+                })?;
             Ok(())
         }
     }
 
-    async fn streaming_upload(&self, path: &str) -> ObjectResult<BoxedStreamingUploader> {
+    async fn streaming_upload(&self, path: &str) -> ObjectResult<Self::StreamingUploader> {
         fail_point!("s3_streaming_upload_err", |_| Err(ObjectError::internal(
             "s3 streaming upload error"
         )));
-        Ok(Box::new(S3StreamingUploader::new(
+        Ok(S3StreamingUploader::new(
             self.client.clone(),
             self.bucket.clone(),
             self.part_size,
             path.to_string(),
             self.metrics.clone(),
-        )))
+            self.config.clone(),
+        ))
     }
 
     /// Amazon S3 doesn't support retrieving multiple ranges of data per GET request.
@@ -356,37 +448,22 @@ impl ObjectStore for S3ObjectStore {
             "s3 read error"
         )));
 
-        // retry if occurs AWS EC2 HTTP timeout error.
-        let val = tokio_retry::RetryIf::spawn(
-            self.get_retry_strategy(),
-            || async {
-                match self.obj_store_request(path, range.clone()).send().await {
-                    Ok(resp) => {
-                        let val = resp
-                            .body
-                            .collect()
-                            .await
-                            .map_err(either::Right)?
-                            .into_bytes();
-                        Ok(val)
-                    }
-                    Err(err) => {
-                        if let SdkError::DispatchFailure(e) = &err
-                            && e.is_timeout()
-                        {
-                            self.metrics
-                                .request_retry_count
-                                .with_label_values(&["read"])
-                                .inc();
-                        }
-
-                        Err(either::Left(err))
-                    }
-                }
-            },
-            RetryCondition::new(&self.config.s3),
-        )
-        .await?;
+        let val = match self.obj_store_request(path, range.clone()).send().await {
+            Ok(resp) => resp
+                .body
+                .collect()
+                .await
+                .map_err(|err| {
+                    set_error_should_retry::<GetObjectError>(self.config.clone(), err.into())
+                })?
+                .into_bytes(),
+            Err(sdk_err) => {
+                return Err(set_error_should_retry::<GetObjectError>(
+                    self.config.clone(),
+                    sdk_err.into(),
+                ));
+            }
+        };
 
         if let Some(len) = range.len()
             && len != val.len()
@@ -413,7 +490,10 @@ impl ObjectStore for S3ObjectStore {
             .bucket(&self.bucket)
             .key(path)
             .send()
-            .await?;
+            .await
+            .map_err(|err| {
+                set_error_should_retry::<HeadObjectError>(self.config.clone(), err.into())
+            })?;
         Ok(ObjectMetadata {
             key: path.to_owned(),
             last_modified: resp
@@ -432,33 +512,20 @@ impl ObjectStore for S3ObjectStore {
         path: &str,
         range: Range<usize>,
     ) -> ObjectResult<ObjectDataStream> {
-        fail_point!("s3_streaming_read_err", |_| Err(ObjectError::internal(
-            "s3 streaming read error"
-        )));
+        fail_point!("s3_streaming_read_init_err", |_| Err(
+            ObjectError::internal("s3 streaming read init error")
+        ));
 
-        // retry if occurs AWS EC2 HTTP timeout error.
-        let resp = tokio_retry::RetryIf::spawn(
-            self.get_retry_strategy(),
-            || async {
-                match self.obj_store_request(path, range.clone()).send().await {
-                    Ok(resp) => Ok(resp),
-                    Err(err) => {
-                        if let SdkError::DispatchFailure(e) = &err
-                            && e.is_timeout()
-                        {
-                            self.metrics
-                                .request_retry_count
-                                .with_label_values(&["streaming_read"])
-                                .inc();
-                        }
+        let resp = match self.obj_store_request(path, range.clone()).send().await {
+            Ok(resp) => resp,
+            Err(sdk_err) => {
+                return Err(set_error_should_retry::<GetObjectError>(
+                    self.config.clone(),
+                    sdk_err.into(),
+                ));
+            }
+        };
 
-                        Err(either::Left(err))
-                    }
-                }
-            },
-            RetryCondition::new(&self.config.s3),
-        )
-        .await?;
         let reader = FuturesStreamCompatByteStream::new(resp.body);
 
         Ok(Box::pin(
@@ -479,7 +546,10 @@ impl ObjectStore for S3ObjectStore {
             .bucket(&self.bucket)
             .key(path)
             .send()
-            .await?;
+            .await
+            .map_err(|err| {
+                set_error_should_retry::<DeleteObjectError>(self.config.clone(), err.into())
+            })?;
         Ok(())
     }
 
@@ -509,7 +579,9 @@ impl ObjectStore for S3ObjectStore {
                 .delete_objects()
                 .bucket(&self.bucket)
                 .delete(delete_builder.build().unwrap()).send()
-                .await?;
+                .await.map_err(|err| {
+                    set_error_should_retry::<DeleteObjectsError>(self.config.clone(),err.into())
+                })?;
 
             // Check if there were errors.
             if !delete_output.errors().is_empty() {
@@ -531,6 +603,7 @@ impl ObjectStore for S3ObjectStore {
             self.client.clone(),
             self.bucket.clone(),
             prefix.to_string(),
+            self.config.clone(),
         )))
     }
 
@@ -579,11 +652,9 @@ impl S3ObjectStore {
     pub async fn new_with_config(
         bucket: String,
         metrics: Arc<ObjectStoreMetrics>,
-        config: ObjectStoreConfig,
+        config: Arc<ObjectStoreConfig>,
     ) -> Self {
-        let sdk_config_loader = aws_config::from_env()
-            .retry_config(RetryConfig::standard().with_max_attempts(4))
-            .http_client(Self::new_http_client(&config));
+        let sdk_config_loader = aws_config::from_env().http_client(Self::new_http_client(&config));
 
         // Retry 3 times if we get server-side errors or throttling errors
         let client = match std::env::var("RW_S3_ENDPOINT") {
@@ -650,10 +721,10 @@ impl S3ObjectStore {
     }
 
     /// Creates a minio client. The server should be like `minio://key:secret@address:port/bucket`.
-    pub async fn with_minio(
+    pub async fn new_minio_engine(
         server: &str,
         metrics: Arc<ObjectStoreMetrics>,
-        s3_object_store_config: ObjectStoreConfig,
+        object_store_config: Arc<ObjectStoreConfig>,
     ) -> Self {
         let server = server.strip_prefix("minio://").unwrap();
         let (access_key_id, rest) = server.split_once(':').unwrap();
@@ -690,11 +761,11 @@ impl S3ObjectStore {
         .identity_cache(
             aws_sdk_s3::config::IdentityCache::lazy()
                 .load_timeout(Duration::from_secs(
-                    s3_object_store_config.s3.identity_resolution_timeout_s,
+                    object_store_config.s3.identity_resolution_timeout_s,
                 ))
                 .build(),
         )
-        .http_client(Self::new_http_client(&s3_object_store_config))
+        .http_client(Self::new_http_client(&object_store_config))
         .behavior_version_latest()
         .stalled_stream_protection(aws_sdk_s3::config::StalledStreamProtectionConfig::disabled());
         let config = builder
@@ -708,7 +779,7 @@ impl S3ObjectStore {
             bucket: bucket.to_string(),
             part_size: MINIO_PART_SIZE,
             metrics,
-            config: s3_object_store_config,
+            config: object_store_config,
         }
     }
 
@@ -839,16 +910,6 @@ impl S3ObjectStore {
         }
         is_expiration_configured
     }
-
-    #[inline(always)]
-    fn get_retry_strategy(&self) -> impl Iterator<Item = Duration> {
-        ExponentialBackoff::from_millis(self.config.s3.object_store_req_retry_interval_ms)
-            .max_delay(Duration::from_millis(
-                self.config.s3.object_store_req_retry_max_delay_ms,
-            ))
-            .take(self.config.s3.object_store_req_retry_max_attempts)
-            .map(jitter)
-    }
 }
 
 struct S3ObjectIter {
@@ -862,16 +923,15 @@ struct S3ObjectIter {
     send_future: Option<
         BoxFuture<
             'static,
-            Result<
-                (Vec<ObjectMetadata>, Option<String>, Option<bool>),
-                SdkError<ListObjectsV2Error, aws_smithy_runtime_api::http::Response<SdkBody>>,
-            >,
+            Result<(Vec<ObjectMetadata>, Option<String>, Option<bool>), ObjectError>,
         >,
     >,
+
+    config: Arc<ObjectStoreConfig>,
 }
 
 impl S3ObjectIter {
-    fn new(client: Client, bucket: String, prefix: String) -> Self {
+    fn new(client: Client, bucket: String, prefix: String, config: Arc<ObjectStoreConfig>) -> Self {
         Self {
             buffer: VecDeque::default(),
             client,
@@ -880,6 +940,7 @@ impl S3ObjectIter {
             next_continuation_token: None,
             is_truncated: Some(true),
             send_future: None,
+            config,
         }
     }
 }
@@ -902,7 +963,7 @@ impl Stream for S3ObjectIter {
                 }
                 Err(e) => {
                     self.send_future = None;
-                    Poll::Ready(Some(Err(e.into())))
+                    Poll::Ready(Some(Err(e)))
                 }
             };
         }
@@ -917,6 +978,7 @@ impl Stream for S3ObjectIter {
         if let Some(continuation_token) = self.next_continuation_token.as_ref() {
             request = request.continuation_token(continuation_token);
         }
+        let config = self.config.clone();
         let f = async move {
             match request.send().await {
                 Ok(r) => {
@@ -936,7 +998,10 @@ impl Stream for S3ObjectIter {
                     let next_continuation_token = r.next_continuation_token;
                     Ok((more, next_continuation_token, is_truncated))
                 }
-                Err(e) => Err(e),
+                Err(e) => Err(set_error_should_retry::<ListObjectsV2Error>(
+                    config,
+                    e.into(),
+                )),
             }
         };
         self.send_future = Some(Box::pin(f));
@@ -944,77 +1009,79 @@ impl Stream for S3ObjectIter {
     }
 }
 
-type RetryError = Either<
-    SdkError<GetObjectError, aws_smithy_runtime_api::http::Response<SdkBody>>,
-    ByteStreamError,
->;
-
-impl From<RetryError> for ObjectError {
-    fn from(e: RetryError) -> Self {
-        match e {
-            Either::Left(e) => e.into(),
-            Either::Right(e) => e.into(),
-        }
-    }
-}
-
-struct RetryCondition {
-    retry_unknown_service_error: bool,
-    retryable_service_error_codes: Vec<String>,
-}
+fn set_error_should_retry<E>(config: Arc<ObjectStoreConfig>, object_err: ObjectError) -> ObjectError
+where
+    E: ProvideErrorMetadata + Into<BoxError> + Sync + Send + std::error::Error + 'static,
+{
+    let not_found = object_err.is_object_not_found_error();
 
-impl RetryCondition {
-    fn new(config: &S3ObjectStoreConfig) -> Self {
-        Self {
-            retry_unknown_service_error: config.developer.object_store_retry_unknown_service_error
-                || config.retry_unknown_service_error,
-            retryable_service_error_codes: config
-                .developer
-                .object_store_retryable_service_error_codes
-                .clone(),
-        }
+    if not_found {
+        return object_err;
     }
-}
 
-impl tokio_retry::Condition<RetryError> for RetryCondition {
-    fn should_retry(&mut self, err: &RetryError) -> bool {
-        match err {
-            Either::Left(err) => match err {
-                SdkError::DispatchFailure(e) => {
+    let mut inner = object_err.into_inner();
+    match inner.borrow_mut() {
+        ObjectErrorInner::S3 {
+            should_retry,
+            inner,
+        } => {
+            let sdk_err = inner
+                .as_ref()
+                .downcast_ref::<SdkError<E, aws_smithy_runtime_api::http::Response<SdkBody>>>();
+
+            let err_should_retry = match sdk_err {
+                Some(SdkError::DispatchFailure(e)) => {
                     if e.is_timeout() {
                         tracing::warn!(target: "http_timeout_retry", "{e:?} occurs, retry S3 get_object request.");
-                        return true;
+                        true
+                    } else {
+                        false
                     }
                 }
-                SdkError::ServiceError(e) => match e.err().code() {
-                    None => {
-                        if self.retry_unknown_service_error {
-                            tracing::warn!(target: "unknown_service_error", "{e:?} occurs, retry S3 get_object request.");
-                            return true;
+
+                Some(SdkError::ServiceError(e)) => {
+                    let retry = match e.err().code() {
+                        None => {
+                            if config.s3.developer.object_store_retry_unknown_service_error
+                                || config.s3.retry_unknown_service_error
+                            {
+                                tracing::warn!(target: "unknown_service_error", "{e:?} occurs, retry S3 get_object request.");
+                                true
+                            } else {
+                                false
+                            }
                         }
-                    }
-                    Some(code) => {
-                        if self
-                            .retryable_service_error_codes
-                            .iter()
-                            .any(|s| s.as_str().eq_ignore_ascii_case(code))
-                        {
-                            tracing::warn!(target: "retryable_service_error", "{e:?} occurs, retry S3 get_object request.");
-                            return true;
+                        Some(code) => {
+                            if config
+                                .s3
+                                .developer
+                                .object_store_retryable_service_error_codes
+                                .iter()
+                                .any(|s| s.as_str().eq_ignore_ascii_case(code))
+                            {
+                                tracing::warn!(target: "retryable_service_error", "{e:?} occurs, retry S3 get_object request.");
+                                true
+                            } else {
+                                false
+                            }
                         }
-                    }
-                },
-                _ => {}
-            },
-            Either::Right(_) => {
-                // Unfortunately `ErrorKind` of `ByteStreamError` is not accessible.
-                // Always returns true and relies on req_retry_max_attempts to avoid infinite loop.
-                return true;
-            }
+                    };
+
+                    retry
+                }
+
+                Some(SdkError::TimeoutError(_err)) => true,
+
+                _ => false,
+            };
+
+            *should_retry = err_should_retry;
         }
 
-        false
+        _ => unreachable!(),
     }
+
+    ObjectError::from(inner)
 }
 
 #[cfg(test)]
diff --git a/src/object_store/src/object/sim/mod.rs b/src/object_store/src/object/sim/mod.rs
index 31f34cbcd3267..4c91531020ea3 100644
--- a/src/object_store/src/object/sim/mod.rs
+++ b/src/object_store/src/object/sim/mod.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![cfg(madsim)]
+
 mod client;
 mod error;
 use bytes::{BufMut, BytesMut};
@@ -33,8 +35,8 @@ use risingwave_common::range::RangeBoundsExt;
 use self::client::Client;
 use self::service::Response;
 use super::{
-    BoxedStreamingUploader, Bytes, ObjectDataStream, ObjectError, ObjectMetadata,
-    ObjectMetadataIter, ObjectRangeBounds, ObjectResult, ObjectStore, StreamingUploader,
+    Bytes, ObjectDataStream, ObjectError, ObjectMetadata, ObjectMetadataIter, ObjectRangeBounds,
+    ObjectResult, ObjectStore, StreamingUploader,
 };
 
 pub struct SimStreamingUploader {
@@ -53,14 +55,13 @@ impl SimStreamingUploader {
     }
 }
 
-#[async_trait::async_trait]
 impl StreamingUploader for SimStreamingUploader {
     async fn write_bytes(&mut self, data: Bytes) -> ObjectResult<()> {
         self.buf.put(data);
         Ok(())
     }
 
-    async fn finish(mut self: Box<Self>) -> ObjectResult<()> {
+    async fn finish(mut self) -> ObjectResult<()> {
         if self.buf.is_empty() {
             Err(ObjectError::internal("upload empty object"))
         } else {
@@ -115,6 +116,8 @@ pub struct SimObjectStore {
 
 #[async_trait::async_trait]
 impl ObjectStore for SimObjectStore {
+    type StreamingUploader = SimStreamingUploader;
+
     fn get_object_prefix(&self, _obj_id: u64) -> String {
         String::default()
     }
@@ -136,11 +139,11 @@ impl ObjectStore for SimObjectStore {
         }
     }
 
-    async fn streaming_upload(&self, path: &str) -> ObjectResult<BoxedStreamingUploader> {
-        Ok(Box::new(SimStreamingUploader::new(
+    async fn streaming_upload(&self, path: &str) -> ObjectResult<Self::StreamingUploader> {
+        Ok(SimStreamingUploader::new(
             self.client.clone(),
             path.to_string(),
-        )))
+        ))
     }
 
     async fn read(&self, path: &str, range: impl ObjectRangeBounds) -> ObjectResult<Bytes> {
diff --git a/src/prost/build.rs b/src/prost/build.rs
index e031e5cfb01ae..6cbfa82225e69 100644
--- a/src/prost/build.rs
+++ b/src/prost/build.rs
@@ -53,6 +53,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         "task_service",
         "telemetry",
         "user",
+        "secret",
     ];
     let protos: Vec<String> = proto_files
         .iter()
@@ -65,6 +66,17 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         ".plan_common.ExternalTableDesc",
         ".hummock.CompactTask",
         ".catalog.StreamSourceInfo",
+        ".catalog.Source",
+        ".catalog.Sink",
+        ".catalog.View",
+        ".connector_service.ValidateSourceRequest",
+        ".connector_service.GetEventStreamRequest",
+        ".connector_service.SinkParam",
+        ".stream_plan.SinkDesc",
+        ".stream_plan.StreamFsFetch",
+        ".stream_plan.SourceBackfillNode",
+        ".stream_plan.StreamSource",
+        ".batch_plan.SourceNode",
     ];
 
     // Build protobuf structs.
@@ -138,6 +150,13 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .type_attribute("plan_common.AdditionalColumnHeader", "#[derive(Eq, Hash)]")
         .type_attribute("plan_common.AdditionalColumnHeaders", "#[derive(Eq, Hash)]")
         .type_attribute("plan_common.AdditionalColumnOffset", "#[derive(Eq, Hash)]")
+        .type_attribute("plan_common.AdditionalDatabaseName", "#[derive(Eq, Hash)]")
+        .type_attribute("plan_common.AdditionalSchemaName", "#[derive(Eq, Hash)]")
+        .type_attribute("plan_common.AdditionalTableName", "#[derive(Eq, Hash)]")
+        .type_attribute(
+            "plan_common.AdditionalCollectionName",
+            "#[derive(Eq, Hash)]",
+        )
         .type_attribute("common.ColumnOrder", "#[derive(Eq, Hash)]")
         .type_attribute("common.OrderType", "#[derive(Eq, Hash)]")
         .type_attribute("common.Buffer", "#[derive(Eq)]")
diff --git a/src/prost/helpers/src/generate.rs b/src/prost/helpers/src/generate.rs
index 432f233e74cb9..f674fec5aa2cf 100644
--- a/src/prost/helpers/src/generate.rs
+++ b/src/prost/helpers/src/generate.rs
@@ -17,8 +17,8 @@ use quote::quote;
 use syn::ext::IdentExt;
 use syn::spanned::Spanned;
 use syn::{
-    self, Error, Expr, ExprLit, Field, GenericArgument, Lit, Meta, PathArguments, PathSegment,
-    Result, Type,
+    Error, Expr, ExprLit, Field, GenericArgument, Lit, Meta, PathArguments, PathSegment, Result,
+    Type,
 };
 
 fn extract_type_from_option(option_segment: &PathSegment) -> Type {
diff --git a/src/prost/src/lib.rs b/src/prost/src/lib.rs
index 825e721891fed..27d0523b84115 100644
--- a/src/prost/src/lib.rs
+++ b/src/prost/src/lib.rs
@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// for derived code of `Message`
 #![expect(clippy::all)]
 #![expect(clippy::doc_markdown)]
-#![allow(non_snake_case)] // for derived code of `Message`
 #![feature(lint_reasons)]
 
 use std::str::FromStr;
@@ -56,7 +56,7 @@ pub mod batch_plan;
 #[cfg_attr(madsim, path = "sim/task_service.rs")]
 pub mod task_service;
 #[rustfmt::skip]
-#[cfg_attr(madsim, path="sim/connector_service.rs")]
+#[cfg_attr(madsim, path = "sim/connector_service.rs")]
 pub mod connector_service;
 #[rustfmt::skip]
 #[cfg_attr(madsim, path = "sim/stream_plan.rs")]
@@ -91,6 +91,10 @@ pub mod health;
 #[rustfmt::skip]
 #[path = "sim/telemetry.rs"]
 pub mod telemetry;
+
+#[rustfmt::skip]
+#[path = "sim/secret.rs"]
+pub mod secret;
 #[rustfmt::skip]
 #[path = "connector_service.serde.rs"]
 pub mod connector_service_serde;
@@ -158,6 +162,10 @@ pub mod java_binding_serde;
 #[path = "telemetry.serde.rs"]
 pub mod telemetry_serde;
 
+#[rustfmt::skip]
+#[path = "secret.serde.rs"]
+pub mod secret_serde;
+
 #[derive(Clone, PartialEq, Eq, Debug, Error)]
 #[error("field `{0}` not found")]
 pub struct PbFieldNotFound(pub &'static str);
diff --git a/src/risedevtool/Cargo.toml b/src/risedevtool/Cargo.toml
index 78ec4b5a63e45..b8a2ca9db14f9 100644
--- a/src/risedevtool/Cargo.toml
+++ b/src/risedevtool/Cargo.toml
@@ -23,7 +23,7 @@ clap = { workspace = true }
 console = "0.15"
 fs-err = "2.11.0"
 glob = "0.3"
-google-cloud-pubsub = "0.24"
+google-cloud-pubsub = "0.25"
 indicatif = "0.17"
 itertools = { workspace = true }
 rdkafka = { workspace = true }
diff --git a/src/risedevtool/common.toml b/src/risedevtool/common.toml
index 97ddcd0485f9d..391d52f399cf9 100644
--- a/src/risedevtool/common.toml
+++ b/src/risedevtool/common.toml
@@ -5,7 +5,6 @@ OS = { source = "${CARGO_MAKE_RUST_TARGET_OS}", mapping = { linux = "linux", mac
 ARCH = { source = "${CARGO_MAKE_RUST_TARGET_ARCH}", mapping = { x86_64 = "amd64", aarch64 = "arm64" } }
 SYSTEM = "${OS}-${ARCH}"
 SYSTEM_UNDERSCORE = "${OS}_${ARCH}"
-SYSTEM_AMD64 = "${OS}-amd64" # some components do not support darwin-arm64 for now, use amd64 for fallback
 PREFIX = "${PWD}/.risingwave"
 JAVA_DIR = "${PWD}/java"
 PREFIX_USR_BIN = "${PWD}/.bin"
diff --git a/src/risedevtool/config/src/main.rs b/src/risedevtool/config/src/main.rs
index 7d36cc8d4646f..27a676c2bfb3e 100644
--- a/src/risedevtool/config/src/main.rs
+++ b/src/risedevtool/config/src/main.rs
@@ -62,7 +62,6 @@ pub enum Components {
     Hdfs,
     PrometheusAndGrafana,
     Etcd,
-    Kafka,
     Pubsub,
     Redis,
     Tracing,
@@ -74,6 +73,12 @@ pub enum Components {
     DynamicLinking,
     HummockTrace,
     Coredump,
+    NoBacktrace,
+    ExternalUdf,
+    WasmUdf,
+    JsUdf,
+    DenoUdf,
+    PythonUdf,
 }
 
 impl Components {
@@ -83,7 +88,6 @@ impl Components {
             Self::Hdfs => "[Component] Hummock: Hdfs Backend",
             Self::PrometheusAndGrafana => "[Component] Metrics: Prometheus + Grafana",
             Self::Etcd => "[Component] Etcd",
-            Self::Kafka => "[Component] Kafka",
             Self::Pubsub => "[Component] Google Pubsub",
             Self::Redis => "[Component] Redis",
             Self::BuildConnectorNode => "[Build] Build RisingWave Connector (Java)",
@@ -95,6 +99,12 @@ impl Components {
             Self::DynamicLinking => "[Build] Enable dynamic linking",
             Self::HummockTrace => "[Build] Hummock Trace",
             Self::Coredump => "[Runtime] Enable coredump",
+            Self::NoBacktrace => "[Runtime] Disable backtrace",
+            Self::ExternalUdf => "[Build] Enable external UDF",
+            Self::WasmUdf => "[Build] Enable Wasm UDF",
+            Self::JsUdf => "[Build] Enable JS UDF",
+            Self::DenoUdf => "[Build] Enable Deno UDF",
+            Self::PythonUdf => "[Build] Enable Python UDF",
         }
         .into()
     }
@@ -118,11 +128,6 @@ Required if you want to view metrics."
 Required if you want to persistent meta-node data.
                 "
             }
-            Self::Kafka => {
-                "
-Required if you want to create source from Kafka.
-                "
-            }
             Self::Pubsub => {
                 "
 Required if you want to create source from Emulated Google Pub/sub.
@@ -192,6 +197,16 @@ the binaries will also be codesigned with `get-task-allow` enabled.
 As a result, RisingWave will dump the core on panics.
                 "
             }
+            Self::NoBacktrace => {
+                "
+With this option enabled, RiseDev will not set `RUST_BACKTRACE` when launching nodes.
+                "
+            }
+            Self::ExternalUdf => "Required if you want to support external UDF.",
+            Self::WasmUdf => "Required if you want to support WASM UDF.",
+            Self::JsUdf => "Required if you want to support JS UDF.",
+            Self::DenoUdf => "Required if you want to support Deno UDF.",
+            Self::PythonUdf => "Required if you want to support Python UDF.",
         }
         .into()
     }
@@ -202,7 +217,6 @@ As a result, RisingWave will dump the core on panics.
             "ENABLE_HDFS" => Some(Self::Hdfs),
             "ENABLE_PROMETHEUS_GRAFANA" => Some(Self::PrometheusAndGrafana),
             "ENABLE_ETCD" => Some(Self::Etcd),
-            "ENABLE_KAFKA" => Some(Self::Kafka),
             "ENABLE_PUBSUB" => Some(Self::Pubsub),
             "ENABLE_BUILD_RUST" => Some(Self::RustComponents),
             "ENABLE_BUILD_DASHBOARD" => Some(Self::Dashboard),
@@ -213,6 +227,13 @@ As a result, RisingWave will dump the core on panics.
             "ENABLE_REDIS" => Some(Self::Redis),
             "ENABLE_BUILD_RW_CONNECTOR" => Some(Self::BuildConnectorNode),
             "ENABLE_HUMMOCK_TRACE" => Some(Self::HummockTrace),
+            "ENABLE_COREDUMP" => Some(Self::Coredump),
+            "DISABLE_BACKTRACE" => Some(Self::NoBacktrace),
+            "ENABLE_EXTERNAL_UDF" => Some(Self::ExternalUdf),
+            "ENABLE_WASM_UDF" => Some(Self::WasmUdf),
+            "ENABLE_JS_UDF" => Some(Self::JsUdf),
+            "ENABLE_DENO_UDF" => Some(Self::DenoUdf),
+            "ENABLE_PYTHON_UDF" => Some(Self::PythonUdf),
             _ => None,
         }
     }
@@ -223,7 +244,6 @@ As a result, RisingWave will dump the core on panics.
             Self::Hdfs => "ENABLE_HDFS",
             Self::PrometheusAndGrafana => "ENABLE_PROMETHEUS_GRAFANA",
             Self::Etcd => "ENABLE_ETCD",
-            Self::Kafka => "ENABLE_KAFKA",
             Self::Pubsub => "ENABLE_PUBSUB",
             Self::Redis => "ENABLE_REDIS",
             Self::RustComponents => "ENABLE_BUILD_RUST",
@@ -235,6 +255,12 @@ As a result, RisingWave will dump the core on panics.
             Self::DynamicLinking => "ENABLE_DYNAMIC_LINKING",
             Self::HummockTrace => "ENABLE_HUMMOCK_TRACE",
             Self::Coredump => "ENABLE_COREDUMP",
+            Self::NoBacktrace => "DISABLE_BACKTRACE",
+            Self::ExternalUdf => "ENABLE_EXTERNAL_UDF",
+            Self::WasmUdf => "ENABLE_WASM_UDF",
+            Self::JsUdf => "ENABLE_JS_UDF",
+            Self::DenoUdf => "ENABLE_DENO_UDF",
+            Self::PythonUdf => "ENABLE_PYTHON_UDF",
         }
         .into()
     }
diff --git a/src/risedevtool/gcloud-pubsub.toml b/src/risedevtool/gcloud-pubsub.toml
index 313561b287cba..85b5eae3e170a 100644
--- a/src/risedevtool/gcloud-pubsub.toml
+++ b/src/risedevtool/gcloud-pubsub.toml
@@ -2,14 +2,16 @@ extend = "common.toml"
 
 [env]
 GCLOUD_DOWNLOAD_PATH = "${PREFIX_TMP}/gcloud.tgz"
-GCLOUD_DOWNLOAD_TGZ = { value = "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-406.0.0-linux-x86_64.tar.gz", condition = { env_not_set = [ "GCLOUD_DOWNLOAD_TGZ" ] } }
+GCLOUD_DOWNLOAD_TGZ = { value = "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-475.0.0-linux-x86_64.tar.gz", condition = { env_not_set = [
+    "GCLOUD_DOWNLOAD_TGZ",
+] } }
 GCLOUD_SDK_DIR = "google-cloud-sdk"
 
 [tasks.download-pubsub]
 private = true
 category = "RiseDev - Components"
 dependencies = ["prepare"]
-condition = { env_set = [ "ENABLE_PUBSUB" ] }
+condition = { env_set = ["ENABLE_PUBSUB"] }
 description = "Download and enable Google Pubsub Emulator"
 script = '''
 #!/usr/bin/env bash
diff --git a/src/risedevtool/grafana.toml b/src/risedevtool/grafana.toml
index e64524aa4ba25..10d30c53c5fa5 100644
--- a/src/risedevtool/grafana.toml
+++ b/src/risedevtool/grafana.toml
@@ -1,9 +1,9 @@
 extend = "common.toml"
 
 [env]
-GRAFANA_SYSTEM = "${SYSTEM_AMD64}"
+GRAFANA_SYSTEM = "${SYSTEM}"
 GRAFANA_DOWNLOAD_PATH = "${PREFIX_TMP}/grafana.tar.gz"
-GRAFANA_VERSION = "10.0.0"
+GRAFANA_VERSION = "10.4.3"
 GRAFANA_RELEASE = "grafana-${GRAFANA_VERSION}"
 GRAFANA_DOWNLOAD_TAR_GZ = "https://dl.grafana.com/oss/release/${GRAFANA_RELEASE}.${GRAFANA_SYSTEM}.tar.gz"
 
@@ -25,7 +25,7 @@ echo "Grafana not found, download ${GRAFANA_RELEASE}"
 curl -fL -o "${GRAFANA_DOWNLOAD_PATH}" "${GRAFANA_DOWNLOAD_TAR_GZ}"
 tar -xf "${GRAFANA_DOWNLOAD_PATH}" -C "${PREFIX_TMP}"
 rm -rf "${PREFIX_BIN}/grafana"
-mv "${PREFIX_TMP}/${GRAFANA_RELEASE}" "${PREFIX_BIN}/grafana"
+mv "${PREFIX_TMP}/grafana-v${GRAFANA_VERSION}" "${PREFIX_BIN}/grafana"
 rm "${GRAFANA_DOWNLOAD_PATH}"
 touch "${PREFIX_BIN}/grafana/RISEDEV-VERSION-${GRAFANA_VERSION}"
 '''
diff --git a/src/risedevtool/kafka.toml b/src/risedevtool/kafka.toml
deleted file mode 100644
index 4353e30aef919..0000000000000
--- a/src/risedevtool/kafka.toml
+++ /dev/null
@@ -1,32 +0,0 @@
-extend = "common.toml"
-
-[env]
-KAFKA_DOWNLOAD_PATH = "${PREFIX_TMP}/kafka.tgz"
-
-[tasks.download-kafka]
-private = true
-category = "RiseDev - Components"
-dependencies = ["prepare"]
-condition = { env_set = [ "ENABLE_KAFKA" ] }
-description = "Download and extract Kafka"
-script = '''
-#!/usr/bin/env bash
-set -e
-if [ -d "${PREFIX_BIN}/kafka" ]; then
-    exit 0
-fi
-
-get_latest_kafka_version() {
-    local versions=$(curl -s https://downloads.apache.org/kafka/ | grep -Eo 'href="[0-9]+\.[0-9]+\.[0-9]+/"' | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+")
-    # Sort the version numbers and get the latest one
-    local latest_version=$(echo "$versions" | sort -V | tail -n1)
-    echo $latest_version
-}
-
-echo "Kafka not found, downloading..."
-latest_version=$(get_latest_kafka_version)
-curl -fL -o "${KAFKA_DOWNLOAD_PATH}" "https://downloads.apache.org/kafka/${latest_version}/kafka_2.13-${latest_version}.tgz"
-tar -xf "${KAFKA_DOWNLOAD_PATH}" -C "${PREFIX_TMP}"
-mv "${PREFIX_TMP}/kafka_2.13-${latest_version}" "${PREFIX_BIN}/kafka"
-rm ${KAFKA_DOWNLOAD_PATH}
-'''
diff --git a/src/risedevtool/src/bin/risedev-compose.rs b/src/risedevtool/src/bin/risedev-compose.rs
index 0f3cb5b020fca..89bb0592d0d85 100644
--- a/src/risedevtool/src/bin/risedev-compose.rs
+++ b/src/risedevtool/src/bin/risedev-compose.rs
@@ -19,7 +19,7 @@ use std::path::Path;
 use anyhow::{anyhow, Result};
 use clap::Parser;
 use console::style;
-use fs_err::{self, File};
+use fs_err::File;
 use itertools::Itertools;
 use risedev::{
     compose_deploy, generate_risedev_env, Compose, ComposeConfig, ComposeDeployConfig, ComposeFile,
@@ -201,9 +201,6 @@ fn main() -> Result<()> {
             ServiceConfig::Pubsub(_) => {
                 return Err(anyhow!("not supported, please use redpanda instead"))
             }
-            ServiceConfig::ZooKeeper(_) => {
-                return Err(anyhow!("not supported, please use redpanda instead"))
-            }
             ServiceConfig::Opendal(_) => continue,
             ServiceConfig::AwsS3(_) => continue,
             ServiceConfig::RedPanda(c) => {
@@ -222,7 +219,10 @@ fn main() -> Result<()> {
                 volumes.insert(c.id.clone(), ComposeVolume::default());
                 (c.address.clone(), c.compose(&compose_config)?)
             }
-            ServiceConfig::Redis(_) => return Err(anyhow!("not supported")),
+            ServiceConfig::Redis(_)
+            | ServiceConfig::MySql(_)
+            | ServiceConfig::Postgres(_)
+            | ServiceConfig::SchemaRegistry(_) => return Err(anyhow!("not supported")),
         };
         compose.container_name = service.id().to_string();
         if opts.deploy {
@@ -245,7 +245,6 @@ fn main() -> Result<()> {
                 }
             });
             let compose_file = ComposeFile {
-                version: "3".into(),
                 services: services.clone(),
                 volumes: node_volumes,
                 name: format!("risingwave-{}", opts.profile),
@@ -303,7 +302,6 @@ fn main() -> Result<()> {
             }
         }
         let compose_file = ComposeFile {
-            version: "3".into(),
             services,
             volumes,
             name: format!("risingwave-{}", opts.profile),
diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs
index d5a14cbf8c1bd..8dbe155bcd086 100644
--- a/src/risedevtool/src/bin/risedev-dev.rs
+++ b/src/risedevtool/src/bin/risedev-dev.rs
@@ -24,11 +24,11 @@ use fs_err::OpenOptions;
 use indicatif::ProgressBar;
 use risedev::util::{complete_spin, fail_spin};
 use risedev::{
-    generate_risedev_env, preflight_check, AwsS3Config, CompactorService, ComputeNodeService,
-    ConfigExpander, ConfigureTmuxTask, EnsureStopService, ExecuteContext, FrontendService,
-    GrafanaService, KafkaService, MetaNodeService, MinioService, OpendalConfig, PrometheusService,
-    PubsubService, RedisService, ServiceConfig, SqliteConfig, Task, TempoService, ZooKeeperService,
-    RISEDEV_NAME,
+    generate_risedev_env, preflight_check, CompactorService, ComputeNodeService, ConfigExpander,
+    ConfigureTmuxTask, DummyService, EnsureStopService, ExecuteContext, FrontendService,
+    GrafanaService, KafkaService, MetaNodeService, MinioService, MySqlService, PostgresService,
+    PrometheusService, PubsubService, RedisService, SchemaRegistryService, ServiceConfig,
+    SqliteConfig, Task, TempoService, RISEDEV_NAME,
 };
 use tempfile::tempdir;
 use thiserror_ext::AsReport;
@@ -135,8 +135,7 @@ fn task_main(
                 // let mut task = risedev::EtcdReadyCheckTask::new(c.clone())?;
                 // TODO(chi): etcd will set its health check to success only after all nodes are
                 // connected and there's a leader, therefore we cannot do health check for now.
-                let mut task =
-                    risedev::ConfigureGrpcNodeTask::new(c.address.clone(), c.port, false)?;
+                let mut task = risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, false)?;
                 task.execute(&mut ctx)?;
             }
             ServiceConfig::Sqlite(c) => {
@@ -172,8 +171,7 @@ fn task_main(
                     ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
                 let mut service = PrometheusService::new(c.clone())?;
                 service.execute(&mut ctx)?;
-                let mut task =
-                    risedev::ConfigureGrpcNodeTask::new(c.address.clone(), c.port, false)?;
+                let mut task = risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, false)?;
                 task.execute(&mut ctx)?;
                 ctx.pb
                     .set_message(format!("api http://{}:{}/", c.address, c.port));
@@ -185,7 +183,7 @@ fn task_main(
                 service.execute(&mut ctx)?;
 
                 let mut task =
-                    risedev::ConfigureGrpcNodeTask::new(c.address.clone(), c.port, c.user_managed)?;
+                    risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, c.user_managed)?;
                 task.execute(&mut ctx)?;
                 ctx.pb
                     .set_message(format!("api grpc://{}:{}/", c.address, c.port));
@@ -196,7 +194,7 @@ fn task_main(
                 let mut service = MetaNodeService::new(c.clone())?;
                 service.execute(&mut ctx)?;
                 let mut task =
-                    risedev::ConfigureGrpcNodeTask::new(c.address.clone(), c.port, c.user_managed)?;
+                    risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, c.user_managed)?;
                 task.execute(&mut ctx)?;
                 ctx.pb.set_message(format!(
                     "api grpc://{}:{}/, dashboard http://{}:{}/",
@@ -209,7 +207,7 @@ fn task_main(
                 let mut service = FrontendService::new(c.clone())?;
                 service.execute(&mut ctx)?;
                 let mut task =
-                    risedev::ConfigureGrpcNodeTask::new(c.address.clone(), c.port, c.user_managed)?;
+                    risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, c.user_managed)?;
                 task.execute(&mut ctx)?;
                 ctx.pb
                     .set_message(format!("api postgres://{}:{}/", c.address, c.port));
@@ -231,7 +229,7 @@ fn task_main(
                 let mut service = CompactorService::new(c.clone())?;
                 service.execute(&mut ctx)?;
                 let mut task =
-                    risedev::ConfigureGrpcNodeTask::new(c.address.clone(), c.port, c.user_managed)?;
+                    risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, c.user_managed)?;
                 task.execute(&mut ctx)?;
                 ctx.pb
                     .set_message(format!("compactor {}:{}", c.address, c.port));
@@ -241,8 +239,7 @@ fn task_main(
                     ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
                 let mut service = GrafanaService::new(c.clone())?;
                 service.execute(&mut ctx)?;
-                let mut task =
-                    risedev::ConfigureGrpcNodeTask::new(c.address.clone(), c.port, false)?;
+                let mut task = risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, false)?;
                 task.execute(&mut ctx)?;
                 ctx.pb
                     .set_message(format!("dashboard http://{}:{}/", c.address, c.port));
@@ -253,7 +250,7 @@ fn task_main(
                 let mut service = TempoService::new(c.clone())?;
                 service.execute(&mut ctx)?;
                 let mut task =
-                    risedev::ConfigureGrpcNodeTask::new(c.listen_address.clone(), c.port, false)?;
+                    risedev::TcpReadyCheckTask::new(c.listen_address.clone(), c.port, false)?;
                 task.execute(&mut ctx)?;
                 ctx.pb
                     .set_message(format!("api http://{}:{}/", c.listen_address, c.port));
@@ -261,70 +258,39 @@ fn task_main(
             ServiceConfig::AwsS3(c) => {
                 let mut ctx =
                     ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
-
-                struct AwsService(AwsS3Config);
-                impl Task for AwsService {
-                    fn execute(
-                        &mut self,
-                        _ctx: &mut ExecuteContext<impl std::io::Write>,
-                    ) -> anyhow::Result<()> {
-                        Ok(())
-                    }
-
-                    fn id(&self) -> String {
-                        self.0.id.clone()
-                    }
-                }
-
-                ctx.service(&AwsService(c.clone()));
-                ctx.complete_spin();
+                DummyService::new(&c.id).execute(&mut ctx)?;
                 ctx.pb
                     .set_message(format!("using AWS s3 bucket {}", c.bucket));
             }
             ServiceConfig::Opendal(c) => {
                 let mut ctx =
                     ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
-
-                struct OpendalService(OpendalConfig);
-                impl Task for OpendalService {
-                    fn execute(
-                        &mut self,
-                        _ctx: &mut ExecuteContext<impl std::io::Write>,
-                    ) -> anyhow::Result<()> {
-                        Ok(())
-                    }
-
-                    fn id(&self) -> String {
-                        self.0.id.clone()
-                    }
-                }
-
-                ctx.service(&OpendalService(c.clone()));
-                ctx.complete_spin();
+                DummyService::new(&c.id).execute(&mut ctx)?;
                 ctx.pb
                     .set_message(format!("using Opendal, namenode =  {}", c.namenode));
             }
-            ServiceConfig::ZooKeeper(c) => {
+            ServiceConfig::Kafka(c) => {
                 let mut ctx =
                     ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
-                let mut service = ZooKeeperService::new(c.clone())?;
+                let mut service = KafkaService::new(c.clone());
                 service.execute(&mut ctx)?;
-                let mut task =
-                    risedev::ConfigureGrpcNodeTask::new(c.address.clone(), c.port, false)?;
+                let mut task = risedev::KafkaReadyCheckTask::new(c.clone())?;
                 task.execute(&mut ctx)?;
                 ctx.pb
-                    .set_message(format!("zookeeper {}:{}", c.address, c.port));
+                    .set_message(format!("kafka {}:{}", c.address, c.port));
             }
-            ServiceConfig::Kafka(c) => {
+            ServiceConfig::SchemaRegistry(c) => {
                 let mut ctx =
                     ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
-                let mut service = KafkaService::new(c.clone())?;
+                let mut service = SchemaRegistryService::new(c.clone());
                 service.execute(&mut ctx)?;
-                let mut task = risedev::KafkaReadyCheckTask::new(c.clone())?;
+                let mut task =
+                    risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, c.user_managed)?;
                 task.execute(&mut ctx)?;
                 ctx.pb
-                    .set_message(format!("kafka {}:{}", c.address, c.port));
+                    .set_message(format!("schema registry http://{}:{}", c.address, c.port));
             }
+
             ServiceConfig::Pubsub(c) => {
                 let mut ctx =
                     ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
@@ -348,6 +314,36 @@ fn task_main(
                 ctx.pb
                     .set_message(format!("redis {}:{}", c.address, c.port));
             }
+            ServiceConfig::MySql(c) => {
+                let mut ctx =
+                    ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
+                MySqlService::new(c.clone()).execute(&mut ctx)?;
+                if c.user_managed {
+                    let mut task =
+                        risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, c.user_managed)?;
+                    task.execute(&mut ctx)?;
+                } else {
+                    let mut task = risedev::LogReadyCheckTask::new("Ready for start up.")?;
+                    task.execute(&mut ctx)?;
+                }
+                ctx.pb
+                    .set_message(format!("mysql {}:{}", c.address, c.port));
+            }
+            ServiceConfig::Postgres(c) => {
+                let mut ctx =
+                    ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
+                PostgresService::new(c.clone()).execute(&mut ctx)?;
+                if c.user_managed {
+                    let mut task =
+                        risedev::TcpReadyCheckTask::new(c.address.clone(), c.port, c.user_managed)?;
+                    task.execute(&mut ctx)?;
+                } else {
+                    let mut task = risedev::LogReadyCheckTask::new("ready to accept connections")?;
+                    task.execute(&mut ctx)?;
+                }
+                ctx.pb
+                    .set_message(format!("postgres {}:{}", c.address, c.port));
+            }
         }
 
         let service_id = service.id().to_string();
diff --git a/src/risedevtool/src/compose.rs b/src/risedevtool/src/compose.rs
index 779ca23557622..a490560157527 100644
--- a/src/risedevtool/src/compose.rs
+++ b/src/risedevtool/src/compose.rs
@@ -56,7 +56,6 @@ pub struct HealthCheck {
 
 #[derive(Debug, Clone, Serialize)]
 pub struct ComposeFile {
-    pub version: String,
     pub services: BTreeMap<String, ComposeService>,
     pub volumes: BTreeMap<String, ComposeVolume>,
     pub name: String,
diff --git a/src/risedevtool/src/config.rs b/src/risedevtool/src/config.rs
index d5fe2d8c220a5..bf768f8e68cd1 100644
--- a/src/risedevtool/src/config.rs
+++ b/src/risedevtool/src/config.rs
@@ -172,8 +172,12 @@ impl ConfigExpander {
                     "kafka" => ServiceConfig::Kafka(serde_yaml::from_str(&out_str)?),
                     "pubsub" => ServiceConfig::Pubsub(serde_yaml::from_str(&out_str)?),
                     "redis" => ServiceConfig::Redis(serde_yaml::from_str(&out_str)?),
-                    "zookeeper" => ServiceConfig::ZooKeeper(serde_yaml::from_str(&out_str)?),
                     "redpanda" => ServiceConfig::RedPanda(serde_yaml::from_str(&out_str)?),
+                    "mysql" => ServiceConfig::MySql(serde_yaml::from_str(&out_str)?),
+                    "postgres" => ServiceConfig::Postgres(serde_yaml::from_str(&out_str)?),
+                    "schema-registry" => {
+                        ServiceConfig::SchemaRegistry(serde_yaml::from_str(&out_str)?)
+                    }
                     other => return Err(anyhow!("unsupported use type: {}", other)),
                 };
                 Ok(result)
diff --git a/src/risedevtool/src/config_gen.rs b/src/risedevtool/src/config_gen.rs
index 32678c8a94549..5b931a5d238cc 100644
--- a/src/risedevtool/src/config_gen.rs
+++ b/src/risedevtool/src/config_gen.rs
@@ -16,9 +16,5 @@ mod prometheus_gen;
 pub use prometheus_gen::*;
 mod grafana_gen;
 pub use grafana_gen::*;
-mod zookeeper_gen;
-pub use zookeeper_gen::*;
-mod kafka_gen;
-pub use kafka_gen::*;
 mod tempo_gen;
 pub use tempo_gen::*;
diff --git a/src/risedevtool/src/config_gen/kafka_gen.rs b/src/risedevtool/src/config_gen/kafka_gen.rs
deleted file mode 100644
index 5f54efd5a4e44..0000000000000
--- a/src/risedevtool/src/config_gen/kafka_gen.rs
+++ /dev/null
@@ -1,176 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use itertools::Itertools;
-
-use crate::KafkaConfig;
-
-pub struct KafkaGen;
-
-impl KafkaGen {
-    pub fn gen_server_properties(&self, config: &KafkaConfig, kafka_log_dirs: &str) -> String {
-        let kafka_listen_host = &config.listen_address;
-        let kafka_advertise_host = &config.address;
-        let kafka_port = &config.port;
-        let zookeeper_hosts = config
-            .provide_zookeeper
-            .as_ref()
-            .unwrap()
-            .iter()
-            .map(|node| format!("{}:{}", node.address, node.port))
-            .join(",");
-        let kafka_broker_id = config.broker_id;
-
-        format!(
-            r#"# --- THIS FILE IS AUTO GENERATED BY RISEDEV ---
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# see kafka.server.KafkaConfig for additional details and defaults
-
-############################# Server Basics #############################
-
-# The id of the broker. This must be set to a unique integer for each broker.
-broker.id={kafka_broker_id}
-
-############################# Socket Server Settings #############################
-
-# The address the socket server listens on. It will get the value returned from
-# java.net.InetAddress.getCanonicalHostName() if not configured.
-#   FORMAT:
-#     listeners = listener_name://host_name:port
-#   EXAMPLE:
-#     listeners = PLAINTEXT://your.host.name:9092
-listeners=PLAINTEXT://{kafka_listen_host}:{kafka_port}
-
-# Hostname and port the broker will advertise to producers and consumers. If not set,
-# it uses the value for "listeners" if configured.  Otherwise, it will use the value
-# returned from java.net.InetAddress.getCanonicalHostName().
-advertised.listeners=PLAINTEXT://{kafka_advertise_host}:{kafka_port}
-
-# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
-#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
-
-# The number of threads that the server uses for receiving requests from the network and sending responses to the network
-num.network.threads=3
-
-# The number of threads that the server uses for processing requests, which may include disk I/O
-num.io.threads=8
-
-# The send buffer (SO_SNDBUF) used by the socket server
-socket.send.buffer.bytes=102400
-
-# The receive buffer (SO_RCVBUF) used by the socket server
-socket.receive.buffer.bytes=102400
-
-# The maximum size of a request that the socket server will accept (protection against OOM)
-socket.request.max.bytes=104857600
-
-############################# Log Basics #############################
-
-# A comma separated list of directories under which to store log files
-log.dirs={kafka_log_dirs}
-
-# The default number of log partitions per topic. More partitions allow greater
-# parallelism for consumption, but this will also result in more files across
-# the brokers.
-num.partitions=1
-
-# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
-# This value is recommended to be increased for installations with data dirs located in RAID array.
-num.recovery.threads.per.data.dir=1
-
-############################# Internal Topic Settings  #############################
-# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
-# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
-offsets.topic.replication.factor=1
-transaction.state.log.replication.factor=1
-transaction.state.log.min.isr=1
-
-############################# Log Flush Policy #############################
-
-# Messages are immediately written to the filesystem but by default we only fsync() to sync
-# the OS cache lazily. The following configurations control the flush of data to disk.
-# There are a few important trade-offs here:
-#    1. Durability: Unflushed data may be lost if you are not using replication.
-#    2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
-#    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
-# The settings below allow one to configure the flush policy to flush data after a period of time or
-# every N messages (or both). This can be done globally and overridden on a per-topic basis.
-
-# The number of messages to accept before forcing a flush of data to disk
-#log.flush.interval.messages=10000
-
-# The maximum amount of time a message can sit in a log before we force a flush
-#log.flush.interval.ms=1000
-
-############################# Log Retention Policy #############################
-
-# The following configurations control the disposal of log segments. The policy can
-# be set to delete segments after a period of time, or after a given size has accumulated.
-# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
-# from the end of the log.
-
-# The minimum age of a log file to be eligible for deletion due to age
-log.retention.hours=168
-
-# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
-# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
-#log.retention.bytes=1073741824
-
-# The maximum size of a log segment file. When this size is reached a new log segment will be created.
-log.segment.bytes=1073741824
-
-# The interval at which log segments are checked to see if they can be deleted according
-# to the retention policies
-log.retention.check.interval.ms=300000
-
-############################# Zookeeper #############################
-
-# Zookeeper connection string (see zookeeper docs for details).
-# This is a comma separated host:port pairs, each corresponding to a zk
-# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
-# You can also append an optional chroot string to the urls to specify the
-# root directory for all kafka znodes.
-zookeeper.connect={zookeeper_hosts}
-
-# Timeout in ms for connecting to zookeeper
-zookeeper.connection.timeout.ms=18000
-
-
-############################# Group Coordinator Settings #############################
-
-# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.
-# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.
-# The default value for this is 3 seconds.
-# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
-# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
-group.initial.rebalance.delay.ms=0
-"#
-        )
-    }
-}
diff --git a/src/risedevtool/src/config_gen/zookeeper_gen.rs b/src/risedevtool/src/config_gen/zookeeper_gen.rs
deleted file mode 100644
index ed340d58db1c9..0000000000000
--- a/src/risedevtool/src/config_gen/zookeeper_gen.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::ZooKeeperConfig;
-
-pub struct ZooKeeperGen;
-
-impl ZooKeeperGen {
-    pub fn gen_server_properties(
-        &self,
-        config: &ZooKeeperConfig,
-        zookeeper_data_dir: &str,
-    ) -> String {
-        let zookeeper_listen_host = &config.listen_address;
-        let zookeeper_port = &config.port;
-
-        format!(
-            r#"# --- THIS FILE IS AUTO GENERATED BY RISEDEV ---
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# the directory where the snapshot is stored.
-dataDir={zookeeper_data_dir}
-# the port at which the clients will connect
-clientPort={zookeeper_port}
-clientPortAddress={zookeeper_listen_host}
-# disable the per-ip limit on the number of connections since this is a non-production config
-maxClientCnxns=0
-# Disable the adminserver by default to avoid port conflicts.
-# Set the port to something non-conflicting if choosing to enable this
-admin.enableServer=false
-# admin.serverPort=8080
-"#
-        )
-    }
-}
diff --git a/src/risedevtool/src/risedev_env.rs b/src/risedevtool/src/risedev_env.rs
index 1efdf1470998d..943f37abf655d 100644
--- a/src/risedevtool/src/risedev_env.rs
+++ b/src/risedevtool/src/risedev_env.rs
@@ -17,7 +17,7 @@
 use std::fmt::Write;
 use std::process::Command;
 
-use crate::{add_hummock_backend, HummockInMemoryStrategy, ServiceConfig};
+use crate::{add_hummock_backend, Application, HummockInMemoryStrategy, ServiceConfig};
 
 /// Generate environment variables (put in file `.risingwave/config/risedev-env`)
 /// from the given service configurations to be used by future
@@ -77,6 +77,49 @@ pub fn generate_risedev_env(services: &Vec<ServiceConfig>) -> String {
                 writeln!(env, r#"RISEDEV_KAFKA_WITH_OPTIONS_COMMON="connector='kafka',properties.bootstrap.server='{brokers}'""#).unwrap();
                 writeln!(env, r#"RPK_BROKERS="{brokers}""#).unwrap();
             }
+            ServiceConfig::SchemaRegistry(c) => {
+                let address = &c.address;
+                let port = &c.port;
+                writeln!(
+                    env,
+                    r#"RISEDEV_SCHEMA_REGISTRY_URL="http://{address}:{port}""#,
+                )
+                .unwrap();
+            }
+            ServiceConfig::MySql(c) if c.application != Application::Metastore => {
+                let host = &c.address;
+                let port = &c.port;
+                let user = &c.user;
+                let password = &c.password;
+                // These envs are used by `mysql` cli.
+                writeln!(env, r#"MYSQL_HOST="{host}""#,).unwrap();
+                writeln!(env, r#"MYSQL_TCP_PORT="{port}""#,).unwrap();
+                // Note: There's no env var for the username read by `mysql` cli. Here we set
+                // `RISEDEV_MYSQL_USER`, which will be read by `e2e_test/commands/mysql` when
+                // running `risedev slt`, as a wrapper of `mysql` cli.
+                writeln!(env, r#"RISEDEV_MYSQL_USER="{user}""#,).unwrap();
+                writeln!(env, r#"MYSQL_PWD="{password}""#,).unwrap();
+                // Note: user and password are not included in the common WITH options.
+                // It's expected to create another dedicated user for the source.
+                writeln!(env, r#"RISEDEV_MYSQL_WITH_OPTIONS_COMMON="connector='mysql-cdc',hostname='{host}',port='{port}'""#,).unwrap();
+            }
+            ServiceConfig::Pubsub(c) => {
+                let address = &c.address;
+                let port = &c.port;
+                writeln!(env, r#"PUBSUB_EMULATOR_HOST="{address}:{port}""#,).unwrap();
+                writeln!(env, r#"RISEDEV_PUBSUB_WITH_OPTIONS_COMMON="connector='google_pubsub',pubsub.emulator_host='{address}:{port}'""#,).unwrap();
+            }
+            ServiceConfig::Postgres(c) if c.application != Application::Metastore => {
+                let host = &c.address;
+                let port = &c.port;
+                let user = &c.user;
+                let password = &c.password;
+                // These envs are used by `postgres` cli.
+                writeln!(env, r#"PGHOST="{host}""#,).unwrap();
+                writeln!(env, r#"PGPORT="{port}""#,).unwrap();
+                writeln!(env, r#"PGUSER="{user}""#,).unwrap();
+                writeln!(env, r#"PGPASSWORD="{password}""#,).unwrap();
+            }
             _ => {}
         }
     }
diff --git a/src/risedevtool/src/service_config.rs b/src/risedevtool/src/service_config.rs
index e5f149b8d10cd..fb95dbb520a0d 100644
--- a/src/risedevtool/src/service_config.rs
+++ b/src/risedevtool/src/service_config.rs
@@ -43,6 +43,17 @@ pub struct ComputeNodeConfig {
     pub role: String,
 }
 
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+#[serde(deny_unknown_fields)]
+pub enum MetaBackend {
+    Memory,
+    Etcd,
+    Sqlite,
+    Postgres,
+    Mysql,
+}
+
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 #[serde(deny_unknown_fields)]
@@ -60,8 +71,11 @@ pub struct MetaNodeConfig {
 
     pub user_managed: bool,
 
+    pub meta_backend: MetaBackend,
     pub provide_etcd_backend: Option<Vec<EtcdConfig>>,
     pub provide_sqlite_backend: Option<Vec<SqliteConfig>>,
+    pub provide_postgres_backend: Option<Vec<PostgresConfig>>,
+    pub provide_mysql_backend: Option<Vec<MySqlConfig>>,
     pub provide_prometheus: Option<Vec<PrometheusConfig>>,
 
     pub provide_compute_node: Option<Vec<ComputeNodeConfig>>,
@@ -269,43 +283,55 @@ pub struct KafkaConfig {
     phantom_use: Option<String>,
     pub id: String,
 
+    /// Advertise address
     pub address: String,
     #[serde(with = "string")]
     pub port: u16,
-    pub listen_address: String,
+    /// Port for other services in docker. They need to connect to `host.docker.internal`, while the host
+    /// need to connect to `localhost`.
+    pub docker_port: u16,
+
+    #[serde(with = "string")]
+    pub controller_port: u16,
 
-    pub provide_zookeeper: Option<Vec<ZooKeeperConfig>>,
+    pub image: String,
     pub persist_data: bool,
-    pub broker_id: u32,
+    pub node_id: u32,
 
     pub user_managed: bool,
 }
+
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 #[serde(deny_unknown_fields)]
-pub struct PubsubConfig {
+pub struct SchemaRegistryConfig {
     #[serde(rename = "use")]
     phantom_use: Option<String>,
+
     pub id: String,
+
+    pub address: String,
     #[serde(with = "string")]
     pub port: u16,
-    pub address: String,
 
-    pub persist_data: bool,
+    pub provide_kafka: Option<Vec<KafkaConfig>>,
+
+    pub image: String,
+    /// Redpanda supports schema registry natively. You can configure a `user_managed` schema registry
+    /// to use with redpanda.
+    pub user_managed: bool,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 #[serde(deny_unknown_fields)]
-pub struct ZooKeeperConfig {
+pub struct PubsubConfig {
     #[serde(rename = "use")]
     phantom_use: Option<String>,
     pub id: String,
-
-    pub address: String,
     #[serde(with = "string")]
     pub port: u16,
-    pub listen_address: String,
+    pub address: String,
 
     pub persist_data: bool,
 }
@@ -336,6 +362,56 @@ pub struct RedisConfig {
     pub address: String,
 }
 
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+#[serde(deny_unknown_fields)]
+pub enum Application {
+    Metastore,
+    Connector,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+#[serde(deny_unknown_fields)]
+pub struct MySqlConfig {
+    #[serde(rename = "use")]
+    phantom_use: Option<String>,
+    pub id: String,
+
+    pub port: u16,
+    pub address: String,
+
+    pub user: String,
+    pub password: String,
+    pub database: String,
+
+    pub application: Application,
+    pub image: String,
+    pub user_managed: bool,
+    pub persist_data: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+#[serde(deny_unknown_fields)]
+pub struct PostgresConfig {
+    #[serde(rename = "use")]
+    phantom_use: Option<String>,
+    pub id: String,
+
+    pub port: u16,
+    pub address: String,
+
+    pub user: String,
+    pub password: String,
+    pub database: String,
+
+    pub application: Application,
+    pub image: String,
+    pub user_managed: bool,
+    pub persist_data: bool,
+}
+
 /// All service configuration
 #[derive(Clone, Debug, PartialEq)]
 pub enum ServiceConfig {
@@ -352,10 +428,12 @@ pub enum ServiceConfig {
     Opendal(OpendalConfig),
     AwsS3(AwsS3Config),
     Kafka(KafkaConfig),
+    SchemaRegistry(SchemaRegistryConfig),
     Pubsub(PubsubConfig),
     Redis(RedisConfig),
-    ZooKeeper(ZooKeeperConfig),
     RedPanda(RedPandaConfig),
+    MySql(MySqlConfig),
+    Postgres(PostgresConfig),
 }
 
 impl ServiceConfig {
@@ -372,15 +450,18 @@ impl ServiceConfig {
             Self::Grafana(c) => &c.id,
             Self::Tempo(c) => &c.id,
             Self::AwsS3(c) => &c.id,
-            Self::ZooKeeper(c) => &c.id,
             Self::Kafka(c) => &c.id,
             Self::Pubsub(c) => &c.id,
             Self::Redis(c) => &c.id,
             Self::RedPanda(c) => &c.id,
             Self::Opendal(c) => &c.id,
+            Self::MySql(c) => &c.id,
+            Self::Postgres(c) => &c.id,
+            Self::SchemaRegistry(c) => &c.id,
         }
     }
 
+    /// Used to check whether the port is occupied before running the service.
     pub fn port(&self) -> Option<u16> {
         match self {
             Self::ComputeNode(c) => Some(c.port),
@@ -394,12 +475,14 @@ impl ServiceConfig {
             Self::Grafana(c) => Some(c.port),
             Self::Tempo(c) => Some(c.port),
             Self::AwsS3(_) => None,
-            Self::ZooKeeper(c) => Some(c.port),
             Self::Kafka(c) => Some(c.port),
             Self::Pubsub(c) => Some(c.port),
             Self::Redis(c) => Some(c.port),
             Self::RedPanda(_c) => None,
             Self::Opendal(_) => None,
+            Self::MySql(c) => Some(c.port),
+            Self::Postgres(c) => Some(c.port),
+            Self::SchemaRegistry(c) => Some(c.port),
         }
     }
 
@@ -416,12 +499,14 @@ impl ServiceConfig {
             Self::Grafana(_c) => false,
             Self::Tempo(_c) => false,
             Self::AwsS3(_c) => false,
-            Self::ZooKeeper(_c) => false,
             Self::Kafka(c) => c.user_managed,
             Self::Pubsub(_c) => false,
             Self::Redis(_c) => false,
             Self::RedPanda(_c) => false,
             Self::Opendal(_c) => false,
+            Self::MySql(c) => c.user_managed,
+            Self::Postgres(c) => c.user_managed,
+            Self::SchemaRegistry(c) => c.user_managed,
         }
     }
 }
diff --git a/src/risedevtool/src/task.rs b/src/risedevtool/src/task.rs
index bec0df5ae812e..21b6f20eec5ee 100644
--- a/src/risedevtool/src/task.rs
+++ b/src/risedevtool/src/task.rs
@@ -15,6 +15,8 @@
 mod compactor_service;
 mod compute_node_service;
 mod configure_tmux_service;
+mod docker_service;
+mod dummy_service;
 mod ensure_stop_service;
 mod etcd_service;
 mod frontend_service;
@@ -22,21 +24,24 @@ mod grafana_service;
 mod kafka_service;
 mod meta_node_service;
 mod minio_service;
+mod mysql_service;
+mod postgres_service;
 mod prometheus_service;
 mod pubsub_service;
 mod redis_service;
-mod task_configure_grpc_node;
+mod schema_registry_service;
 mod task_configure_minio;
 mod task_etcd_ready_check;
 mod task_kafka_ready_check;
+mod task_log_ready_check;
 mod task_pubsub_emu_ready_check;
 mod task_redis_ready_check;
+mod task_tcp_ready_check;
 mod tempo_service;
 mod utils;
-mod zookeeper_service;
 
 use std::env;
-use std::net::TcpStream;
+use std::net::{TcpStream, ToSocketAddrs};
 use std::path::{Path, PathBuf};
 use std::process::{Command, Output};
 use std::sync::Arc;
@@ -51,6 +56,7 @@ pub use utils::*;
 pub use self::compactor_service::*;
 pub use self::compute_node_service::*;
 pub use self::configure_tmux_service::*;
+pub use self::dummy_service::DummyService;
 pub use self::ensure_stop_service::*;
 pub use self::etcd_service::*;
 pub use self::frontend_service::*;
@@ -58,17 +64,20 @@ pub use self::grafana_service::*;
 pub use self::kafka_service::*;
 pub use self::meta_node_service::*;
 pub use self::minio_service::*;
+pub use self::mysql_service::*;
+pub use self::postgres_service::*;
 pub use self::prometheus_service::*;
 pub use self::pubsub_service::*;
 pub use self::redis_service::*;
-pub use self::task_configure_grpc_node::*;
+pub use self::schema_registry_service::SchemaRegistryService;
 pub use self::task_configure_minio::*;
 pub use self::task_etcd_ready_check::*;
 pub use self::task_kafka_ready_check::*;
+pub use self::task_log_ready_check::*;
 pub use self::task_pubsub_emu_ready_check::*;
 pub use self::task_redis_ready_check::*;
+pub use self::task_tcp_ready_check::*;
 pub use self::tempo_service::*;
-pub use self::zookeeper_service::*;
 use crate::util::{complete_spin, get_program_args, get_program_name};
 use crate::wait::{wait, wait_tcp_available};
 
@@ -104,6 +113,9 @@ where
 
     /// The status file corresponding to the current context.
     pub status_file: Option<PathBuf>,
+
+    /// The log file corresponding to the current context. (e.g. frontend-4566.log)
+    pub log_file: Option<PathBuf>,
 }
 
 impl<W> ExecuteContext<W>
@@ -116,6 +128,7 @@ where
             pb,
             status_dir,
             status_file: None,
+            log_file: None,
             id: None,
         }
     }
@@ -124,8 +137,18 @@ where
         let id = task.id();
         if !id.is_empty() {
             self.pb.set_prefix(id.clone());
-            self.status_file = Some(self.status_dir.path().join(format!("{}.status", id)));
-            self.id = Some(id);
+            self.id = Some(id.clone());
+
+            // Remove the old status file if exists to avoid confusion.
+            let status_file = self.status_dir.path().join(format!("{}.status", id));
+            fs_err::remove_file(&status_file).ok();
+            self.status_file = Some(status_file);
+
+            // Remove the old log file if exists to avoid confusion.
+            let log_file = Path::new(&env::var("PREFIX_LOG").unwrap())
+                .join(format!("{}.log", self.id.as_ref().unwrap()));
+            fs_err::remove_file(&log_file).ok();
+            self.log_file = Some(log_file);
         }
     }
 
@@ -150,7 +173,7 @@ where
 
         writeln!(self.log, "---")?;
 
-        output.status.exit_ok()?;
+        output.status.exit_ok().context(full_output)?;
 
         Ok(output)
     }
@@ -163,13 +186,16 @@ where
         self.status_file.clone().unwrap()
     }
 
-    pub fn log_path(&self) -> anyhow::Result<PathBuf> {
-        let prefix_log = env::var("PREFIX_LOG")?;
-        Ok(Path::new(&prefix_log).join(format!("{}.log", self.id.as_ref().unwrap())))
+    pub fn log_path(&self) -> &Path {
+        self.log_file.as_ref().unwrap().as_path()
     }
 
     pub fn wait_tcp(&mut self, server: impl AsRef<str>) -> anyhow::Result<()> {
-        let addr = server.as_ref().parse()?;
+        let addr = server
+            .as_ref()
+            .to_socket_addrs()?
+            .next()
+            .with_context(|| format!("failed to resolve {}", server.as_ref()))?;
         wait(
             || {
                 TcpStream::connect_timeout(&addr, Duration::from_secs(1)).with_context(|| {
@@ -255,7 +281,11 @@ where
 
     /// Wait for a user-managed service to be available
     pub fn wait_tcp_user(&mut self, server: impl AsRef<str>) -> anyhow::Result<()> {
-        let addr = server.as_ref().parse()?;
+        let addr = server
+            .as_ref()
+            .to_socket_addrs()?
+            .next()
+            .unwrap_or_else(|| panic!("failed to resolve {}", server.as_ref()));
         wait(
             || {
                 TcpStream::connect_timeout(&addr, Duration::from_secs(1))?;
@@ -294,7 +324,7 @@ where
             }
         }
         cmd.arg(Path::new(&prefix_path).join("run_command.sh"));
-        cmd.arg(self.log_path()?);
+        cmd.arg(self.log_path());
         cmd.arg(self.status_path());
         cmd.arg(user_cmd.get_program());
         for arg in user_cmd.get_args() {
diff --git a/src/risedevtool/src/task/compactor_service.rs b/src/risedevtool/src/task/compactor_service.rs
index 1a2b24b36bc97..c79f4efd06ee5 100644
--- a/src/risedevtool/src/task/compactor_service.rs
+++ b/src/risedevtool/src/task/compactor_service.rs
@@ -76,7 +76,9 @@ impl Task for CompactorService {
 
         let mut cmd = self.compactor()?;
 
-        cmd.env("RUST_BACKTRACE", "1");
+        if crate::util::is_enable_backtrace() {
+            cmd.env("RUST_BACKTRACE", "1");
+        }
 
         if crate::util::is_env_set("RISEDEV_ENABLE_PROFILE") {
             cmd.env(
diff --git a/src/risedevtool/src/task/compute_node_service.rs b/src/risedevtool/src/task/compute_node_service.rs
index 3dfba5f3e3e16..95ff9d6412fa2 100644
--- a/src/risedevtool/src/task/compute_node_service.rs
+++ b/src/risedevtool/src/task/compute_node_service.rs
@@ -55,9 +55,9 @@ impl ComputeNodeService {
             .arg("--async-stack-trace")
             .arg(&config.async_stack_trace)
             .arg("--parallelism")
-            .arg(&config.parallelism.to_string())
+            .arg(config.parallelism.to_string())
             .arg("--total-memory-bytes")
-            .arg(&config.total_memory_bytes.to_string())
+            .arg(config.total_memory_bytes.to_string())
             .arg("--role")
             .arg(&config.role);
 
@@ -80,10 +80,15 @@ impl Task for ComputeNodeService {
 
         let mut cmd = self.compute_node()?;
 
-        cmd.env("RUST_BACKTRACE", "1").env(
+        cmd.env(
             "TOKIO_CONSOLE_BIND",
             format!("127.0.0.1:{}", self.config.port + 1000),
         );
+
+        if crate::util::is_enable_backtrace() {
+            cmd.env("RUST_BACKTRACE", "1");
+        }
+
         if crate::util::is_env_set("RISEDEV_ENABLE_PROFILE") {
             cmd.env(
                 "RW_PROFILE_PATH",
diff --git a/src/risedevtool/src/task/docker_service.rs b/src/risedevtool/src/task/docker_service.rs
new file mode 100644
index 0000000000000..b87ee8a6a8aef
--- /dev/null
+++ b/src/risedevtool/src/task/docker_service.rs
@@ -0,0 +1,168 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::env;
+use std::path::Path;
+use std::process::{Command, Stdio};
+
+use anyhow::{Context, Result};
+
+use super::{ExecuteContext, Task};
+use crate::DummyService;
+
+/// Configuration for a docker-backed service.
+///
+/// The trait can be implemented for the configuration struct of a service.
+/// After that, use `DockerService<Config>` as the service type.
+pub trait DockerServiceConfig: Send + 'static {
+    /// The unique identifier of the service.
+    fn id(&self) -> String;
+
+    /// Whether the service is managed by the user.
+    ///
+    /// If true, no docker container will be started and the service will be forwarded to
+    /// the [`DummyService`].
+    fn is_user_managed(&self) -> bool;
+
+    /// The docker image to use, e.g. `mysql:5.7`.
+    fn image(&self) -> String;
+
+    /// The environment variables to pass to the docker container.
+    fn envs(&self) -> Vec<(String, String)> {
+        vec![]
+    }
+
+    /// The ports to expose on the host, e.g. `("0.0.0.0:23306", "3306")`.
+    ///
+    /// The first element of the tuple is the host port (or address), the second is the
+    /// container port.
+    fn ports(&self) -> Vec<(String, String)> {
+        vec![]
+    }
+
+    /// The path in the container to persist data to, e.g. `/var/lib/mysql`.
+    ///
+    /// `Some` if the service is specified to persist data, `None` otherwise.
+    fn data_path(&self) -> Option<String> {
+        None
+    }
+}
+
+/// A service that runs a docker container with the given configuration.
+pub struct DockerService<B> {
+    config: B,
+}
+
+impl<B> DockerService<B>
+where
+    B: DockerServiceConfig,
+{
+    pub fn new(config: B) -> Self {
+        Self { config }
+    }
+
+    /// Run `docker image inspect <image>` to check if the image exists locally.
+    ///
+    /// `docker run --pull=missing` does the same thing, but as we split the pull and run
+    /// into two commands while `pull` does not provide such an option, we need to check
+    /// the image existence manually.
+    fn check_image_exists(&self) -> bool {
+        Command::new("docker")
+            .arg("image")
+            .arg("inspect")
+            .arg(self.config.image())
+            .stdout(Stdio::null())
+            .stderr(Stdio::null())
+            .status()
+            .map(|status| status.success())
+            .unwrap_or(false)
+    }
+
+    fn docker_pull(&self) -> Command {
+        let mut cmd = Command::new("docker");
+        cmd.arg("pull").arg(self.config.image());
+        cmd
+    }
+
+    fn docker_run(&self) -> Result<Command> {
+        let mut cmd = Command::new("docker");
+        cmd.arg("run")
+            .arg("--rm")
+            .arg("--name")
+            .arg(format!("risedev-{}", self.id()))
+            .arg("--add-host")
+            .arg("host.docker.internal:host-gateway");
+
+        for (k, v) in self.config.envs() {
+            cmd.arg("-e").arg(format!("{k}={v}"));
+        }
+        for (container, host) in self.config.ports() {
+            cmd.arg("-p").arg(format!("{container}:{host}"));
+        }
+
+        if let Some(data_path) = self.config.data_path() {
+            let path = Path::new(&env::var("PREFIX_DATA")?).join(self.id());
+            fs_err::create_dir_all(&path)?;
+            cmd.arg("-v")
+                .arg(format!("{}:{}", path.to_string_lossy(), data_path));
+        }
+
+        cmd.arg(self.config.image());
+
+        Ok(cmd)
+    }
+}
+
+impl<B> Task for DockerService<B>
+where
+    B: DockerServiceConfig,
+{
+    fn execute(&mut self, ctx: &mut ExecuteContext<impl std::io::Write>) -> anyhow::Result<()> {
+        if self.config.is_user_managed() {
+            return DummyService::new(&self.id()).execute(ctx);
+        }
+
+        ctx.service(self);
+
+        check_docker_installed()?;
+
+        if !self.check_image_exists() {
+            ctx.pb
+                .set_message(format!("pulling image `{}`...", self.config.image()));
+            ctx.run_command(self.docker_pull())?;
+        }
+
+        ctx.pb.set_message("starting...");
+        ctx.run_command(ctx.tmux_run(self.docker_run()?)?)?;
+
+        ctx.pb.set_message("started");
+
+        Ok(())
+    }
+
+    fn id(&self) -> String {
+        self.config.id()
+    }
+}
+
+fn check_docker_installed() -> Result<()> {
+    Command::new("docker")
+        .arg("--version")
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status()
+        .map_err(anyhow::Error::from)
+        .and_then(|status| status.exit_ok().map_err(anyhow::Error::from))
+        .context("service requires docker to be installed")
+}
diff --git a/java/udf/src/main/java/com/risingwave/functions/PeriodDuration.java b/src/risedevtool/src/task/dummy_service.rs
similarity index 50%
rename from java/udf/src/main/java/com/risingwave/functions/PeriodDuration.java
rename to src/risedevtool/src/task/dummy_service.rs
index 6d704100f6f35..f3d1054b1ca23 100644
--- a/java/udf/src/main/java/com/risingwave/functions/PeriodDuration.java
+++ b/src/risedevtool/src/task/dummy_service.rs
@@ -12,18 +12,31 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.risingwave.functions;
+use crate::{ExecuteContext, Task};
 
-import java.time.Duration;
-import java.time.Period;
+pub struct DummyService {
+    id: String,
+}
+
+impl DummyService {
+    pub fn new(id: &str) -> Self {
+        Self { id: id.to_string() }
+    }
+}
 
-/** Combination of Period and Duration. */
-public class PeriodDuration extends org.apache.arrow.vector.PeriodDuration {
-    public PeriodDuration(Period period, Duration duration) {
-        super(period, duration);
+impl Task for DummyService {
+    fn execute(&mut self, ctx: &mut ExecuteContext<impl std::io::Write>) -> anyhow::Result<()> {
+        ctx.service(self);
+        writeln!(
+            &mut ctx.log,
+            "{} is a dummy service. Please ensure it's correctly configured on your own! 🙏",
+            self.id
+        )?;
+        ctx.complete_spin();
+        Ok(())
     }
 
-    PeriodDuration(org.apache.arrow.vector.PeriodDuration base) {
-        super(base.getPeriod(), base.getDuration());
+    fn id(&self) -> String {
+        self.id.clone()
     }
 }
diff --git a/src/risedevtool/src/task/frontend_service.rs b/src/risedevtool/src/task/frontend_service.rs
index 2ce67db6b7582..db944ad476278 100644
--- a/src/risedevtool/src/task/frontend_service.rs
+++ b/src/risedevtool/src/task/frontend_service.rs
@@ -87,7 +87,9 @@ impl Task for FrontendService {
 
         let mut cmd = self.frontend()?;
 
-        cmd.env("RUST_BACKTRACE", "1");
+        if crate::util::is_enable_backtrace() {
+            cmd.env("RUST_BACKTRACE", "1");
+        }
 
         let prefix_config = env::var("PREFIX_CONFIG")?;
         cmd.arg("--config-path")
diff --git a/src/risedevtool/src/task/kafka_service.rs b/src/risedevtool/src/task/kafka_service.rs
index df0eb4a0fa313..7c415b6d9749a 100644
--- a/src/risedevtool/src/task/kafka_service.rs
+++ b/src/risedevtool/src/task/kafka_service.rs
@@ -12,85 +12,71 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::env;
-use std::path::{Path, PathBuf};
-use std::process::Command;
+use super::docker_service::{DockerService, DockerServiceConfig};
+use crate::KafkaConfig;
 
-use anyhow::{anyhow, Result};
-
-use super::{ExecuteContext, Task};
-use crate::{KafkaConfig, KafkaGen};
-
-pub struct KafkaService {
-    config: KafkaConfig,
-}
-
-impl KafkaService {
-    pub fn new(config: KafkaConfig) -> Result<Self> {
-        Ok(Self { config })
+impl DockerServiceConfig for KafkaConfig {
+    fn id(&self) -> String {
+        self.id.clone()
     }
 
-    fn kafka_path(&self) -> Result<PathBuf> {
-        let prefix_bin = env::var("PREFIX_BIN")?;
-        Ok(Path::new(&prefix_bin)
-            .join("kafka")
-            .join("bin")
-            .join("kafka-server-start.sh"))
+    fn is_user_managed(&self) -> bool {
+        self.user_managed
     }
 
-    fn kafka(&self) -> Result<Command> {
-        Ok(Command::new(self.kafka_path()?))
+    fn image(&self) -> String {
+        self.image.clone()
     }
-}
-
-impl Task for KafkaService {
-    fn execute(&mut self, ctx: &mut ExecuteContext<impl std::io::Write>) -> anyhow::Result<()> {
-        ctx.service(self);
-        ctx.pb.set_message("starting...");
-
-        let path = self.kafka_path()?;
-        if !path.exists() {
-            return Err(anyhow!("Kafka binary not found in {:?}\nDid you enable kafka feature in `./risedev configure`?", path));
-        }
-
-        let prefix_config = env::var("PREFIX_CONFIG")?;
 
-        let path = if self.config.persist_data {
-            Path::new(&env::var("PREFIX_DATA")?).join(self.id())
-        } else {
-            let path = Path::new("/tmp/risedev").join(self.id());
-            fs_err::remove_dir_all(&path).ok();
-            path
-        };
-        fs_err::create_dir_all(&path)?;
-
-        let config_path = Path::new(&prefix_config).join(format!("{}.properties", self.id()));
-        fs_err::write(
-            &config_path,
-            KafkaGen.gen_server_properties(&self.config, &path.to_string_lossy()),
-        )?;
-
-        let mut cmd = self.kafka()?;
-
-        cmd.arg(config_path);
-
-        if !self.config.user_managed {
-            ctx.run_command(ctx.tmux_run(cmd)?)?;
-        } else {
-            ctx.pb.set_message("user managed");
-            writeln!(
-                &mut ctx.log,
-                "Please start your Kafka at {}:{}\n\n",
-                self.config.listen_address, self.config.port
-            )?;
-        }
-
-        ctx.pb.set_message("started");
+    fn envs(&self) -> Vec<(String, String)> {
+        vec![
+            ("KAFKA_NODE_ID".to_owned(), self.node_id.to_string()),
+            (
+                "KAFKA_PROCESS_ROLES".to_owned(),
+                "controller,broker".to_owned(),
+            ),
+            (
+                "KAFKA_LISTENERS".to_owned(),
+                "HOST://:9092,CONTROLLER://:9093,DOCKER://:9094".to_owned(),
+            ),
+            (
+                "KAFKA_ADVERTISED_LISTENERS".to_owned(),
+                format!(
+                    "HOST://{}:{},DOCKER://host.docker.internal:{}",
+                    self.address, self.port, self.docker_port
+                ),
+            ),
+            (
+                "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP".to_owned(),
+                "HOST:PLAINTEXT,CONTROLLER:PLAINTEXT,DOCKER:PLAINTEXT".to_owned(),
+            ),
+            (
+                "KAFKA_CONTROLLER_QUORUM_VOTERS".to_owned(),
+                format!("{}@localhost:9093", self.node_id),
+            ),
+            (
+                "KAFKA_CONTROLLER_LISTENER_NAMES".to_owned(),
+                "CONTROLLER".to_owned(),
+            ),
+            (
+                "KAFKA_INTER_BROKER_LISTENER_NAME".to_owned(),
+                "HOST".to_owned(),
+            ),
+            ("CLUSTER_ID".to_owned(), "RiseDevRiseDevRiseDev1".to_owned()),
+        ]
+    }
 
-        Ok(())
+    fn ports(&self) -> Vec<(String, String)> {
+        vec![
+            (self.port.to_string(), "9092".to_owned()),
+            (self.docker_port.to_string(), "9094".to_owned()),
+        ]
     }
 
-    fn id(&self) -> String {
-        self.config.id.clone()
+    fn data_path(&self) -> Option<String> {
+        self.persist_data.then(|| "/var/lib/kafka/data".to_owned())
     }
 }
+
+/// Docker-backed Kafka service.
+pub type KafkaService = DockerService<KafkaConfig>;
diff --git a/src/risedevtool/src/task/meta_node_service.rs b/src/risedevtool/src/task/meta_node_service.rs
index 4a121bb630d3a..caa22aed24c14 100644
--- a/src/risedevtool/src/task/meta_node_service.rs
+++ b/src/risedevtool/src/task/meta_node_service.rs
@@ -21,7 +21,10 @@ use itertools::Itertools;
 
 use super::{ExecuteContext, Task};
 use crate::util::{get_program_args, get_program_env_cmd, get_program_name};
-use crate::{add_hummock_backend, add_tempo_endpoint, HummockInMemoryStrategy, MetaNodeConfig};
+use crate::{
+    add_hummock_backend, add_tempo_endpoint, Application, HummockInMemoryStrategy, MetaBackend,
+    MetaNodeConfig,
+};
 
 pub struct MetaNodeService {
     config: MetaNodeConfig,
@@ -77,35 +80,80 @@ impl MetaNodeService {
 
         let mut is_persistent_meta_store = false;
 
-        if let Some(sqlite_config) = &config.provide_sqlite_backend
-            && !sqlite_config.is_empty()
-        {
-            is_persistent_meta_store = true;
-            let prefix_data = env::var("PREFIX_DATA")?;
-            let file_path = PathBuf::from(&prefix_data)
-                .join(&sqlite_config[0].id)
-                .join(&sqlite_config[0].file);
-            cmd.arg("--backend")
-                .arg("sql")
-                .arg("--sql-endpoint")
-                .arg(format!("sqlite://{}?mode=rwc", file_path.display()));
-        } else {
-            match config.provide_etcd_backend.as_ref().unwrap().as_slice() {
-                [] => {
-                    cmd.arg("--backend").arg("mem");
-                }
-                etcds => {
-                    is_persistent_meta_store = true;
-                    cmd.arg("--backend")
-                        .arg("etcd")
-                        .arg("--etcd-endpoints")
-                        .arg(
-                            etcds
-                                .iter()
-                                .map(|etcd| format!("{}:{}", etcd.address, etcd.port))
-                                .join(","),
-                        );
-                }
+        match &config.meta_backend {
+            MetaBackend::Memory => {
+                cmd.arg("--backend").arg("mem");
+            }
+            MetaBackend::Etcd => {
+                let etcd_config = config.provide_etcd_backend.as_ref().unwrap();
+                assert!(!etcd_config.is_empty());
+                is_persistent_meta_store = true;
+
+                cmd.arg("--backend")
+                    .arg("etcd")
+                    .arg("--etcd-endpoints")
+                    .arg(
+                        etcd_config
+                            .iter()
+                            .map(|etcd| format!("{}:{}", etcd.address, etcd.port))
+                            .join(","),
+                    );
+            }
+            MetaBackend::Sqlite => {
+                let sqlite_config = config.provide_sqlite_backend.as_ref().unwrap();
+                assert_eq!(sqlite_config.len(), 1);
+                is_persistent_meta_store = true;
+
+                let prefix_data = env::var("PREFIX_DATA")?;
+                let file_path = PathBuf::from(&prefix_data)
+                    .join(&sqlite_config[0].id)
+                    .join(&sqlite_config[0].file);
+                cmd.arg("--backend")
+                    .arg("sql")
+                    .arg("--sql-endpoint")
+                    .arg(format!("sqlite://{}?mode=rwc", file_path.display()));
+            }
+            MetaBackend::Postgres => {
+                let pg_config = config.provide_postgres_backend.as_ref().unwrap();
+                let pg_store_config = pg_config
+                    .iter()
+                    .filter(|c| c.application == Application::Metastore)
+                    .exactly_one()
+                    .expect("more than one or no pg store config found for metastore");
+                is_persistent_meta_store = true;
+
+                cmd.arg("--backend")
+                    .arg("sql")
+                    .arg("--sql-endpoint")
+                    .arg(format!(
+                        "postgres://{}:{}@{}:{}/{}",
+                        pg_store_config.user,
+                        pg_store_config.password,
+                        pg_store_config.address,
+                        pg_store_config.port,
+                        pg_store_config.database
+                    ));
+            }
+            MetaBackend::Mysql => {
+                let mysql_config = config.provide_mysql_backend.as_ref().unwrap();
+                let mysql_store_config = mysql_config
+                    .iter()
+                    .filter(|c| c.application == Application::Metastore)
+                    .exactly_one()
+                    .expect("more than one or no mysql store config found for metastore");
+                is_persistent_meta_store = true;
+
+                cmd.arg("--backend")
+                    .arg("sql")
+                    .arg("--sql-endpoint")
+                    .arg(format!(
+                        "mysql://{}:{}@{}:{}/{}",
+                        mysql_store_config.user,
+                        mysql_store_config.password,
+                        mysql_store_config.address,
+                        mysql_store_config.port,
+                        mysql_store_config.database
+                    ));
             }
         }
 
@@ -181,7 +229,9 @@ impl Task for MetaNodeService {
 
         let mut cmd = self.meta_node()?;
 
-        cmd.env("RUST_BACKTRACE", "1");
+        if crate::util::is_enable_backtrace() {
+            cmd.env("RUST_BACKTRACE", "1");
+        }
 
         if crate::util::is_env_set("RISEDEV_ENABLE_PROFILE") {
             cmd.env(
diff --git a/src/risedevtool/src/task/mysql_service.rs b/src/risedevtool/src/task/mysql_service.rs
new file mode 100644
index 0000000000000..06fbb1a62bb08
--- /dev/null
+++ b/src/risedevtool/src/task/mysql_service.rs
@@ -0,0 +1,59 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::docker_service::{DockerService, DockerServiceConfig};
+use crate::MySqlConfig;
+
+impl DockerServiceConfig for MySqlConfig {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn is_user_managed(&self) -> bool {
+        self.user_managed
+    }
+
+    fn image(&self) -> String {
+        self.image.clone()
+    }
+
+    fn envs(&self) -> Vec<(String, String)> {
+        let mut envs = vec![("MYSQL_DATABASE".to_owned(), self.database.clone())];
+        if self.user == "root" {
+            if self.password.is_empty() {
+                envs.push(("MYSQL_ALLOW_EMPTY_PASSWORD".to_owned(), "1".to_owned()));
+            } else {
+                envs.push(("MYSQL_ROOT_PASSWORD".to_owned(), self.password.clone()));
+            }
+        } else {
+            envs.extend([
+                ("MYSQL_ALLOW_EMPTY_PASSWORD".to_owned(), "1".to_owned()),
+                ("MYSQL_USER".to_owned(), self.user.clone()),
+                ("MYSQL_PASSWORD".to_owned(), self.password.clone()),
+            ]);
+        }
+        envs
+    }
+
+    fn ports(&self) -> Vec<(String, String)> {
+        vec![(self.port.to_string(), "3306".to_owned())]
+    }
+
+    fn data_path(&self) -> Option<String> {
+        self.persist_data.then(|| "/var/lib/mysql".to_owned())
+    }
+}
+
+/// Docker-backed MySQL service.
+pub type MySqlService = DockerService<MySqlConfig>;
diff --git a/src/risedevtool/src/task/postgres_service.rs b/src/risedevtool/src/task/postgres_service.rs
new file mode 100644
index 0000000000000..45700c93a1a97
--- /dev/null
+++ b/src/risedevtool/src/task/postgres_service.rs
@@ -0,0 +1,55 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::task::docker_service::{DockerService, DockerServiceConfig};
+use crate::PostgresConfig;
+
+impl DockerServiceConfig for PostgresConfig {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn is_user_managed(&self) -> bool {
+        self.user_managed
+    }
+
+    fn image(&self) -> String {
+        self.image.clone()
+    }
+
+    fn envs(&self) -> Vec<(String, String)> {
+        vec![
+            ("POSTGRES_HOST_AUTH_METHOD".to_owned(), "trust".to_owned()),
+            ("POSTGRES_USER".to_owned(), self.user.clone()),
+            ("POSTGRES_PASSWORD".to_owned(), self.password.clone()),
+            ("POSTGRES_DB".to_owned(), self.database.clone()),
+            (
+                "POSTGRES_INITDB_ARGS".to_owned(),
+                "--encoding=UTF-8 --lc-collate=C --lc-ctype=C".to_owned(),
+            ),
+        ]
+    }
+
+    fn ports(&self) -> Vec<(String, String)> {
+        vec![(self.port.to_string(), "5432".to_owned())]
+    }
+
+    fn data_path(&self) -> Option<String> {
+        self.persist_data
+            .then(|| "/var/lib/postgresql/data".to_owned())
+    }
+}
+
+/// Docker-backed PostgreSQL service.
+pub type PostgresService = DockerService<PostgresConfig>;
diff --git a/src/risedevtool/src/task/schema_registry_service.rs b/src/risedevtool/src/task/schema_registry_service.rs
new file mode 100644
index 0000000000000..5c5eba4fa8f35
--- /dev/null
+++ b/src/risedevtool/src/task/schema_registry_service.rs
@@ -0,0 +1,65 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::docker_service::{DockerService, DockerServiceConfig};
+use crate::SchemaRegistryConfig;
+
+impl DockerServiceConfig for SchemaRegistryConfig {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn is_user_managed(&self) -> bool {
+        self.user_managed
+    }
+
+    fn image(&self) -> String {
+        self.image.clone()
+    }
+
+    fn envs(&self) -> Vec<(String, String)> {
+        // https://docs.confluent.io/platform/current/installation/docker/config-reference.html#sr-long-configuration
+        // https://docs.confluent.io/platform/current/schema-registry/installation/config.html
+        let kafka = self
+            .provide_kafka
+            .as_ref()
+            .expect("Kafka is required for Schema Registry");
+        if kafka.len() != 1 {
+            panic!("More than one Kafka is not supported yet");
+        }
+        let kafka = &kafka[0];
+        vec![
+            ("SCHEMA_REGISTRY_HOST_NAME".to_owned(), self.address.clone()),
+            (
+                "SCHEMA_REGISTRY_LISTENERS".to_owned(),
+                format!("http://{}:{}", self.address, self.port),
+            ),
+            (
+                "SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS".to_owned(),
+                format!("host.docker.internal:{}", kafka.docker_port),
+            ),
+        ]
+    }
+
+    fn ports(&self) -> Vec<(String, String)> {
+        vec![(self.port.to_string(), "8081".to_owned())]
+    }
+
+    fn data_path(&self) -> Option<String> {
+        None
+    }
+}
+
+/// Docker-backed Schema Registry service.
+pub type SchemaRegistryService = DockerService<SchemaRegistryConfig>;
diff --git a/src/risedevtool/src/task/task_log_ready_check.rs b/src/risedevtool/src/task/task_log_ready_check.rs
new file mode 100644
index 0000000000000..a81d6961e491c
--- /dev/null
+++ b/src/risedevtool/src/task/task_log_ready_check.rs
@@ -0,0 +1,86 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::io::{Read as _, Seek as _, SeekFrom};
+use std::time::Duration;
+
+use anyhow::{bail, Context, Result};
+use fs_err::File;
+
+use super::{ExecuteContext, Task};
+use crate::wait::wait;
+
+/// Check if a log pattern is found in the log output indicating the service is ready.
+pub struct LogReadyCheckTask {
+    pattern: String,
+}
+
+impl LogReadyCheckTask {
+    pub fn new(pattern: impl Into<String>) -> Result<Self> {
+        Ok(Self {
+            pattern: pattern.into(),
+        })
+    }
+}
+
+impl Task for LogReadyCheckTask {
+    fn execute(&mut self, ctx: &mut ExecuteContext<impl std::io::Write>) -> anyhow::Result<()> {
+        let Some(id) = ctx.id.clone() else {
+            panic!("Service should be set before executing LogReadyCheckTask");
+        };
+
+        ctx.pb.set_message("waiting for ready...");
+        ctx.wait_log_contains(&self.pattern)
+            .with_context(|| format!("failed to wait for service `{id}` to be ready"))?;
+
+        ctx.complete_spin();
+
+        Ok(())
+    }
+}
+
+impl<W> ExecuteContext<W>
+where
+    W: std::io::Write,
+{
+    fn wait_log_contains(&mut self, pattern: impl AsRef<str>) -> anyhow::Result<()> {
+        let pattern = pattern.as_ref();
+        let log_path = self.log_path().to_path_buf();
+
+        let mut content = String::new();
+        let mut offset = 0;
+
+        wait(
+            || {
+                let mut file = File::open(&log_path).context("log file does not exist")?;
+                file.seek(SeekFrom::Start(offset as u64))?;
+                offset += file.read_to_string(&mut content)?;
+
+                // Always going through the whole log file could be stupid, but it's reliable.
+                if content.contains(pattern) {
+                    Ok(())
+                } else {
+                    bail!("pattern \"{}\" not found in log", pattern)
+                }
+            },
+            &mut self.log,
+            self.status_file.as_ref().unwrap(),
+            self.id.as_ref().unwrap(),
+            Some(Duration::from_secs(30)),
+            true,
+        )?;
+
+        Ok(())
+    }
+}
diff --git a/src/risedevtool/src/task/task_configure_grpc_node.rs b/src/risedevtool/src/task/task_tcp_ready_check.rs
similarity index 67%
rename from src/risedevtool/src/task/task_configure_grpc_node.rs
rename to src/risedevtool/src/task/task_tcp_ready_check.rs
index 75a69a7cdb29f..98b4238ca8830 100644
--- a/src/risedevtool/src/task/task_configure_grpc_node.rs
+++ b/src/risedevtool/src/task/task_tcp_ready_check.rs
@@ -12,17 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 
 use super::{ExecuteContext, Task};
 
-pub struct ConfigureGrpcNodeTask {
+/// Check if a TCP port can be connected to.
+///
+/// Note that accepting a connection does not always mean the service is ready.
+pub struct TcpReadyCheckTask {
     advertise_address: String,
     port: u16,
     user_managed: bool,
 }
 
-impl ConfigureGrpcNodeTask {
+impl TcpReadyCheckTask {
     pub fn new(advertise_address: String, port: u16, user_managed: bool) -> Result<Self> {
         Ok(Self {
             advertise_address,
@@ -32,18 +35,24 @@ impl ConfigureGrpcNodeTask {
     }
 }
 
-impl Task for ConfigureGrpcNodeTask {
+impl Task for TcpReadyCheckTask {
     fn execute(&mut self, ctx: &mut ExecuteContext<impl std::io::Write>) -> anyhow::Result<()> {
+        let Some(id) = ctx.id.clone() else {
+            panic!("Service should be set before executing TcpReadyCheckTask");
+        };
         let address = format!("{}:{}", self.advertise_address, self.port);
 
         if self.user_managed {
             ctx.pb.set_message(
                 "waiting for user-managed service online... (see `risedev.log` for cli args)",
             );
-            ctx.wait_tcp_user(&address)?;
+            ctx.wait_tcp_user(&address).with_context(|| {
+                format!("failed to wait for user-managed service `{id}` to be online")
+            })?;
         } else {
             ctx.pb.set_message("waiting for online...");
-            ctx.wait_tcp(&address)?;
+            ctx.wait_tcp(&address)
+                .with_context(|| format!("failed to wait for service `{id}` to be online"))?;
         }
 
         ctx.complete_spin();
diff --git a/src/risedevtool/src/task/zookeeper_service.rs b/src/risedevtool/src/task/zookeeper_service.rs
deleted file mode 100644
index 5a94de095cab9..0000000000000
--- a/src/risedevtool/src/task/zookeeper_service.rs
+++ /dev/null
@@ -1,87 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::env;
-use std::path::{Path, PathBuf};
-use std::process::Command;
-
-use anyhow::{anyhow, Result};
-
-use super::{ExecuteContext, Task};
-use crate::{ZooKeeperConfig, ZooKeeperGen};
-
-pub struct ZooKeeperService {
-    config: ZooKeeperConfig,
-}
-
-impl ZooKeeperService {
-    pub fn new(config: ZooKeeperConfig) -> Result<Self> {
-        Ok(Self { config })
-    }
-
-    fn zookeeper_path(&self) -> Result<PathBuf> {
-        let prefix_bin = env::var("PREFIX_BIN")?;
-        Ok(Path::new(&prefix_bin)
-            .join("kafka")
-            .join("bin")
-            .join("zookeeper-server-start.sh"))
-    }
-
-    fn zookeeper(&self) -> Result<Command> {
-        Ok(Command::new(self.zookeeper_path()?))
-    }
-}
-
-impl Task for ZooKeeperService {
-    fn execute(&mut self, ctx: &mut ExecuteContext<impl std::io::Write>) -> anyhow::Result<()> {
-        ctx.service(self);
-        ctx.pb.set_message("starting...");
-
-        let path = self.zookeeper_path()?;
-        if !path.exists() {
-            return Err(anyhow!("ZooKeeper binary not found in {:?}\nDid you enable kafka feature in `./risedev configure`?", path));
-        }
-
-        let prefix_config = env::var("PREFIX_CONFIG")?;
-
-        let path = if self.config.persist_data {
-            Path::new(&env::var("PREFIX_DATA")?).join(self.id())
-        } else {
-            let path = Path::new("/tmp/risedev").join(self.id());
-            fs_err::remove_dir_all(&path).ok();
-            path
-        };
-        fs_err::create_dir_all(&path)?;
-
-        let config_path = Path::new(&prefix_config).join(format!("{}.properties", self.id()));
-        fs_err::write(
-            &config_path,
-            ZooKeeperGen.gen_server_properties(&self.config, &path.to_string_lossy()),
-        )?;
-
-        let mut cmd = self.zookeeper()?;
-
-        cmd.arg(config_path);
-
-        ctx.run_command(ctx.tmux_run(cmd)?)?;
-
-        ctx.pb.set_message("started");
-
-        Ok(())
-    }
-
-    fn id(&self) -> String {
-        self.config.id.clone()
-    }
-}
diff --git a/src/risedevtool/src/util.rs b/src/risedevtool/src/util.rs
index a0cd7153515c5..42aa85730aa6c 100644
--- a/src/risedevtool/src/util.rs
+++ b/src/risedevtool/src/util.rs
@@ -79,3 +79,7 @@ pub fn is_env_set(var: &str) -> bool {
     }
     false
 }
+
+pub fn is_enable_backtrace() -> bool {
+    !is_env_set("DISABLE_BACKTRACE")
+}
diff --git a/src/risedevtool/src/wait.rs b/src/risedevtool/src/wait.rs
index ddedeb547270b..f2530371b83f8 100644
--- a/src/risedevtool/src/wait.rs
+++ b/src/risedevtool/src/wait.rs
@@ -13,12 +13,12 @@
 // limitations under the License.
 
 use std::io::Read;
-use std::net::TcpStream;
+use std::net::{TcpStream, ToSocketAddrs};
 use std::path::Path;
 use std::thread::sleep;
 use std::time::Duration;
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, Context as _, Result};
 use console::style;
 
 pub fn wait(
@@ -75,7 +75,7 @@ pub fn wait(
             });
         }
 
-        sleep(Duration::from_millis(30));
+        sleep(Duration::from_millis(100));
     }
 }
 
@@ -84,7 +84,10 @@ pub fn wait_tcp_available(
     timeout: Option<std::time::Duration>,
 ) -> anyhow::Result<()> {
     let server = server.as_ref();
-    let addr = server.parse()?;
+    let addr = server
+        .to_socket_addrs()?
+        .next()
+        .with_context(|| format!("failed to resolve {}", server))?;
     let start_time = std::time::Instant::now();
 
     loop {
@@ -101,6 +104,6 @@ pub fn wait_tcp_available(
             }
         }
 
-        sleep(Duration::from_millis(50));
+        sleep(Duration::from_millis(100));
     }
 }
diff --git a/src/rpc_client/Cargo.toml b/src/rpc_client/Cargo.toml
index c6e31e8eb0d76..70bebfa154fc4 100644
--- a/src/rpc_client/Cargo.toml
+++ b/src/rpc_client/Cargo.toml
@@ -17,7 +17,7 @@ normal = ["workspace-hack"]
 anyhow = "1"
 async-trait = "0.1"
 easy-ext = "1"
-either = "1.11.0"
+either = "1.12.0"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 http = "0.2"
 hyper = "0.14" # required by tonic
diff --git a/src/rpc_client/src/connector_client.rs b/src/rpc_client/src/connector_client.rs
index 896f198b8a5f1..30d78290b6d98 100644
--- a/src/rpc_client/src/connector_client.rs
+++ b/src/rpc_client/src/connector_client.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::fmt::Debug;
 use std::time::Duration;
 
@@ -30,6 +30,7 @@ use risingwave_pb::connector_service::sink_writer_stream_request::{
 };
 use risingwave_pb::connector_service::sink_writer_stream_response::CommitResponse;
 use risingwave_pb::connector_service::*;
+use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn;
 use thiserror_ext::AsReport;
 use tokio_stream::wrappers::ReceiverStream;
 use tonic::transport::{Channel, Endpoint};
@@ -201,7 +202,7 @@ impl ConnectorClient {
         source_id: u64,
         source_type: SourceType,
         start_offset: Option<String>,
-        properties: HashMap<String, String>,
+        properties: BTreeMap<String, String>,
         snapshot_done: bool,
         is_source_job: bool,
     ) -> Result<Streaming<GetEventStreamResponse>> {
@@ -233,11 +234,20 @@ impl ConnectorClient {
         &self,
         source_id: u64,
         source_type: SourceType,
-        properties: HashMap<String, String>,
+        properties: BTreeMap<String, String>,
         table_schema: Option<TableSchema>,
         is_source_job: bool,
         is_backfill_table: bool,
     ) -> Result<()> {
+        let table_schema = table_schema.map(|mut table_schema| {
+            table_schema.columns.retain(|c| {
+                !matches!(
+                    c.generated_or_default_column,
+                    Some(GeneratedOrDefaultColumn::GeneratedColumn(_))
+                )
+            });
+            table_schema
+        });
         let response = self
             .rpc_client
             .clone()
@@ -268,7 +278,6 @@ impl ConnectorClient {
         &self,
         payload_schema: Option<TableSchema>,
         sink_proto: PbSinkParam,
-        sink_payload_format: SinkPayloadFormat,
     ) -> Result<SinkWriterStreamHandle> {
         let mut rpc_client = self.rpc_client.clone();
         let (handle, first_rsp) = SinkWriterStreamHandle::initialize(
@@ -276,7 +285,6 @@ impl ConnectorClient {
                 request: Some(SinkRequest::Start(StartSink {
                     payload_schema,
                     sink_param: Some(sink_proto),
-                    format: sink_payload_format as i32,
                 })),
             },
             |rx| async move {
diff --git a/src/rpc_client/src/meta_client.rs b/src/rpc_client/src/meta_client.rs
index fbb8dff1f5a98..296f8de4d888f 100644
--- a/src/rpc_client/src/meta_client.rs
+++ b/src/rpc_client/src/meta_client.rs
@@ -23,9 +23,9 @@ use async_trait::async_trait;
 use either::Either;
 use futures::stream::BoxStream;
 use lru::LruCache;
-use risingwave_common::catalog::{CatalogVersion, FunctionId, IndexId, TableId};
+use risingwave_common::catalog::{CatalogVersion, FunctionId, IndexId, SecretId, TableId};
 use risingwave_common::config::{MetaConfig, MAX_CONNECTION_WINDOW_SIZE};
-use risingwave_common::hash::ParallelUnitMapping;
+use risingwave_common::hash::WorkerSlotMapping;
 use risingwave_common::system_param::reader::SystemParamsReader;
 use risingwave_common::telemetry::report::TelemetryInfoFetcher;
 use risingwave_common::util::addr::HostAddr;
@@ -172,6 +172,25 @@ impl MetaClient {
         Ok(resp.version)
     }
 
+    pub async fn create_secret(
+        &self,
+        secret_name: String,
+        database_id: u32,
+        schema_id: u32,
+        owner_id: u32,
+        value: Vec<u8>,
+    ) -> Result<CatalogVersion> {
+        let request = CreateSecretRequest {
+            name: secret_name,
+            database_id,
+            schema_id,
+            owner_id,
+            value,
+        };
+        let resp = self.inner.create_secret(request).await?;
+        Ok(resp.version)
+    }
+
     pub async fn list_connections(&self, _name: Option<&str>) -> Result<Vec<Connection>> {
         let request = ListConnectionsRequest {};
         let resp = self.inner.list_connections(request).await?;
@@ -184,6 +203,14 @@ impl MetaClient {
         Ok(resp.version)
     }
 
+    pub async fn drop_secret(&self, secret_id: SecretId) -> Result<CatalogVersion> {
+        let request = DropSecretRequest {
+            secret_id: secret_id.into(),
+        };
+        let resp = self.inner.drop_secret(request).await?;
+        Ok(resp.version)
+    }
+
     /// Register the current node to the cluster and set the corresponding worker id.
     pub async fn register_new(
         addr_strategy: MetaAddressStrategy,
@@ -384,11 +411,9 @@ impl MetaClient {
     pub async fn create_subscription(
         &self,
         subscription: PbSubscription,
-        graph: StreamFragmentGraph,
     ) -> Result<CatalogVersion> {
         let request = CreateSubscriptionRequest {
             subscription: Some(subscription),
-            fragment_graph: Some(graph),
         };
 
         let resp = self.inner.create_subscription(request).await?;
@@ -596,6 +621,21 @@ impl MetaClient {
         Ok(resp.version)
     }
 
+    pub async fn list_change_log_epochs(
+        &self,
+        table_id: u32,
+        min_epoch: u64,
+        max_count: u32,
+    ) -> Result<Vec<u64>> {
+        let request = ListChangeLogEpochsRequest {
+            table_id,
+            min_epoch,
+            max_count,
+        };
+        let resp = self.inner.list_change_log_epochs(request).await?;
+        Ok(resp.epochs)
+    }
+
     pub async fn drop_index(&self, index_id: IndexId, cascade: bool) -> Result<CatalogVersion> {
         let request = DropIndexRequest {
             index_id: index_id.index_id,
@@ -1158,11 +1198,11 @@ impl MetaClient {
 
     pub async fn list_serving_vnode_mappings(
         &self,
-    ) -> Result<HashMap<u32, (u32, ParallelUnitMapping)>> {
+    ) -> Result<HashMap<u32, (u32, WorkerSlotMapping)>> {
         let req = GetServingVnodeMappingsRequest {};
         let resp = self.inner.get_serving_vnode_mappings(req).await?;
         let mappings = resp
-            .mappings
+            .worker_slot_mappings
             .into_iter()
             .map(|p| {
                 (
@@ -1172,7 +1212,7 @@ impl MetaClient {
                             .get(&p.fragment_id)
                             .cloned()
                             .unwrap_or(0),
-                        ParallelUnitMapping::from_protobuf(p.mapping.as_ref().unwrap()),
+                        WorkerSlotMapping::from_protobuf(p.mapping.as_ref().unwrap()),
                     ),
                 )
             })
@@ -1918,12 +1958,14 @@ macro_rules! for_all_meta_rpc {
             ,{ ddl_client, create_subscription, CreateSubscriptionRequest, CreateSubscriptionResponse }
             ,{ ddl_client, create_schema, CreateSchemaRequest, CreateSchemaResponse }
             ,{ ddl_client, create_database, CreateDatabaseRequest, CreateDatabaseResponse }
+             ,{ ddl_client, create_secret, CreateSecretRequest, CreateSecretResponse }
             ,{ ddl_client, create_index, CreateIndexRequest, CreateIndexResponse }
             ,{ ddl_client, create_function, CreateFunctionRequest, CreateFunctionResponse }
             ,{ ddl_client, drop_table, DropTableRequest, DropTableResponse }
             ,{ ddl_client, drop_materialized_view, DropMaterializedViewRequest, DropMaterializedViewResponse }
             ,{ ddl_client, drop_view, DropViewRequest, DropViewResponse }
             ,{ ddl_client, drop_source, DropSourceRequest, DropSourceResponse }
+             , {ddl_client, drop_secret, DropSecretRequest, DropSecretResponse}
             ,{ ddl_client, drop_sink, DropSinkRequest, DropSinkResponse }
             ,{ ddl_client, drop_subscription, DropSubscriptionRequest, DropSubscriptionResponse }
             ,{ ddl_client, drop_database, DropDatabaseRequest, DropDatabaseResponse }
@@ -1976,6 +2018,7 @@ macro_rules! for_all_meta_rpc {
             ,{ hummock_client, list_compact_task_assignment, ListCompactTaskAssignmentRequest, ListCompactTaskAssignmentResponse }
             ,{ hummock_client, list_compact_task_progress, ListCompactTaskProgressRequest, ListCompactTaskProgressResponse }
             ,{ hummock_client, cancel_compact_task, CancelCompactTaskRequest, CancelCompactTaskResponse}
+            ,{ hummock_client, list_change_log_epochs, ListChangeLogEpochsRequest, ListChangeLogEpochsResponse }
             ,{ user_client, create_user, CreateUserRequest, CreateUserResponse }
             ,{ user_client, update_user, UpdateUserRequest, UpdateUserResponse }
             ,{ user_client, drop_user, DropUserRequest, DropUserResponse }
diff --git a/src/sqlparser/Cargo.toml b/src/sqlparser/Cargo.toml
index ef57da9aa62f8..bf6307e3ace68 100644
--- a/src/sqlparser/Cargo.toml
+++ b/src/sqlparser/Cargo.toml
@@ -1,10 +1,7 @@
 [package]
 name = "risingwave_sqlparser"
 license = "Apache-2.0"
-include = [
-    "src/**/*.rs",
-    "Cargo.toml",
-]
+include = ["src/**/*.rs", "Cargo.toml"]
 version = { workspace = true }
 edition = { workspace = true }
 homepage = { workspace = true }
@@ -27,14 +24,28 @@ normal = ["workspace-hack"]
 [dependencies]
 itertools = { workspace = true }
 serde = { version = "1.0", features = ["derive"], optional = true }
+thiserror = "1.0.61"
+tokio = { version = "0.2", package = "madsim-tokio", features = [
+    "rt",
+    "rt-multi-thread",
+    "macros",
+] }
 tracing = "0.1"
 tracing-subscriber = "0.3"
+winnow = "0.6.11"
 
 [target.'cfg(not(madsim))'.dependencies]
 workspace-hack = { path = "../workspace-hack" }
 
 [dev-dependencies]
+anyhow = "1"
+console = "0.15"
+libtest-mimic = "0.7"
 matches = "0.1"
+serde = { version = "1", features = ["derive"] }
+serde_with = "3"
+serde_yaml = "0.9"
+walkdir = "2"
 
 [package.metadata.release]
 # Instruct `cargo release` to not run `cargo publish` locally:
@@ -46,5 +57,9 @@ disable-publish = true
 name = "sqlparser"
 path = "src/bin/sqlparser.rs"
 
+[[test]]
+name = "parser_test"
+harness = false
+
 [lints]
 workspace = true
diff --git a/src/sqlparser/README.md b/src/sqlparser/README.md
index 2a829102710ba..e4e90a06eab84 100644
--- a/src/sqlparser/README.md
+++ b/src/sqlparser/README.md
@@ -5,4 +5,4 @@ This parser is a fork of <https://github.com/andygrove/sqlparser-rs>.
 
 ## Add a new test case
 1. Copy an item in the yaml file and edit the `input` to the sql you want to test.
-2. Run `./risedev do-apply-parser-test` to regenerate the `formatted_sql` which is the expected output.
\ No newline at end of file
+2. Run `./risedev update-parser-test` to regenerate the `formatted_sql` which is the expected output.
\ No newline at end of file
diff --git a/src/sqlparser/sqlparser_test.toml b/src/sqlparser/sqlparser_test.toml
new file mode 100644
index 0000000000000..8e348e8db925b
--- /dev/null
+++ b/src/sqlparser/sqlparser_test.toml
@@ -0,0 +1,8 @@
+[tasks.update-parser-test]
+description = "Update parser test data"
+private = true
+script = '''
+#!/usr/bin/env bash
+set -e
+UPDATE_PARSER_TEST=1 cargo test --test parser_test
+'''
diff --git a/src/sqlparser/src/ast/ddl.rs b/src/sqlparser/src/ast/ddl.rs
index a166182ac78e1..92683f42e742a 100644
--- a/src/sqlparser/src/ast/ddl.rs
+++ b/src/sqlparser/src/ast/ddl.rs
@@ -28,7 +28,6 @@ use crate::tokenizer::Token;
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub enum AlterDatabaseOperation {
     ChangeOwner { new_owner_name: Ident },
     RenameDatabase { database_name: ObjectName },
@@ -36,7 +35,6 @@ pub enum AlterDatabaseOperation {
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub enum AlterSchemaOperation {
     ChangeOwner { new_owner_name: Ident },
     RenameSchema { schema_name: ObjectName },
@@ -104,11 +102,14 @@ pub enum AlterTableOperation {
         deferred: bool,
     },
     RefreshSchema,
+    /// `SET STREAMING_RATE_LIMIT TO <rate_limit>`
+    SetStreamingRateLimit {
+        rate_limit: i32,
+    },
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub enum AlterIndexOperation {
     RenameIndex {
         index_name: ObjectName,
@@ -122,7 +123,6 @@ pub enum AlterIndexOperation {
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub enum AlterViewOperation {
     RenameView {
         view_name: ObjectName,
@@ -138,11 +138,14 @@ pub enum AlterViewOperation {
         parallelism: SetVariableValue,
         deferred: bool,
     },
+    /// `SET STREAMING_RATE_LIMIT TO <rate_limit>`
+    SetStreamingRateLimit {
+        rate_limit: i32,
+    },
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub enum AlterSinkOperation {
     RenameSink {
         sink_name: ObjectName,
@@ -162,27 +165,14 @@ pub enum AlterSinkOperation {
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub enum AlterSubscriptionOperation {
-    RenameSubscription {
-        subscription_name: ObjectName,
-    },
-    ChangeOwner {
-        new_owner_name: Ident,
-    },
-    SetSchema {
-        new_schema_name: ObjectName,
-    },
-    /// `SET PARALLELISM TO <parallelism> [ DEFERRED ]`
-    SetParallelism {
-        parallelism: SetVariableValue,
-        deferred: bool,
-    },
+    RenameSubscription { subscription_name: ObjectName },
+    ChangeOwner { new_owner_name: Ident },
+    SetSchema { new_schema_name: ObjectName },
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub enum AlterSourceOperation {
     RenameSource { source_name: ObjectName },
     AddColumn { column_def: ColumnDef },
@@ -190,18 +180,17 @@ pub enum AlterSourceOperation {
     SetSchema { new_schema_name: ObjectName },
     FormatEncode { connector_schema: ConnectorSchema },
     RefreshSchema,
+    SetStreamingRateLimit { rate_limit: i32 },
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub enum AlterFunctionOperation {
     SetSchema { new_schema_name: ObjectName },
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub enum AlterConnectionOperation {
     SetSchema { new_schema_name: ObjectName },
 }
@@ -301,6 +290,9 @@ impl fmt::Display for AlterTableOperation {
             AlterTableOperation::RefreshSchema => {
                 write!(f, "REFRESH SCHEMA")
             }
+            AlterTableOperation::SetStreamingRateLimit { rate_limit } => {
+                write!(f, "SET STREAMING_RATE_LIMIT TO {}", rate_limit)
+            }
         }
     }
 }
@@ -349,6 +341,9 @@ impl fmt::Display for AlterViewOperation {
                     if *deferred { " DEFERRED" } else { "" }
                 )
             }
+            AlterViewOperation::SetStreamingRateLimit { rate_limit } => {
+                write!(f, "SET STREAMING_RATE_LIMIT TO {}", rate_limit)
+            }
         }
     }
 }
@@ -392,17 +387,6 @@ impl fmt::Display for AlterSubscriptionOperation {
             AlterSubscriptionOperation::SetSchema { new_schema_name } => {
                 write!(f, "SET SCHEMA {}", new_schema_name)
             }
-            AlterSubscriptionOperation::SetParallelism {
-                parallelism,
-                deferred,
-            } => {
-                write!(
-                    f,
-                    "SET PARALLELISM TO {} {}",
-                    parallelism,
-                    if *deferred { " DEFERRED" } else { "" }
-                )
-            }
         }
     }
 }
@@ -428,6 +412,9 @@ impl fmt::Display for AlterSourceOperation {
             AlterSourceOperation::RefreshSchema => {
                 write!(f, "REFRESH SCHEMA")
             }
+            AlterSourceOperation::SetStreamingRateLimit { rate_limit } => {
+                write!(f, "SET STREAMING_RATE_LIMIT TO {}", rate_limit)
+            }
         }
     }
 }
diff --git a/src/sqlparser/src/ast/legacy_source.rs b/src/sqlparser/src/ast/legacy_source.rs
index ad005c49d0a13..6a079688c2d4e 100644
--- a/src/sqlparser/src/ast/legacy_source.rs
+++ b/src/sqlparser/src/ast/legacy_source.rs
@@ -20,14 +20,15 @@ use std::fmt;
 use itertools::Itertools as _;
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
+use winnow::PResult;
 
 use crate::ast::{
     AstString, AstVec, ConnectorSchema, Encode, Format, Ident, ObjectName, ParseTo, SqlOption,
     Value,
 };
 use crate::keywords::Keyword;
-use crate::parser::{Parser, ParserError};
-use crate::{impl_fmt_display, impl_parse_to};
+use crate::parser::{Parser, StrError};
+use crate::{impl_fmt_display, impl_parse_to, parser_err};
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
@@ -64,8 +65,11 @@ impl From<ConnectorSchema> for CompatibleSourceSchema {
     }
 }
 
-pub fn parse_source_schema(p: &mut Parser) -> Result<CompatibleSourceSchema, ParserError> {
+pub fn parse_source_schema(p: &mut Parser<'_>) -> PResult<CompatibleSourceSchema> {
     if let Some(schema_v2) = p.parse_schema()? {
+        if schema_v2.key_encode.is_some() {
+            parser_err!("key encode clause is not supported in source schema");
+        }
         Ok(CompatibleSourceSchema::V2(schema_v2))
     } else if p.peek_nth_any_of_keywords(0, &[Keyword::ROW])
         && p.peek_nth_any_of_keywords(1, &[Keyword::FORMAT])
@@ -104,26 +108,24 @@ pub fn parse_source_schema(p: &mut Parser) -> Result<CompatibleSourceSchema, Par
             }
             "BYTES" => SourceSchema::Bytes,
             _ => {
-                return Err(ParserError::ParserError(
+                parser_err!(
                     "expected JSON | UPSERT_JSON | PROTOBUF | DEBEZIUM_JSON | DEBEZIUM_AVRO \
                     | AVRO | UPSERT_AVRO | MAXWELL | CANAL_JSON | BYTES | NATIVE after ROW FORMAT"
-                        .to_string(),
-                ))
+                );
             }
         };
         Ok(CompatibleSourceSchema::RowFormat(schema))
     } else {
-        p.expected("description of the format", p.peek_token())
+        p.expected("description of the format")
     }
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub enum SourceSchema {
-    Protobuf(ProtobufSchema),
-    // Keyword::PROTOBUF ProtobufSchema
-    Json,         // Keyword::JSON
-    DebeziumJson, // Keyword::DEBEZIUM_JSON
+    Protobuf(ProtobufSchema), // Keyword::PROTOBUF ProtobufSchema
+    Json,                     // Keyword::JSON
+    DebeziumJson,             // Keyword::DEBEZIUM_JSON
     DebeziumMongoJson,
     UpsertJson,             // Keyword::UPSERT_JSON
     Avro(AvroSchema),       // Keyword::AVRO
@@ -241,6 +243,7 @@ impl SourceSchema {
             format,
             row_encode,
             row_options,
+            key_encode: None,
         }
     }
 }
@@ -281,7 +284,7 @@ pub struct ProtobufSchema {
 }
 
 impl ParseTo for ProtobufSchema {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!([Keyword::MESSAGE], p);
         impl_parse_to!(message_name: AstString, p);
         impl_parse_to!([Keyword::ROW, Keyword::SCHEMA, Keyword::LOCATION], p);
@@ -317,8 +320,9 @@ pub struct AvroSchema {
     pub row_schema_location: AstString,
     pub use_schema_registry: bool,
 }
+
 impl ParseTo for AvroSchema {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!([Keyword::ROW, Keyword::SCHEMA, Keyword::LOCATION], p);
         impl_parse_to!(use_schema_registry => [Keyword::CONFLUENT, Keyword::SCHEMA, Keyword::REGISTRY], p);
         impl_parse_to!(row_schema_location: AstString, p);
@@ -365,7 +369,7 @@ impl fmt::Display for DebeziumAvroSchema {
 }
 
 impl ParseTo for DebeziumAvroSchema {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!(
             [
                 Keyword::ROW,
@@ -391,19 +395,18 @@ pub struct CsvInfo {
     pub has_header: bool,
 }
 
-pub fn get_delimiter(chars: &str) -> Result<u8, ParserError> {
+pub fn get_delimiter(chars: &str) -> Result<u8, StrError> {
     match chars {
         "," => Ok(b','),   // comma
         "\t" => Ok(b'\t'), // tab
-        other => Err(ParserError::ParserError(format!(
-            "The delimiter should be one of ',', E'\\t', but got {:?}",
-            other
+        other => Err(StrError(format!(
+            "The delimiter should be one of ',', E'\\t', but got {other:?}",
         ))),
     }
 }
 
 impl ParseTo for CsvInfo {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!(without_header => [Keyword::WITHOUT, Keyword::HEADER], p);
         impl_parse_to!([Keyword::DELIMITED, Keyword::BY], p);
         impl_parse_to!(delimiter: AstString, p);
diff --git a/src/sqlparser/src/ast/mod.rs b/src/sqlparser/src/ast/mod.rs
index 919d87356a3f2..49fddbfaa4b82 100644
--- a/src/sqlparser/src/ast/mod.rs
+++ b/src/sqlparser/src/ast/mod.rs
@@ -27,10 +27,13 @@ use alloc::{
 };
 use core::fmt;
 use core::fmt::Display;
+use std::collections::HashSet;
+use std::sync::Arc;
 
 use itertools::Itertools;
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
+use winnow::PResult;
 
 pub use self::data_type::{DataType, StructField};
 pub use self::ddl::{
@@ -57,7 +60,13 @@ pub use crate::ast::ddl::{
     AlterViewOperation,
 };
 use crate::keywords::Keyword;
-use crate::parser::{IncludeOption, IncludeOptionItem, Parser, ParserError};
+use crate::parser::{IncludeOption, IncludeOptionItem, Parser, ParserError, StrError};
+
+pub type RedactSqlOptionKeywordsRef = Arc<HashSet<String>>;
+
+tokio::task_local! {
+    pub static REDACT_SQL_OPTION_KEYWORDS: RedactSqlOptionKeywordsRef;
+}
 
 pub struct DisplaySeparated<'a, T>
 where
@@ -183,7 +192,7 @@ impl From<&str> for Ident {
 }
 
 impl ParseTo for Ident {
-    fn parse_to(parser: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(parser: &mut Parser<'_>) -> PResult<Self> {
         parser.parse_identifier()
     }
 }
@@ -227,7 +236,7 @@ impl fmt::Display for ObjectName {
 }
 
 impl ParseTo for ObjectName {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         p.parse_object_name()
     }
 }
@@ -860,11 +869,7 @@ impl fmt::Display for WindowSpec {
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct WindowFrame {
     pub units: WindowFrameUnits,
-    pub start_bound: WindowFrameBound,
-    /// The right bound of the `BETWEEN .. AND` clause. The end bound of `None`
-    /// indicates the shorthand form (e.g. `ROWS 1 PRECEDING`), which must
-    /// behave the same as `end_bound = WindowFrameBound::CurrentRow`.
-    pub end_bound: Option<WindowFrameBound>,
+    pub bounds: WindowFrameBounds,
     pub exclusion: Option<WindowFrameExclusion>,
 }
 
@@ -874,18 +879,36 @@ pub enum WindowFrameUnits {
     Rows,
     Range,
     Groups,
+    Session,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub enum WindowFrameBounds {
+    Bounds {
+        start: WindowFrameBound,
+        /// The right bound of the `BETWEEN .. AND` clause. The end bound of `None`
+        /// indicates the shorthand form (e.g. `ROWS 1 PRECEDING`), which must
+        /// behave the same as `end_bound = WindowFrameBound::CurrentRow`.
+        end: Option<WindowFrameBound>,
+    },
+    Gap(Box<Expr>),
 }
 
 impl fmt::Display for WindowFrame {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if let Some(end_bound) = &self.end_bound {
-            write!(
-                f,
-                "{} BETWEEN {} AND {}",
-                self.units, self.start_bound, end_bound
-            )
-        } else {
-            write!(f, "{} {}", self.units, self.start_bound)
+        write!(f, "{} ", self.units)?;
+        match &self.bounds {
+            WindowFrameBounds::Bounds { start, end } => {
+                if let Some(end) = end {
+                    write!(f, "BETWEEN {} AND {}", start, end)
+                } else {
+                    write!(f, "{}", start)
+                }
+            }
+            WindowFrameBounds::Gap(gap) => {
+                write!(f, "WITH GAP {}", gap)
+            }
         }
     }
 }
@@ -896,6 +919,7 @@ impl fmt::Display for WindowFrameUnits {
             WindowFrameUnits::Rows => "ROWS",
             WindowFrameUnits::Range => "RANGE",
             WindowFrameUnits::Groups => "GROUPS",
+            WindowFrameUnits::Session => "SESSION",
         })
     }
 }
@@ -977,6 +1001,7 @@ pub enum ShowObject {
     Subscription { schema: Option<Ident> },
     Columns { table: ObjectName },
     Connection { schema: Option<Ident> },
+    Secret { schema: Option<Ident> },
     Function { schema: Option<Ident> },
     Indexes { table: ObjectName },
     Cluster,
@@ -1025,6 +1050,7 @@ impl fmt::Display for ShowObject {
             ShowObject::Jobs => write!(f, "JOBS"),
             ShowObject::ProcessList => write!(f, "PROCESSLIST"),
             ShowObject::Subscription { schema } => write!(f, "SUBSCRIPTIONS{}", fmt_schema(schema)),
+            ShowObject::Secret { schema } => write!(f, "SECRETS{}", fmt_schema(schema)),
         }
     }
 }
@@ -1101,6 +1127,7 @@ pub struct ExplainOptions {
     // explain's plan type
     pub explain_type: ExplainType,
 }
+
 impl Default for ExplainOptions {
     fn default() -> Self {
         Self {
@@ -1110,6 +1137,7 @@ impl Default for ExplainOptions {
         }
     }
 }
+
 impl fmt::Display for ExplainOptions {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let default = Self::default();
@@ -1263,6 +1291,9 @@ pub enum Statement {
     CreateConnection {
         stmt: CreateConnectionStatement,
     },
+    CreateSecret {
+        stmt: CreateSecretStatement,
+    },
     /// CREATE FUNCTION
     ///
     /// Postgres: <https://www.postgresql.org/docs/15/sql-createfunction.html>
@@ -1283,8 +1314,8 @@ pub enum Statement {
         or_replace: bool,
         name: ObjectName,
         args: Vec<OperateFunctionArg>,
+        returns: DataType,
         /// Optional parameters.
-        returns: Option<DataType>,
         append_only: bool,
         params: CreateFunctionBody,
     },
@@ -1387,7 +1418,7 @@ pub enum Statement {
     Kill(i32),
     /// DROP
     Drop(DropStatement),
-    /// DROP Function
+    /// DROP FUNCTION
     DropFunction {
         if_exists: bool,
         /// One or more function to drop
@@ -1395,6 +1426,14 @@ pub enum Statement {
         /// `CASCADE` or `RESTRICT`
         option: Option<ReferentialAction>,
     },
+    /// DROP AGGREGATE
+    DropAggregate {
+        if_exists: bool,
+        /// One or more function to drop
+        func_desc: Vec<FunctionDesc>,
+        /// `CASCADE` or `RESTRICT`
+        option: Option<ReferentialAction>,
+    },
     /// `SET <variable>`
     ///
     /// Note: this is not a standard SQL statement, but it is supported by at
@@ -1452,6 +1491,7 @@ pub enum Statement {
     CreateSchema {
         schema_name: ObjectName,
         if_not_exists: bool,
+        user_specified: Option<ObjectName>,
     },
     /// CREATE DATABASE
     CreateDatabase {
@@ -1559,14 +1599,14 @@ impl fmt::Display for Statement {
                 write!(f, "DESCRIBE {}", name)?;
                 Ok(())
             }
-            Statement::ShowObjects{ object: show_object, filter} => {
+            Statement::ShowObjects { object: show_object, filter } => {
                 write!(f, "SHOW {}", show_object)?;
                 if let Some(filter) = filter {
                     write!(f, " {}", filter)?;
                 }
                 Ok(())
             }
-            Statement::ShowCreateObject{ create_type: show_type, name } => {
+            Statement::ShowCreateObject { create_type: show_type, name } => {
                 write!(f, "SHOW CREATE {} {}", show_type, name)?;
                 Ok(())
             }
@@ -1580,7 +1620,7 @@ impl fmt::Display for Statement {
                 source,
                 returning,
             } => {
-                write!(f, "INSERT INTO {table_name} ", table_name = table_name,)?;
+                write!(f, "INSERT INTO {table_name} ", table_name = table_name, )?;
                 if !columns.is_empty() {
                     write!(f, "({}) ", display_comma_separated(columns))?;
                 }
@@ -1697,9 +1737,7 @@ impl fmt::Display for Statement {
                     or_replace = if *or_replace { "OR REPLACE " } else { "" },
                 )?;
                 write!(f, "({})", display_comma_separated(args))?;
-                if let Some(return_type) = returns {
-                    write!(f, " RETURNS {}", return_type)?;
-                }
+                write!(f, " RETURNS {}", returns)?;
                 if *append_only {
                     write!(f, " APPEND ONLY")?;
                 }
@@ -1790,18 +1828,18 @@ impl fmt::Display for Statement {
                     write!(f, "{}", display_comma_separated(
                         include_column_options.iter().map(|option_item: &IncludeOptionItem| {
                             format!("INCLUDE {}{}{}",
-                            option_item.column_type,
+                                    option_item.column_type,
                                     if let Some(inner_field) = &option_item.inner_field {
                                         format!(" {}", inner_field)
                                     } else {
                                         "".into()
                                     }
                                     , if let Some(alias) = &option_item.column_alias {
-                                        format!(" AS {}", alias)
-                                    } else {
-                                        "".into()
-                                    }
-                                )
+                                    format!(" AS {}", alias)
+                                } else {
+                                    "".into()
+                                }
+                            )
                         }).collect_vec().as_slice()
                     ))?;
                 }
@@ -1860,6 +1898,7 @@ impl fmt::Display for Statement {
             Statement::DeclareCursor { stmt } => write!(f, "DECLARE {}", stmt,),
             Statement::FetchCursor { stmt } => write!(f, "FETCH {}", stmt),
             Statement::CloseCursor { stmt } => write!(f, "CLOSE {}", stmt),
+            Statement::CreateSecret { stmt } => write!(f, "CREATE SECRET {}", stmt),
             Statement::AlterDatabase { name, operation } => {
                 write!(f, "ALTER DATABASE {} {}", name, operation)
             }
@@ -1912,6 +1951,22 @@ impl fmt::Display for Statement {
                 }
                 Ok(())
             }
+            Statement::DropAggregate {
+                if_exists,
+                func_desc,
+                option,
+            } => {
+                write!(
+                    f,
+                    "DROP AGGREGATE{} {}",
+                    if *if_exists { " IF EXISTS" } else { "" },
+                    display_comma_separated(func_desc),
+                )?;
+                if let Some(op) = option {
+                    write!(f, " {}", op)?;
+                }
+                Ok(())
+            }
             Statement::SetVariable {
                 local,
                 variable,
@@ -1972,20 +2027,27 @@ impl fmt::Display for Statement {
                 Ok(())
             }
             Statement::Commit { chain } => {
-                write!(f, "COMMIT{}", if *chain { " AND CHAIN" } else { "" },)
+                write!(f, "COMMIT{}", if *chain { " AND CHAIN" } else { "" }, )
             }
             Statement::Rollback { chain } => {
-                write!(f, "ROLLBACK{}", if *chain { " AND CHAIN" } else { "" },)
+                write!(f, "ROLLBACK{}", if *chain { " AND CHAIN" } else { "" }, )
             }
             Statement::CreateSchema {
                 schema_name,
                 if_not_exists,
-            } => write!(
-                f,
-                "CREATE SCHEMA {if_not_exists}{name}",
-                if_not_exists = if *if_not_exists { "IF NOT EXISTS " } else { "" },
-                name = schema_name
-            ),
+                user_specified,
+            } => {
+                write!(
+                    f,
+                    "CREATE SCHEMA {if_not_exists}{name}",
+                    if_not_exists = if *if_not_exists { "IF NOT EXISTS " } else { "" },
+                    name = schema_name
+                )?;
+                if let Some(user) = user_specified {
+                    write!(f, " AUTHORIZATION {}", user)?;
+                }
+                Ok(())
+            },
             Statement::Grant {
                 privileges,
                 objects,
@@ -2072,7 +2134,7 @@ impl fmt::Display for Statement {
             Statement::AlterUser(statement) => {
                 write!(f, "ALTER USER {}", statement)
             }
-            Statement::AlterSystem{param, value} => {
+            Statement::AlterSystem { param, value } => {
                 f.write_str("ALTER SYSTEM SET ")?;
                 write!(
                     f,
@@ -2490,6 +2552,7 @@ pub enum ObjectType {
     Database,
     User,
     Connection,
+    Secret,
     Subscription,
 }
 
@@ -2505,6 +2568,7 @@ impl fmt::Display for ObjectType {
             ObjectType::Sink => "SINK",
             ObjectType::Database => "DATABASE",
             ObjectType::User => "USER",
+            ObjectType::Secret => "SECRET",
             ObjectType::Connection => "CONNECTION",
             ObjectType::Subscription => "SUBSCRIPTION",
         })
@@ -2512,7 +2576,7 @@ impl fmt::Display for ObjectType {
 }
 
 impl ParseTo for ObjectType {
-    fn parse_to(parser: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(parser: &mut Parser<'_>) -> PResult<Self> {
         let object_type = if parser.parse_keyword(Keyword::TABLE) {
             ObjectType::Table
         } else if parser.parse_keyword(Keyword::VIEW) {
@@ -2533,12 +2597,13 @@ impl ParseTo for ObjectType {
             ObjectType::User
         } else if parser.parse_keyword(Keyword::CONNECTION) {
             ObjectType::Connection
+        } else if parser.parse_keyword(Keyword::SECRET) {
+            ObjectType::Secret
         } else if parser.parse_keyword(Keyword::SUBSCRIPTION) {
             ObjectType::Subscription
         } else {
             return parser.expected(
-                "TABLE, VIEW, INDEX, MATERIALIZED VIEW, SOURCE, SINK, SUBSCRIPTION, SCHEMA, DATABASE, USER or CONNECTION after DROP",
-                parser.peek_token(),
+                "TABLE, VIEW, INDEX, MATERIALIZED VIEW, SOURCE, SINK, SUBSCRIPTION, SCHEMA, DATABASE, USER, SECRET or CONNECTION after DROP",
             );
         };
         Ok(object_type)
@@ -2554,7 +2619,17 @@ pub struct SqlOption {
 
 impl fmt::Display for SqlOption {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{} = {}", self.name, self.value)
+        let should_redact = REDACT_SQL_OPTION_KEYWORDS
+            .try_with(|keywords| {
+                let sql_option_name = self.name.real_value().to_lowercase();
+                keywords.iter().any(|k| sql_option_name.contains(k))
+            })
+            .unwrap_or(false);
+        if should_redact {
+            write!(f, "{} = [REDACTED]", self.name)
+        } else {
+            write!(f, "{} = {}", self.name, self.value)
+        }
     }
 }
 
@@ -2706,7 +2781,6 @@ impl fmt::Display for DropFunctionOption {
 /// Function describe in DROP FUNCTION.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub struct FunctionDesc {
     pub name: ObjectName,
     pub args: Option<Vec<OperateFunctionArg>>,
@@ -2887,8 +2961,8 @@ impl fmt::Display for TableColumnDef {
 pub struct CreateFunctionBody {
     /// LANGUAGE lang_name
     pub language: Option<Ident>,
-
-    pub runtime: Option<FunctionRuntime>,
+    /// RUNTIME runtime_name
+    pub runtime: Option<Ident>,
 
     /// IMMUTABLE | STABLE | VOLATILE
     pub behavior: Option<FunctionBehavior>,
@@ -2909,11 +2983,9 @@ impl fmt::Display for CreateFunctionBody {
         if let Some(language) = &self.language {
             write!(f, " LANGUAGE {language}")?;
         }
-
         if let Some(runtime) = &self.runtime {
             write!(f, " RUNTIME {runtime}")?;
         }
-
         if let Some(behavior) = &self.behavior {
             write!(f, " {behavior}")?;
         }
@@ -2932,6 +3004,7 @@ impl fmt::Display for CreateFunctionBody {
         Ok(())
     }
 }
+
 #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct CreateFunctionWithOptions {
@@ -2948,7 +3021,7 @@ impl CreateFunctionWithOptions {
 
 /// TODO(kwannoel): Generate from the struct definition instead.
 impl TryFrom<Vec<SqlOption>> for CreateFunctionWithOptions {
-    type Error = ParserError;
+    type Error = StrError;
 
     fn try_from(with_options: Vec<SqlOption>) -> Result<Self, Self::Error> {
         let mut always_retry_on_network_error = None;
@@ -2956,10 +3029,7 @@ impl TryFrom<Vec<SqlOption>> for CreateFunctionWithOptions {
             if option.name.to_string().to_lowercase() == "always_retry_on_network_error" {
                 always_retry_on_network_error = Some(option.value == Value::Boolean(true));
             } else {
-                return Err(ParserError::ParserError(format!(
-                    "Unsupported option: {}",
-                    option.name
-                )));
+                return Err(StrError(format!("Unsupported option: {}", option.name)));
             }
         }
         Ok(Self {
@@ -3003,22 +3073,6 @@ impl fmt::Display for CreateFunctionUsing {
     }
 }
 
-#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
-#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-pub enum FunctionRuntime {
-    QuickJs,
-    Deno,
-}
-
-impl fmt::Display for FunctionRuntime {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            FunctionRuntime::QuickJs => write!(f, "quickjs"),
-            FunctionRuntime::Deno => write!(f, "deno"),
-        }
-    }
-}
-
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub enum CreateFunctionType {
@@ -3130,6 +3184,12 @@ impl fmt::Display for DiscardType {
     }
 }
 
+impl Statement {
+    pub fn to_redacted_string(&self, keywords: RedactSqlOptionKeywordsRef) -> String {
+        REDACT_SQL_OPTION_KEYWORDS.sync_scope(keywords, || self.to_string())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -3320,7 +3380,7 @@ mod tests {
                 as_: Some(FunctionDefinition::SingleQuotedDef("SELECT 1".to_string())),
                 return_: None,
                 using: None,
-                runtime: Some(FunctionRuntime::Deno),
+                runtime: Some(Ident::new_unchecked("deno")),
                 function_type: Some(CreateFunctionType::AsyncGenerator),
             },
             with_options: CreateFunctionWithOptions {
diff --git a/src/sqlparser/src/ast/statement.rs b/src/sqlparser/src/ast/statement.rs
index a821cd77e70e3..0c92209471450 100644
--- a/src/sqlparser/src/ast/statement.rs
+++ b/src/sqlparser/src/ast/statement.rs
@@ -13,25 +13,29 @@
 // limitations under the License.
 
 use core::fmt;
+use core::fmt::Formatter;
 use std::fmt::Write;
 
 use itertools::Itertools;
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
+use winnow::PResult;
 
 use super::ddl::SourceWatermark;
 use super::legacy_source::{parse_source_schema, CompatibleSourceSchema};
-use super::{EmitMode, Ident, ObjectType, Query};
+use super::{EmitMode, Ident, ObjectType, Query, Value};
 use crate::ast::{
     display_comma_separated, display_separated, ColumnDef, ObjectName, SqlOption, TableConstraint,
 };
 use crate::keywords::Keyword;
-use crate::parser::{IncludeOption, IsOptional, Parser, ParserError, UPSTREAM_SOURCE_KEY};
+use crate::parser::{IncludeOption, IsOptional, Parser, UPSTREAM_SOURCE_KEY};
+use crate::parser_err;
+use crate::parser_v2::literal_u32;
 use crate::tokenizer::Token;
 
 /// Consumes token from the parser into an AST node.
 pub trait ParseTo: Sized {
-    fn parse_to(parser: &mut Parser) -> Result<Self, ParserError>;
+    fn parse_to(parser: &mut Parser<'_>) -> PResult<Self>;
 }
 
 #[macro_export]
@@ -94,13 +98,20 @@ pub struct CreateSourceStatement {
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub enum Format {
     Native,
-    None,          // Keyword::NONE
-    Debezium,      // Keyword::DEBEZIUM
-    DebeziumMongo, // Keyword::DEBEZIUM_MONGO
-    Maxwell,       // Keyword::MAXWELL
-    Canal,         // Keyword::CANAL
-    Upsert,        // Keyword::UPSERT
-    Plain,         // Keyword::PLAIN
+    // Keyword::NONE
+    None,
+    // Keyword::DEBEZIUM
+    Debezium,
+    // Keyword::DEBEZIUM_MONGO
+    DebeziumMongo,
+    // Keyword::MAXWELL
+    Maxwell,
+    // Keyword::CANAL
+    Canal,
+    // Keyword::UPSERT
+    Upsert,
+    // Keyword::PLAIN
+    Plain,
 }
 
 // TODO: unify with `from_keyword`
@@ -124,7 +135,7 @@ impl fmt::Display for Format {
 }
 
 impl Format {
-    pub fn from_keyword(s: &str) -> Result<Self, ParserError> {
+    pub fn from_keyword(s: &str) -> PResult<Self> {
         Ok(match s {
             "DEBEZIUM" => Format::Debezium,
             "DEBEZIUM_MONGO" => Format::DebeziumMongo,
@@ -133,13 +144,10 @@ impl Format {
             "PLAIN" => Format::Plain,
             "UPSERT" => Format::Upsert,
             "NATIVE" => Format::Native, // used internally for schema change
-            "NONE" => Format::None, // used by iceberg
-            _ => {
-                return Err(ParserError::ParserError(
-                    "expected CANAL | PROTOBUF | DEBEZIUM | MAXWELL | PLAIN | NATIVE | NONE after FORMAT"
-                        .to_string(),
-                ))
-            }
+            "NONE" => Format::None,     // used by iceberg
+            _ => parser_err!(
+                "expected CANAL | PROTOBUF | DEBEZIUM | MAXWELL | PLAIN | NATIVE | NONE after FORMAT"
+            ),
         })
     }
 }
@@ -153,6 +161,7 @@ pub enum Encode {
     Json,     // Keyword::JSON
     Bytes,    // Keyword::BYTES
     None,     // Keyword::None
+    Text,     // Keyword::TEXT
     Native,
     Template,
 }
@@ -172,15 +181,17 @@ impl fmt::Display for Encode {
                 Encode::Native => "NATIVE",
                 Encode::Template => "TEMPLATE",
                 Encode::None => "NONE",
+                Encode::Text => "TEXT",
             }
         )
     }
 }
 
 impl Encode {
-    pub fn from_keyword(s: &str) -> Result<Self, ParserError> {
+    pub fn from_keyword(s: &str) -> PResult<Self> {
         Ok(match s {
             "AVRO" => Encode::Avro,
+            "TEXT" => Encode::Text,
             "BYTES" => Encode::Bytes,
             "CSV" => Encode::Csv,
             "PROTOBUF" => Encode::Protobuf,
@@ -188,10 +199,9 @@ impl Encode {
             "TEMPLATE" => Encode::Template,
             "NATIVE" => Encode::Native, // used internally for schema change
             "NONE" => Encode::None, // used by iceberg
-            _ => return Err(ParserError::ParserError(
+            _ => parser_err!(
                 "expected AVRO | BYTES | CSV | PROTOBUF | JSON | NATIVE | TEMPLATE | NONE after Encode"
-                    .to_string(),
-            )),
+            ),
         })
     }
 }
@@ -202,9 +212,11 @@ pub struct ConnectorSchema {
     pub format: Format,
     pub row_encode: Encode,
     pub row_options: Vec<SqlOption>,
+
+    pub key_encode: Option<Encode>,
 }
 
-impl Parser {
+impl Parser<'_> {
     /// Peek the next tokens to see if it is `FORMAT` or `ROW FORMAT` (for compatibility).
     fn peek_source_schema_format(&mut self) -> bool {
         (self.peek_nth_any_of_keywords(0, &[Keyword::ROW])
@@ -217,7 +229,7 @@ impl Parser {
         &mut self,
         connector: &str,
         cdc_source_job: bool,
-    ) -> Result<CompatibleSourceSchema, ParserError> {
+    ) -> PResult<CompatibleSourceSchema> {
         // row format for cdc source must be debezium json
         // row format for nexmark source must be native
         // default row format for datagen source is native
@@ -234,10 +246,10 @@ impl Parser {
             if self.peek_source_schema_format() {
                 let schema = parse_source_schema(self)?.into_v2();
                 if schema != expected {
-                    return Err(ParserError::ParserError(format!(
+                    parser_err!(
                         "Row format for CDC connectors should be \
                          either omitted or set to `{expected}`",
-                    )));
+                    );
                 }
             }
             Ok(expected.into())
@@ -246,10 +258,10 @@ impl Parser {
             if self.peek_source_schema_format() {
                 let schema = parse_source_schema(self)?.into_v2();
                 if schema != expected {
-                    return Err(ParserError::ParserError(format!(
+                    parser_err!(
                         "Row format for nexmark connectors should be \
                          either omitted or set to `{expected}`",
-                    )));
+                    );
                 }
             }
             Ok(expected.into())
@@ -264,10 +276,10 @@ impl Parser {
             if self.peek_source_schema_format() {
                 let schema = parse_source_schema(self)?.into_v2();
                 if schema != expected {
-                    return Err(ParserError::ParserError(format!(
+                    parser_err!(
                         "Row format for iceberg connectors should be \
                          either omitted or set to `{expected}`",
-                    )));
+                    );
                 }
             }
             Ok(expected.into())
@@ -277,7 +289,7 @@ impl Parser {
     }
 
     /// Parse `FORMAT ... ENCODE ... (...)`.
-    pub fn parse_schema(&mut self) -> Result<Option<ConnectorSchema>, ParserError> {
+    pub fn parse_schema(&mut self) -> PResult<Option<ConnectorSchema>> {
         if !self.parse_keyword(Keyword::FORMAT) {
             return Ok(None);
         }
@@ -291,10 +303,19 @@ impl Parser {
         let row_encode = Encode::from_keyword(&s)?;
         let row_options = self.parse_options()?;
 
+        let key_encode = if self.parse_keywords(&[Keyword::KEY, Keyword::ENCODE]) {
+            Some(Encode::from_keyword(
+                self.parse_identifier()?.value.to_ascii_uppercase().as_str(),
+            )?)
+        } else {
+            None
+        };
+
         Ok(Some(ConnectorSchema {
             format,
             row_encode,
             row_options,
+            key_encode,
         }))
     }
 }
@@ -305,6 +326,7 @@ impl ConnectorSchema {
             format: Format::Plain,
             row_encode: Encode::Json,
             row_options: Vec::new(),
+            key_encode: None,
         }
     }
 
@@ -314,6 +336,7 @@ impl ConnectorSchema {
             format: Format::Debezium,
             row_encode: Encode::Json,
             row_options: Vec::new(),
+            key_encode: None,
         }
     }
 
@@ -322,6 +345,7 @@ impl ConnectorSchema {
             format: Format::DebeziumMongo,
             row_encode: Encode::Json,
             row_options: Vec::new(),
+            key_encode: None,
         }
     }
 
@@ -331,6 +355,7 @@ impl ConnectorSchema {
             format: Format::Native,
             row_encode: Encode::Native,
             row_options: Vec::new(),
+            key_encode: None,
         }
     }
 
@@ -341,6 +366,7 @@ impl ConnectorSchema {
             format: Format::None,
             row_encode: Encode::None,
             row_options: Vec::new(),
+            key_encode: None,
         }
     }
 
@@ -362,7 +388,7 @@ impl fmt::Display for ConnectorSchema {
 }
 
 impl ParseTo for CreateSourceStatement {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!(if_not_exists => [Keyword::IF, Keyword::NOT, Keyword::EXISTS], p);
         impl_parse_to!(source_name: ObjectName, p);
 
@@ -376,9 +402,11 @@ impl ParseTo for CreateSourceStatement {
             .iter()
             .find(|&opt| opt.name.real_value() == UPSTREAM_SOURCE_KEY);
         let connector: String = option.map(|opt| opt.value.to_string()).unwrap_or_default();
-        // The format of cdc source job is fixed to `FORMAT PLAIN ENCODE JSON`
-        let cdc_source_job =
-            connector.contains("-cdc") && columns.is_empty() && constraints.is_empty();
+        let cdc_source_job = connector.contains("-cdc");
+        if cdc_source_job && (!columns.is_empty() || !constraints.is_empty()) {
+            parser_err!("CDC source cannot define columns and constraints");
+        }
+
         // row format for nexmark source must be native
         // default row format for datagen source is native
         let source_schema = p.parse_source_schema_with_connector(&connector, cdc_source_job)?;
@@ -472,7 +500,6 @@ impl fmt::Display for CreateSink {
         }
     }
 }
-
 // sql_grammar!(CreateSinkStatement {
 //     if_not_exists => [Keyword::IF, Keyword::NOT, Keyword::EXISTS],
 //     sink_name: Ident,
@@ -494,7 +521,7 @@ pub struct CreateSinkStatement {
 }
 
 impl ParseTo for CreateSinkStatement {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!(if_not_exists => [Keyword::IF, Keyword::NOT, Keyword::EXISTS], p);
         impl_parse_to!(sink_name: ObjectName, p);
 
@@ -514,22 +541,20 @@ impl ParseTo for CreateSinkStatement {
             let query = Box::new(p.parse_query()?);
             CreateSink::AsQuery(query)
         } else {
-            p.expected("FROM or AS after CREATE SINK sink_name", p.peek_token())?
+            p.expected("FROM or AS after CREATE SINK sink_name")?
         };
 
-        let emit_mode = p.parse_emit_mode()?;
+        let emit_mode: Option<EmitMode> = p.parse_emit_mode()?;
 
         // This check cannot be put into the `WithProperties::parse_to`, since other
         // statements may not need the with properties.
         if !p.peek_nth_any_of_keywords(0, &[Keyword::WITH]) && into_table_name.is_none() {
-            p.expected("WITH", p.peek_token())?
+            p.expected("WITH")?
         }
         impl_parse_to!(with_properties: WithProperties, p);
 
         if with_properties.0.is_empty() && into_table_name.is_none() {
-            return Err(ParserError::ParserError(
-                "sink properties not provided".to_string(),
-            ));
+            parser_err!("sink properties not provided");
         }
 
         let sink_schema = p.parse_schema()?;
@@ -586,7 +611,7 @@ pub struct CreateSubscriptionStatement {
 }
 
 impl ParseTo for CreateSubscriptionStatement {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!(if_not_exists => [Keyword::IF, Keyword::NOT, Keyword::EXISTS], p);
         impl_parse_to!(subscription_name: ObjectName, p);
 
@@ -594,10 +619,7 @@ impl ParseTo for CreateSubscriptionStatement {
             impl_parse_to!(from_name: ObjectName, p);
             from_name
         } else {
-            p.expected(
-                "FROM after CREATE SUBSCRIPTION subscription_name",
-                p.peek_token(),
-            )?
+            p.expected("FROM after CREATE SUBSCRIPTION subscription_name")?
         };
 
         // let emit_mode = p.parse_emit_mode()?;
@@ -605,14 +627,12 @@ impl ParseTo for CreateSubscriptionStatement {
         // This check cannot be put into the `WithProperties::parse_to`, since other
         // statements may not need the with properties.
         if !p.peek_nth_any_of_keywords(0, &[Keyword::WITH]) {
-            p.expected("WITH", p.peek_token())?
+            p.expected("WITH")?
         }
         impl_parse_to!(with_properties: WithProperties, p);
 
         if with_properties.0.is_empty() {
-            return Err(ParserError::ParserError(
-                "subscription properties not provided".to_string(),
-            ));
+            parser_err!("subscription properties not provided");
         }
 
         Ok(Self {
@@ -623,6 +643,7 @@ impl ParseTo for CreateSubscriptionStatement {
         })
     }
 }
+
 impl fmt::Display for CreateSubscriptionStatement {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let mut v: Vec<String> = vec![];
@@ -654,6 +675,7 @@ impl fmt::Display for DeclareCursor {
         v.iter().join(" ").fmt(f)
     }
 }
+
 // sql_grammar!(DeclareCursorStatement {
 //     cursor_name: Ident,
 //     [Keyword::SUBSCRIPTION]
@@ -671,7 +693,7 @@ pub struct DeclareCursorStatement {
 }
 
 impl ParseTo for DeclareCursorStatement {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!(cursor_name: ObjectName, p);
 
         let declare_cursor = if !p.parse_keyword(Keyword::SUBSCRIPTION) {
@@ -692,6 +714,7 @@ impl ParseTo for DeclareCursorStatement {
         })
     }
 }
+
 impl fmt::Display for DeclareCursorStatement {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let mut v: Vec<String> = vec![];
@@ -713,14 +736,11 @@ pub struct FetchCursorStatement {
 }
 
 impl ParseTo for FetchCursorStatement {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         let count = if p.parse_keyword(Keyword::NEXT) {
             1
         } else {
-            let count_str = p.parse_number_value()?;
-            count_str.parse::<u32>().map_err(|e| {
-                ParserError::ParserError(format!("Could not parse '{}' as i32: {}", count_str, e))
-            })?
+            literal_u32(p)?
         };
         p.expect_keyword(Keyword::FROM)?;
         impl_parse_to!(cursor_name: ObjectName, p);
@@ -753,7 +773,7 @@ pub struct CloseCursorStatement {
 }
 
 impl ParseTo for CloseCursorStatement {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         let cursor_name = if p.parse_keyword(Keyword::ALL) {
             None
         } else {
@@ -763,6 +783,7 @@ impl ParseTo for CloseCursorStatement {
         Ok(Self { cursor_name })
     }
 }
+
 impl fmt::Display for CloseCursorStatement {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let mut v: Vec<String> = vec![];
@@ -789,14 +810,12 @@ pub struct CreateConnectionStatement {
 }
 
 impl ParseTo for CreateConnectionStatement {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!(if_not_exists => [Keyword::IF, Keyword::NOT, Keyword::EXISTS], p);
         impl_parse_to!(connection_name: ObjectName, p);
         impl_parse_to!(with_properties: WithProperties, p);
         if with_properties.0.is_empty() {
-            return Err(ParserError::ParserError(
-                "connection properties not provided".to_string(),
-            ));
+            parser_err!("connection properties not provided");
         }
 
         Ok(Self {
@@ -817,6 +836,47 @@ impl fmt::Display for CreateConnectionStatement {
     }
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct CreateSecretStatement {
+    pub if_not_exists: bool,
+    pub secret_name: ObjectName,
+    pub credential: Value,
+    pub with_properties: WithProperties,
+}
+
+impl ParseTo for CreateSecretStatement {
+    fn parse_to(parser: &mut Parser<'_>) -> PResult<Self> {
+        impl_parse_to!(if_not_exists => [Keyword::IF, Keyword::NOT, Keyword::EXISTS], parser);
+        impl_parse_to!(secret_name: ObjectName, parser);
+        impl_parse_to!(with_properties: WithProperties, parser);
+        let mut credential = Value::Null;
+        if parser.parse_keyword(Keyword::AS) {
+            credential = parser.parse_value()?;
+        }
+        Ok(Self {
+            if_not_exists,
+            secret_name,
+            credential,
+            with_properties,
+        })
+    }
+}
+
+impl fmt::Display for CreateSecretStatement {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        let mut v: Vec<String> = vec![];
+        impl_fmt_display!(if_not_exists => [Keyword::IF, Keyword::NOT, Keyword::EXISTS], v, self);
+        impl_fmt_display!(secret_name, v, self);
+        impl_fmt_display!(with_properties, v, self);
+        if self.credential != Value::Null {
+            v.push("AS".to_string());
+            impl_fmt_display!(credential, v, self);
+        }
+        v.iter().join(" ").fmt(f)
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct AstVec<T>(pub Vec<T>);
@@ -832,7 +892,7 @@ impl<T: fmt::Display> fmt::Display for AstVec<T> {
 pub struct WithProperties(pub Vec<SqlOption>);
 
 impl ParseTo for WithProperties {
-    fn parse_to(parser: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(parser: &mut Parser<'_>) -> PResult<Self> {
         Ok(Self(
             parser.parse_options_with_preceding_keyword(Keyword::WITH)?,
         ))
@@ -875,7 +935,7 @@ pub struct RowSchemaLocation {
 }
 
 impl ParseTo for RowSchemaLocation {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!([Keyword::ROW, Keyword::SCHEMA, Keyword::LOCATION], p);
         impl_parse_to!(value: AstString, p);
         Ok(Self { value })
@@ -898,7 +958,7 @@ impl fmt::Display for RowSchemaLocation {
 pub struct AstString(pub String);
 
 impl ParseTo for AstString {
-    fn parse_to(parser: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(parser: &mut Parser<'_>) -> PResult<Self> {
         Ok(Self(parser.parse_literal_string()?))
     }
 }
@@ -921,7 +981,7 @@ pub enum AstOption<T> {
 }
 
 impl<T: ParseTo> ParseTo for AstOption<T> {
-    fn parse_to(parser: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(parser: &mut Parser<'_>) -> PResult<Self> {
         match T::parse_to(parser) {
             Ok(t) => Ok(AstOption::Some(t)),
             Err(_) => Ok(AstOption::None),
@@ -1041,17 +1101,14 @@ impl UserOptionsBuilder {
 }
 
 impl ParseTo for UserOptions {
-    fn parse_to(parser: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(parser: &mut Parser<'_>) -> PResult<Self> {
         let mut builder = UserOptionsBuilder::default();
         let add_option = |item: &mut Option<UserOption>, user_option| {
             let old_value = item.replace(user_option);
             if old_value.is_some() {
-                Err(ParserError::ParserError(
-                    "conflicting or redundant options".to_string(),
-                ))
-            } else {
-                Ok(())
+                parser_err!("conflicting or redundant options");
             }
+            Ok(())
         };
         let _ = parser.parse_keyword(Keyword::WITH);
         loop {
@@ -1061,6 +1118,7 @@ impl ParseTo for UserOptions {
             }
 
             if let Token::Word(ref w) = token.token {
+                let checkpoint = *parser;
                 parser.next_token();
                 let (item_mut_ref, user_option) = match w.keyword {
                     Keyword::SUPERUSER => (&mut builder.super_user, UserOption::SuperUser),
@@ -1093,10 +1151,10 @@ impl ParseTo for UserOptions {
                         (&mut builder.password, UserOption::OAuth(options))
                     }
                     _ => {
-                        parser.expected(
+                        parser.expected_at(
+                            checkpoint,
                             "SUPERUSER | NOSUPERUSER | CREATEDB | NOCREATEDB | LOGIN \
                             | NOLOGIN | CREATEUSER | NOCREATEUSER | [ENCRYPTED] PASSWORD | NULL | OAUTH",
-                            token,
                         )?;
                         unreachable!()
                     }
@@ -1106,7 +1164,6 @@ impl ParseTo for UserOptions {
                 parser.expected(
                     "SUPERUSER | NOSUPERUSER | CREATEDB | NOCREATEDB | LOGIN | NOLOGIN \
                         | CREATEUSER | NOCREATEUSER | [ENCRYPTED] PASSWORD | NULL | OAUTH",
-                    token,
                 )?
             }
         }
@@ -1125,7 +1182,7 @@ impl fmt::Display for UserOptions {
 }
 
 impl ParseTo for CreateUserStatement {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!(user_name: ObjectName, p);
         impl_parse_to!(with_options: UserOptions, p);
 
@@ -1168,7 +1225,7 @@ impl fmt::Display for AlterUserStatement {
 }
 
 impl ParseTo for AlterUserStatement {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!(user_name: ObjectName, p);
         impl_parse_to!(mode: AlterUserMode, p);
 
@@ -1177,7 +1234,7 @@ impl ParseTo for AlterUserStatement {
 }
 
 impl ParseTo for AlterUserMode {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         if p.parse_keyword(Keyword::RENAME) {
             p.expect_keyword(Keyword::TO)?;
             impl_parse_to!(new_name: ObjectName, p);
@@ -1210,7 +1267,7 @@ pub struct DropStatement {
 //     drop_mode: AstOption<DropMode>,
 // });
 impl ParseTo for DropStatement {
-    fn parse_to(p: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(p: &mut Parser<'_>) -> PResult<Self> {
         impl_parse_to!(object_type: ObjectType, p);
         impl_parse_to!(if_exists => [Keyword::IF, Keyword::EXISTS], p);
         let object_name = p.parse_object_name()?;
@@ -1243,13 +1300,13 @@ pub enum DropMode {
 }
 
 impl ParseTo for DropMode {
-    fn parse_to(parser: &mut Parser) -> Result<Self, ParserError> {
+    fn parse_to(parser: &mut Parser<'_>) -> PResult<Self> {
         let drop_mode = if parser.parse_keyword(Keyword::CASCADE) {
             DropMode::Cascade
         } else if parser.parse_keyword(Keyword::RESTRICT) {
             DropMode::Restrict
         } else {
-            return parser.expected("CASCADE | RESTRICT", parser.peek_token());
+            return parser.expected("CASCADE | RESTRICT");
         };
         Ok(drop_mode)
     }
diff --git a/src/sqlparser/src/ast/value.rs b/src/sqlparser/src/ast/value.rs
index 2ce52f3c18bf9..79f2a6ebd99ca 100644
--- a/src/sqlparser/src/ast/value.rs
+++ b/src/sqlparser/src/ast/value.rs
@@ -17,6 +17,8 @@ use core::fmt;
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
 
+use super::ObjectName;
+
 /// Primitive SQL values such as number and string
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
@@ -57,6 +59,8 @@ pub enum Value {
     },
     /// `NULL` value
     Null,
+    /// name of the reference to secret
+    Ref(ObjectName),
 }
 
 impl fmt::Display for Value {
@@ -111,6 +115,7 @@ impl fmt::Display for Value {
                 Ok(())
             }
             Value::Null => write!(f, "NULL"),
+            Value::Ref(v) => write!(f, "ref secret {}", v),
         }
     }
 }
diff --git a/src/sqlparser/src/bin/sqlparser.rs b/src/sqlparser/src/bin/sqlparser.rs
index be2ec51bc78fc..57a984a8c1cd1 100644
--- a/src/sqlparser/src/bin/sqlparser.rs
+++ b/src/sqlparser/src/bin/sqlparser.rs
@@ -1,3 +1,19 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![feature(register_tool)]
+#![register_tool(rw)]
+#![allow(rw::format_error)] // test code
+
 use std::io;
 
 use risingwave_sqlparser::parser::Parser;
@@ -12,5 +28,14 @@ fn main() {
     let mut buffer = String::new();
     io::stdin().read_line(&mut buffer).unwrap();
     let result = Parser::parse_sql(&buffer);
-    println!("{:#?}", result);
+    match result {
+        Ok(statements) => {
+            for statement in statements {
+                println!("{:#?}", statement);
+            }
+        }
+        Err(e) => {
+            eprintln!("{}", e);
+        }
+    }
 }
diff --git a/src/sqlparser/src/keywords.rs b/src/sqlparser/src/keywords.rs
index 5647fde58e4e8..fc5971bf4640b 100644
--- a/src/sqlparser/src/keywords.rs
+++ b/src/sqlparser/src/keywords.rs
@@ -12,7 +12,7 @@
 
 //! This module defines
 //! 1) a list of constants for every keyword that
-//! can appear in [crate::tokenizer::Word::keyword]:
+//!    can appear in [crate::tokenizer::Word::keyword]:
 //!    pub const KEYWORD = "KEYWORD"
 //! 2) an `ALL_KEYWORDS` array with every keyword in it
 //!     This is not a list of *reserved* keywords: some of these can be
@@ -22,7 +22,7 @@
 //!     As a matter of fact, most of these keywords are not used at all
 //!     and could be removed.
 //! 3) a `RESERVED_FOR_TABLE_ALIAS` array with keywords reserved in a
-//! "table alias" context.
+//!    "table alias" context.
 
 use core::fmt;
 
@@ -59,7 +59,7 @@ macro_rules! define_keywords {
         ];
 
         $(kw_def!($ident $(= $string_keyword)?);)*
-        pub const ALL_KEYWORDS: &[&str] = &[
+        pub const ALL_KEYWORDS: &[&'static str] = &[
             $($ident),*
         ];
     };
@@ -248,6 +248,7 @@ define_keywords!(
     FUNCTION,
     FUNCTIONS,
     FUSION,
+    GAP,
     GENERATOR,
     GET,
     GLOBAL,
@@ -411,9 +412,7 @@ define_keywords!(
     REFERENCES,
     REFERENCING,
     REFRESH,
-    REGCLASS,
     REGISTRY,
-    REGPROC,
     REGR_AVGX,
     REGR_AVGY,
     REGR_COUNT,
@@ -450,6 +449,8 @@ define_keywords!(
     SCROLL,
     SEARCH,
     SECOND,
+    SECRET,
+    SECRETS,
     SELECT,
     SENSITIVE,
     SEQUENCE,
@@ -487,6 +488,7 @@ define_keywords!(
     STDDEV_SAMP,
     STDIN,
     STORED,
+    STREAMING_RATE_LIMIT,
     STRING,
     STRUCT,
     SUBMULTISET,
@@ -650,7 +652,7 @@ pub const RESERVED_FOR_COLUMN_ALIAS: &[Keyword] = &[
 /// Can't be used as a column or table name in PostgreSQL.
 ///
 /// This list is taken from the following table, for all "reserved" words in the PostgreSQL column,
-/// includinhg "can be function or type" and "requires AS". <https://www.postgresql.org/docs/14/sql-keywords-appendix.html#KEYWORDS-TABLE>
+/// including "can be function or type" and "requires AS". <https://www.postgresql.org/docs/14/sql-keywords-appendix.html#KEYWORDS-TABLE>
 ///
 /// `SELECT` and `WITH` were commented out because the following won't parse:
 /// `SELECT (SELECT 1)` or `SELECT (WITH a AS (SELECT 1) SELECT 1)`
diff --git a/src/sqlparser/src/lib.rs b/src/sqlparser/src/lib.rs
index 612b11078eac4..a102e5428edae 100644
--- a/src/sqlparser/src/lib.rs
+++ b/src/sqlparser/src/lib.rs
@@ -45,6 +45,7 @@ extern crate alloc;
 pub mod ast;
 pub mod keywords;
 pub mod parser;
+pub mod parser_v2;
 pub mod tokenizer;
 
 #[doc(hidden)]
diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs
index 2cc74c8d0a392..dbc79542949d2 100644
--- a/src/sqlparser/src/parser.rs
+++ b/src/sqlparser/src/parser.rs
@@ -24,14 +24,13 @@ use core::fmt;
 
 use itertools::Itertools;
 use tracing::{debug, instrument};
+use winnow::combinator::{alt, cut_err, dispatch, fail, opt, peek, preceded, repeat, separated};
+use winnow::{PResult, Parser as _};
 
-use self::ddl::AlterSubscriptionOperation;
-use crate::ast::ddl::{
-    AlterConnectionOperation, AlterDatabaseOperation, AlterFunctionOperation, AlterIndexOperation,
-    AlterSchemaOperation, AlterSinkOperation, AlterViewOperation, SourceWatermark,
-};
-use crate::ast::{ParseTo, *};
+use crate::ast::*;
 use crate::keywords::{self, Keyword};
+use crate::parser_v2;
+use crate::parser_v2::{keyword, literal_i64, literal_uint, single_quoted_string, ParserExt as _};
 use crate::tokenizer::*;
 
 pub(crate) const UPSTREAM_SOURCE_KEY: &str = "connector";
@@ -49,14 +48,33 @@ impl ParserError {
         }
     }
 }
+
+#[derive(Debug, thiserror::Error)]
+#[error("{0}")]
+pub struct StrError(pub String);
+
 // Use `Parser::expected` instead, if possible
 #[macro_export]
 macro_rules! parser_err {
-    ($MSG:expr) => {
-        Err(ParserError::ParserError($MSG.to_string()))
+    ($($arg:tt)*) => {
+        return Err(winnow::error::ErrMode::Backtrack(<winnow::error::ContextError as winnow::error::FromExternalError<_, _>>::from_external_error(
+            &Parser::default(),
+            winnow::error::ErrorKind::Fail,
+            $crate::parser::StrError(format!($($arg)*)),
+        )))
     };
 }
 
+impl From<StrError> for winnow::error::ErrMode<winnow::error::ContextError> {
+    fn from(e: StrError) -> Self {
+        winnow::error::ErrMode::Backtrack(<winnow::error::ContextError as winnow::error::FromExternalError<_, _>>::from_external_error(
+            &Parser::default(),
+            winnow::error::ErrorKind::Fail,
+            e,
+        ))
+    }
+}
+
 // Returns a successful result if the optional expression is some
 macro_rules! return_ok_if_some {
     ($e:expr) => {{
@@ -152,7 +170,7 @@ type ColumnsDefTuple = (
 
 /// Reference:
 /// <https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-PRECEDENCE>
-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub enum Precedence {
     Zero = 0,
     LogicalOr, // 5 in upstream
@@ -173,55 +191,71 @@ pub enum Precedence {
     DoubleColon, // 50 in upstream
 }
 
-pub struct Parser {
-    tokens: Vec<TokenWithLocation>,
-    /// The index of the first unprocessed token in `self.tokens`
-    index: usize,
-    /// Since we cannot distinguish `>>` and double `>`, so use `angle_brackets_num` to store the
-    /// number of `<` to match `>` in sql like `struct<v1 struct<v2 int>>`.
-    angle_brackets_num: i32,
-    /// It's important that already in named Array or not. so use this field check in or not.
-    /// Consider 0 is you're not in named Array. if more than 0 is you're in named Array
-    array_depth: usize,
-    /// We cannot know current array should be keep named or not, so by using this field store
-    /// every depth of array that should be keep named or not.
-    array_named_stack: Vec<bool>,
-}
-
-impl Parser {
-    /// Parse the specified tokens
-    pub fn new(tokens: Vec<TokenWithLocation>) -> Self {
-        Parser {
-            tokens,
-            index: 0,
-            angle_brackets_num: 0,
-            array_depth: 0,
-            array_named_stack: Vec::new(),
-        }
-    }
+#[derive(Clone, Copy, Default)]
+pub struct Parser<'a>(pub(crate) &'a [TokenWithLocation]);
 
+impl Parser<'_> {
     /// Parse a SQL statement and produce an Abstract Syntax Tree (AST)
     #[instrument(level = "debug")]
     pub fn parse_sql(sql: &str) -> Result<Vec<Statement>, ParserError> {
         let mut tokenizer = Tokenizer::new(sql);
         let tokens = tokenizer.tokenize_with_location()?;
-        let mut parser = Parser::new(tokens);
+        let parser = Parser(&tokens);
+        let stmts = Parser::parse_statements.parse(parser).map_err(|e| {
+            // append SQL context to the error message, e.g.:
+            // LINE 1: SELECT 1::int(2);
+            let loc = match tokens.get(e.offset()) {
+                Some(token) => token.location.clone(),
+                None => {
+                    // get location of EOF
+                    Location {
+                        line: sql.lines().count() as u64,
+                        column: sql.lines().last().map_or(0, |l| l.len() as u64) + 1,
+                    }
+                }
+            };
+            let prefix = format!("LINE {}: ", loc.line);
+            let sql_line = sql.split('\n').nth(loc.line as usize - 1).unwrap();
+            let cursor = " ".repeat(prefix.len() + loc.column as usize - 1);
+            ParserError::ParserError(format!(
+                "{}\n{}{}\n{}^",
+                e.inner().to_string().replace('\n', ": "),
+                prefix,
+                sql_line,
+                cursor
+            ))
+        })?;
+        Ok(stmts)
+    }
+
+    /// Parse object name from a string.
+    pub fn parse_object_name_str(s: &str) -> Result<ObjectName, ParserError> {
+        let mut tokenizer = Tokenizer::new(s);
+        let tokens = tokenizer.tokenize_with_location()?;
+        let parser = Parser(&tokens);
+        Parser::parse_object_name
+            .parse(parser)
+            .map_err(|e| ParserError::ParserError(e.inner().to_string()))
+    }
+
+    /// Parse a list of semicolon-separated statements.
+    fn parse_statements(&mut self) -> PResult<Vec<Statement>> {
         let mut stmts = Vec::new();
         let mut expecting_statement_delimiter = false;
         loop {
             // ignore empty statements (between successive statement delimiters)
-            while parser.consume_token(&Token::SemiColon) {
+            while self.consume_token(&Token::SemiColon) {
                 expecting_statement_delimiter = false;
             }
 
-            if parser.peek_token() == Token::EOF {
+            if self.peek_token() == Token::EOF {
                 break;
             }
             if expecting_statement_delimiter {
-                return parser.expected("end of statement", parser.peek_token());
+                return self.expected("end of statement");
             }
 
-            let statement = parser.parse_statement()?;
+            let statement = self.parse_statement()?;
             stmts.push(statement);
             expecting_statement_delimiter = true;
         }
@@ -231,14 +265,15 @@ impl Parser {
 
     /// Parse a single top-level statement (such as SELECT, INSERT, CREATE, etc.),
     /// stopping before the statement separator, if any.
-    pub fn parse_statement(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_statement(&mut self) -> PResult<Statement> {
+        let checkpoint = *self;
         let token = self.next_token();
         match token.token {
             Token::Word(w) => match w.keyword {
                 Keyword::EXPLAIN => Ok(self.parse_explain()?),
                 Keyword::ANALYZE => Ok(self.parse_analyze()?),
                 Keyword::SELECT | Keyword::WITH | Keyword::VALUES => {
-                    self.prev_token();
+                    *self = checkpoint;
                     Ok(Statement::Query(Box::new(self.parse_query()?)))
                 }
                 Keyword::DECLARE => Ok(self.parse_declare()?),
@@ -285,52 +320,28 @@ impl Parser {
                 Keyword::FLUSH => Ok(Statement::Flush),
                 Keyword::WAIT => Ok(Statement::Wait),
                 Keyword::RECOVER => Ok(Statement::Recover),
-                _ => self.expected(
-                    "an SQL statement",
-                    Token::Word(w).with_location(token.location),
-                ),
+                _ => self.expected_at(checkpoint, "statement"),
             },
             Token::LParen => {
-                self.prev_token();
+                *self = checkpoint;
                 Ok(Statement::Query(Box::new(self.parse_query()?)))
             }
-            unexpected => {
-                self.expected("an SQL statement", unexpected.with_location(token.location))
-            }
+            _ => self.expected_at(checkpoint, "statement"),
         }
     }
 
-    pub fn parse_truncate(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_truncate(&mut self) -> PResult<Statement> {
         let _ = self.parse_keyword(Keyword::TABLE);
         let table_name = self.parse_object_name()?;
         Ok(Statement::Truncate { table_name })
     }
 
-    pub fn parse_analyze(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_analyze(&mut self) -> PResult<Statement> {
         let table_name = self.parse_object_name()?;
 
         Ok(Statement::Analyze { table_name })
     }
 
-    /// Check is enter array expression.
-    pub fn peek_array_depth(&self) -> usize {
-        self.array_depth
-    }
-
-    /// When enter specify ARRAY prefix expression.
-    pub fn increase_array_depth(&mut self, num: usize) {
-        self.array_depth += num;
-    }
-
-    /// When exit specify ARRAY prefix expression.
-    pub fn decrease_array_depth(&mut self, num: usize) {
-        self.array_depth -= num;
-    }
-
-    pub fn is_in_array(&self) -> bool {
-        self.peek_array_depth() > 0
-    }
-
     /// Tries to parse a wildcard expression. If it is not a wildcard, parses an expression.
     ///
     /// A wildcard expression either means:
@@ -339,14 +350,14 @@ impl Parser {
     ///   contain parentheses.
     /// - Selecting all columns from a table. In this case, it is a
     ///   [`WildcardOrExpr::QualifiedWildcard`] or a [`WildcardOrExpr::Wildcard`].
-    pub fn parse_wildcard_or_expr(&mut self) -> Result<WildcardOrExpr, ParserError> {
-        let index = self.index;
+    pub fn parse_wildcard_or_expr(&mut self) -> PResult<WildcardOrExpr> {
+        let checkpoint = *self;
 
         match self.next_token().token {
             Token::Word(w) if self.peek_token() == Token::Period => {
                 // Since there's no parenthesis, `w` must be a column or a table
                 // So what follows must be dot-delimited identifiers, e.g. `a.b.c.*`
-                let wildcard_expr = self.parse_simple_wildcard_expr(index)?;
+                let wildcard_expr = self.parse_simple_wildcard_expr(checkpoint)?;
                 return self.word_concat_wildcard_expr(w.to_ident()?, wildcard_expr);
             }
             Token::Mul => {
@@ -363,14 +374,14 @@ impl Parser {
                     }
                     // Now that we have an expr, what follows must be
                     // dot-delimited identifiers, e.g. `b.c.*` in `(a).b.c.*`
-                    let wildcard_expr = self.parse_simple_wildcard_expr(index)?;
+                    let wildcard_expr = self.parse_simple_wildcard_expr(checkpoint)?;
                     return self.expr_concat_wildcard_expr(expr, wildcard_expr);
                 }
             }
             _ => (),
         };
 
-        self.index = index;
+        *self = checkpoint;
         self.parse_expr().map(WildcardOrExpr::Expr)
     }
 
@@ -379,7 +390,7 @@ impl Parser {
         &mut self,
         ident: Ident,
         simple_wildcard_expr: WildcardOrExpr,
-    ) -> Result<WildcardOrExpr, ParserError> {
+    ) -> PResult<WildcardOrExpr> {
         let mut idents = vec![ident];
         let mut except_cols = vec![];
         match simple_wildcard_expr {
@@ -412,7 +423,7 @@ impl Parser {
         &mut self,
         expr: Expr,
         simple_wildcard_expr: WildcardOrExpr,
-    ) -> Result<WildcardOrExpr, ParserError> {
+    ) -> PResult<WildcardOrExpr> {
         if let WildcardOrExpr::Expr(e) = simple_wildcard_expr {
             return Ok(WildcardOrExpr::Expr(e));
         }
@@ -442,19 +453,13 @@ impl Parser {
         match simple_wildcard_expr {
             WildcardOrExpr::QualifiedWildcard(ids, except) => {
                 if except.is_some() {
-                    return self.expected(
-                        "Expr quantified wildcard does not support except",
-                        self.peek_token(),
-                    );
+                    return self.expected("Expr quantified wildcard does not support except");
                 }
                 idents.extend(ids.0);
             }
             WildcardOrExpr::Wildcard(except) => {
                 if except.is_some() {
-                    return self.expected(
-                        "Expr quantified wildcard does not support except",
-                        self.peek_token(),
-                    );
+                    return self.expected("Expr quantified wildcard does not support except");
                 }
             }
             WildcardOrExpr::ExprQualifiedWildcard(_, _) => unreachable!(),
@@ -466,12 +471,10 @@ impl Parser {
     /// Tries to parses a wildcard expression without any parentheses.
     ///
     /// If wildcard is not found, go back to `index` and parse an expression.
-    pub fn parse_simple_wildcard_expr(
-        &mut self,
-        index: usize,
-    ) -> Result<WildcardOrExpr, ParserError> {
+    pub fn parse_simple_wildcard_expr(&mut self, checkpoint: Self) -> PResult<WildcardOrExpr> {
         let mut id_parts = vec![];
         while self.consume_token(&Token::Period) {
+            let ckpt = *self;
             let token = self.next_token();
             match token.token {
                 Token::Word(w) => id_parts.push(w.to_ident()?),
@@ -483,45 +486,40 @@ impl Parser {
                             ObjectName(id_parts),
                             self.parse_except()?,
                         ))
-                    }
+                    };
                 }
-                unexpected => {
-                    return self.expected(
-                        "an identifier or a '*' after '.'",
-                        unexpected.with_location(token.location),
-                    );
+                _ => {
+                    *self = ckpt;
+                    return self.expected("an identifier or a '*' after '.'");
                 }
             }
         }
-        self.index = index;
+        *self = checkpoint;
         self.parse_expr().map(WildcardOrExpr::Expr)
     }
 
-    pub fn parse_except(&mut self) -> Result<Option<Vec<Expr>>, ParserError> {
+    pub fn parse_except(&mut self) -> PResult<Option<Vec<Expr>>> {
         if !self.parse_keyword(Keyword::EXCEPT) {
             return Ok(None);
         }
         if !self.consume_token(&Token::LParen) {
-            return self.expected("EXCEPT should be followed by (", self.peek_token());
+            return self.expected("EXCEPT should be followed by (");
         }
         let exprs = self.parse_comma_separated(Parser::parse_expr)?;
         if self.consume_token(&Token::RParen) {
             Ok(Some(exprs))
         } else {
-            self.expected(
-                "( should be followed by ) after column names",
-                self.peek_token(),
-            )
+            self.expected("( should be followed by ) after column names")
         }
     }
 
     /// Parse a new expression
-    pub fn parse_expr(&mut self) -> Result<Expr, ParserError> {
+    pub fn parse_expr(&mut self) -> PResult<Expr> {
         self.parse_subexpr(Precedence::Zero)
     }
 
     /// Parse tokens until the precedence changes
-    pub fn parse_subexpr(&mut self, precedence: Precedence) -> Result<Expr, ParserError> {
+    pub fn parse_subexpr(&mut self, precedence: Precedence) -> PResult<Expr> {
         debug!("parsing expr, current token: {:?}", self.peek_token().token);
         let mut expr = self.parse_prefix()?;
         debug!("prefix: {:?}", expr);
@@ -539,7 +537,7 @@ impl Parser {
     }
 
     /// Parse an expression prefix
-    pub fn parse_prefix(&mut self) -> Result<Expr, ParserError> {
+    pub fn parse_prefix(&mut self) -> PResult<Expr> {
         // PostgreSQL allows any string literal to be preceded by a type name, indicating that the
         // string literal represents a literal of that type. Some examples:
         //
@@ -574,11 +572,12 @@ impl Parser {
             }
         }));
 
+        let checkpoint = *self;
         let token = self.next_token();
         let expr = match token.token.clone() {
             Token::Word(w) => match w.keyword {
                 Keyword::TRUE | Keyword::FALSE | Keyword::NULL => {
-                    self.prev_token();
+                    *self = checkpoint;
                     Ok(Expr::Value(self.parse_value()?))
                 }
                 Keyword::CASE => self.parse_case_expr(),
@@ -596,10 +595,6 @@ impl Parser {
                     expr: Box::new(self.parse_subexpr(Precedence::UnaryNot)?),
                 }),
                 Keyword::ROW => self.parse_row_expr(),
-                Keyword::ARRAY if self.peek_token() == Token::LBracket => {
-                    self.expect_token(&Token::LBracket)?;
-                    self.parse_array_expr(true)
-                }
                 Keyword::ARRAY if self.peek_token() == Token::LParen => {
                     // similar to `exists(subquery)`
                     self.expect_token(&Token::LParen)?;
@@ -607,6 +602,7 @@ impl Parser {
                     self.expect_token(&Token::RParen)?;
                     Ok(exists_node)
                 }
+                Keyword::ARRAY if self.peek_token() == Token::LBracket => self.parse_array_expr(),
                 // `LEFT` and `RIGHT` are reserved as identifier but okay as function
                 Keyword::LEFT | Keyword::RIGHT => {
                     self.parse_function(ObjectName(vec![w.to_ident()?]))
@@ -627,7 +623,7 @@ impl Parser {
 
                     // TODO: support `all/any/some(subquery)`.
                     if let Expr::Subquery(_) = &sub {
-                        parser_err!("ANY/SOME/ALL(Subquery) is not implemented")?;
+                        parser_err!("ANY/SOME/ALL(Subquery) is not implemented");
                     }
 
                     Ok(match keyword {
@@ -638,7 +634,7 @@ impl Parser {
                     })
                 }
                 k if keywords::RESERVED_FOR_COLUMN_OR_TABLE_NAME.contains(&k) => {
-                    parser_err!(format!("syntax error at or near {token}"))
+                    parser_err!("syntax error at or near {token}")
                 }
                 // Here `w` is a word, check if it's a part of a multi-part
                 // identifier, a function call, or a simple identifier:
@@ -646,20 +642,18 @@ impl Parser {
                     Token::LParen | Token::Period => {
                         let mut id_parts: Vec<Ident> = vec![w.to_ident()?];
                         while self.consume_token(&Token::Period) {
+                            let ckpt = *self;
                             let token = self.next_token();
                             match token.token {
                                 Token::Word(w) => id_parts.push(w.to_ident()?),
-                                unexpected => {
-                                    return self.expected(
-                                        "an identifier or a '*' after '.'",
-                                        unexpected.with_location(token.location),
-                                    );
+                                _ => {
+                                    *self = ckpt;
+                                    return self.expected("an identifier or a '*' after '.'");
                                 }
                             }
                         }
 
-                        if self.consume_token(&Token::LParen) {
-                            self.prev_token();
+                        if self.peek_token().token == Token::LParen {
                             self.parse_function(ObjectName(id_parts))
                         } else {
                             Ok(Expr::CompoundIdentifier(id_parts))
@@ -669,8 +663,6 @@ impl Parser {
                 },
             }, // End of Token::Word
 
-            Token::LBracket if self.is_in_array() => self.parse_array_expr(false),
-
             tok @ Token::Minus | tok @ Token::Plus => {
                 let op = if tok == Token::Plus {
                     UnaryOperator::Plus
@@ -715,7 +707,7 @@ impl Parser {
             | Token::NationalStringLiteral(_)
             | Token::HexStringLiteral(_)
             | Token::CstyleEscapesString(_) => {
-                self.prev_token();
+                *self = checkpoint;
                 Ok(Expr::Value(self.parse_value()?))
             }
             Token::Parameter(number) => self.parse_param(number),
@@ -729,18 +721,17 @@ impl Parser {
                 })
             }
             Token::LParen => {
-                let expr =
-                    if self.parse_keyword(Keyword::SELECT) || self.parse_keyword(Keyword::WITH) {
-                        self.prev_token();
-                        Expr::Subquery(Box::new(self.parse_query()?))
+                let expr = if matches!(self.peek_token().token, Token::Word(w) if w.keyword == Keyword::SELECT || w.keyword == Keyword::WITH)
+                {
+                    Expr::Subquery(Box::new(self.parse_query()?))
+                } else {
+                    let mut exprs = self.parse_comma_separated(Parser::parse_expr)?;
+                    if exprs.len() == 1 {
+                        Expr::Nested(Box::new(exprs.pop().unwrap()))
                     } else {
-                        let mut exprs = self.parse_comma_separated(Parser::parse_expr)?;
-                        if exprs.len() == 1 {
-                            Expr::Nested(Box::new(exprs.pop().unwrap()))
-                        } else {
-                            Expr::Row(exprs)
-                        }
-                    };
+                        Expr::Row(exprs)
+                    }
+                };
                 self.expect_token(&Token::RParen)?;
                 if self.peek_token() == Token::Period && matches!(expr, Expr::Nested(_)) {
                     self.parse_struct_selection(expr)
@@ -748,7 +739,7 @@ impl Parser {
                     Ok(expr)
                 }
             }
-            unexpected => self.expected("an expression:", unexpected.with_location(token.location)),
+            _ => self.expected_at(checkpoint, "an expression"),
         }?;
 
         if self.parse_keyword(Keyword::COLLATE) {
@@ -761,16 +752,15 @@ impl Parser {
         }
     }
 
-    fn parse_param(&mut self, param: String) -> Result<Expr, ParserError> {
-        Ok(Expr::Parameter {
-            index: param.parse().map_err(|_| {
-                ParserError::ParserError(format!("Parameter symbol has a invalid index {}.", param))
-            })?,
-        })
+    fn parse_param(&mut self, param: String) -> PResult<Expr> {
+        let Ok(index) = param.parse() else {
+            parser_err!("Parameter symbol has a invalid index {}.", param);
+        };
+        Ok(Expr::Parameter { index })
     }
 
     /// Parses a field selection expression. See also [`Expr::FieldIdentifier`].
-    pub fn parse_struct_selection(&mut self, expr: Expr) -> Result<Expr, ParserError> {
+    pub fn parse_struct_selection(&mut self, expr: Expr) -> PResult<Expr> {
         let mut nested_expr = expr;
         // Unwrap parentheses
         while let Expr::Nested(inner) = nested_expr {
@@ -781,35 +771,21 @@ impl Parser {
     }
 
     /// Parses consecutive field identifiers after a period. i.e., `.foo.bar.baz`
-    pub fn parse_fields(&mut self) -> Result<Vec<Ident>, ParserError> {
-        let mut idents = vec![];
-        while self.consume_token(&Token::Period) {
-            let token = self.next_token();
-            match token.token {
-                Token::Word(w) => {
-                    idents.push(w.to_ident()?);
-                }
-                unexpected => {
-                    return self.expected(
-                        "an identifier after '.'",
-                        unexpected.with_location(token.location),
-                    );
-                }
-            }
-        }
-        Ok(idents)
+    pub fn parse_fields(&mut self) -> PResult<Vec<Ident>> {
+        repeat(.., preceded(Token::Period, cut_err(Self::parse_identifier))).parse_next(self)
     }
 
-    pub fn parse_qualified_operator(&mut self) -> Result<QualifiedOperator, ParserError> {
+    pub fn parse_qualified_operator(&mut self) -> PResult<QualifiedOperator> {
         self.expect_token(&Token::LParen)?;
 
+        let checkpoint = *self;
         let schema = match self.parse_identifier_non_reserved() {
             Ok(ident) => {
                 self.expect_token(&Token::Period)?;
                 Some(ident)
             }
             Err(_) => {
-                self.prev_token();
+                *self = checkpoint;
                 None
             }
         };
@@ -825,11 +801,12 @@ impl Parser {
             //
             // To support custom operators and be fully compatible with PostgreSQL later, the
             // tokenizer should also be updated.
+            let checkpoint = *self;
             let token = self.next_token();
             let name = token.token.to_string();
             if !name.trim_matches(OP_CHARS).is_empty() {
-                self.prev_token();
-                return self.expected(&format!("one of {}", OP_CHARS.iter().join(" ")), token);
+                return self
+                    .expected_at(checkpoint, &format!("one of {}", OP_CHARS.iter().join(" ")));
             }
             name
         };
@@ -838,12 +815,12 @@ impl Parser {
         Ok(QualifiedOperator { schema, name })
     }
 
-    pub fn parse_function(&mut self, name: ObjectName) -> Result<Expr, ParserError> {
+    pub fn parse_function(&mut self, name: ObjectName) -> PResult<Expr> {
         self.expect_token(&Token::LParen)?;
         let distinct = self.parse_all_or_distinct()?;
         let (args, order_by, variadic) = self.parse_optional_args()?;
         let over = if self.parse_keyword(Keyword::OVER) {
-            // TBD: support window names (`OVER mywin`) in place of inline specification
+            // TODO: support window names (`OVER mywin`) in place of inline specification
             self.expect_token(&Token::LParen)?;
             let partition_by = if self.parse_keywords(&[Keyword::PARTITION, Keyword::BY]) {
                 // a list of possibly-qualified column names
@@ -886,10 +863,7 @@ impl Parser {
         let within_group = if self.parse_keywords(&[Keyword::WITHIN, Keyword::GROUP]) {
             self.expect_token(&Token::LParen)?;
             self.expect_keywords(&[Keyword::ORDER, Keyword::BY])?;
-            let order_by_parsed = self.parse_comma_separated(Parser::parse_order_by_expr)?;
-            let order_by = order_by_parsed.iter().exactly_one().map_err(|_| {
-                ParserError::ParserError("only one arg in order by is expected here".to_string())
-            })?;
+            let order_by = self.parse_order_by_expr()?;
             self.expect_token(&Token::RParen)?;
             Some(Box::new(order_by.clone()))
         } else {
@@ -908,34 +882,35 @@ impl Parser {
         }))
     }
 
-    pub fn parse_window_frame_units(&mut self) -> Result<WindowFrameUnits, ParserError> {
-        let token = self.next_token();
-        match token.token {
-            Token::Word(w) => match w.keyword {
-                Keyword::ROWS => Ok(WindowFrameUnits::Rows),
-                Keyword::RANGE => Ok(WindowFrameUnits::Range),
-                Keyword::GROUPS => Ok(WindowFrameUnits::Groups),
-                _ => self.expected(
-                    "ROWS, RANGE, GROUPS",
-                    Token::Word(w).with_location(token.location),
-                )?,
-            },
-            unexpected => self.expected(
-                "ROWS, RANGE, GROUPS",
-                unexpected.with_location(token.location),
-            ),
+    pub fn parse_window_frame_units(&mut self) -> PResult<WindowFrameUnits> {
+        dispatch! { peek(keyword);
+            Keyword::ROWS => keyword.value(WindowFrameUnits::Rows),
+            Keyword::RANGE => keyword.value(WindowFrameUnits::Range),
+            Keyword::GROUPS => keyword.value(WindowFrameUnits::Groups),
+            Keyword::SESSION => keyword.value(WindowFrameUnits::Session),
+            _ => fail,
         }
+        .expect("ROWS, RANGE, or GROUPS")
+        .parse_next(self)
     }
 
-    pub fn parse_window_frame(&mut self) -> Result<WindowFrame, ParserError> {
+    pub fn parse_window_frame(&mut self) -> PResult<WindowFrame> {
         let units = self.parse_window_frame_units()?;
-        let (start_bound, end_bound) = if self.parse_keyword(Keyword::BETWEEN) {
-            let start_bound = self.parse_window_frame_bound()?;
+        let bounds = if self.parse_keyword(Keyword::BETWEEN) {
+            // `BETWEEN <frame_start> AND <frame_end>`
+            let start = self.parse_window_frame_bound()?;
             self.expect_keyword(Keyword::AND)?;
-            let end_bound = Some(self.parse_window_frame_bound()?);
-            (start_bound, end_bound)
+            let end = Some(self.parse_window_frame_bound()?);
+            WindowFrameBounds::Bounds { start, end }
+        } else if self.parse_keywords(&[Keyword::WITH, Keyword::GAP]) {
+            // `WITH GAP <gap>`, only for session frames
+            WindowFrameBounds::Gap(Box::new(self.parse_expr()?))
         } else {
-            (self.parse_window_frame_bound()?, None)
+            // `<frame_start>`
+            WindowFrameBounds::Bounds {
+                start: self.parse_window_frame_bound()?,
+                end: None,
+            }
         };
         let exclusion = if self.parse_keyword(Keyword::EXCLUDE) {
             Some(self.parse_window_frame_exclusion()?)
@@ -944,14 +919,13 @@ impl Parser {
         };
         Ok(WindowFrame {
             units,
-            start_bound,
-            end_bound,
+            bounds,
             exclusion,
         })
     }
 
     /// Parse `CURRENT ROW` or `{ <non-negative numeric | datetime | interval> | UNBOUNDED } { PRECEDING | FOLLOWING }`
-    pub fn parse_window_frame_bound(&mut self) -> Result<WindowFrameBound, ParserError> {
+    pub fn parse_window_frame_bound(&mut self) -> PResult<WindowFrameBound> {
         if self.parse_keywords(&[Keyword::CURRENT, Keyword::ROW]) {
             Ok(WindowFrameBound::CurrentRow)
         } else {
@@ -965,12 +939,12 @@ impl Parser {
             } else if self.parse_keyword(Keyword::FOLLOWING) {
                 Ok(WindowFrameBound::Following(rows))
             } else {
-                self.expected("PRECEDING or FOLLOWING", self.peek_token())
+                self.expected("PRECEDING or FOLLOWING")
             }
         }
     }
 
-    pub fn parse_window_frame_exclusion(&mut self) -> Result<WindowFrameExclusion, ParserError> {
+    pub fn parse_window_frame_exclusion(&mut self) -> PResult<WindowFrameExclusion> {
         if self.parse_keywords(&[Keyword::CURRENT, Keyword::ROW]) {
             Ok(WindowFrameExclusion::CurrentRow)
         } else if self.parse_keyword(Keyword::GROUP) {
@@ -980,13 +954,13 @@ impl Parser {
         } else if self.parse_keywords(&[Keyword::NO, Keyword::OTHERS]) {
             Ok(WindowFrameExclusion::NoOthers)
         } else {
-            self.expected("CURRENT ROW, GROUP, TIES, or NO OTHERS", self.peek_token())
+            self.expected("CURRENT ROW, GROUP, TIES, or NO OTHERS")
         }
     }
 
     /// parse a group by expr. a group by expr can be one of group sets, roll up, cube, or simple
     /// expr.
-    fn parse_group_by_expr(&mut self) -> Result<Expr, ParserError> {
+    fn parse_group_by_expr(&mut self) -> PResult<Expr> {
         if self.parse_keywords(&[Keyword::GROUPING, Keyword::SETS]) {
             self.expect_token(&Token::LParen)?;
             let result = self.parse_comma_separated(|p| p.parse_tuple(true, true))?;
@@ -1010,11 +984,7 @@ impl Parser {
     /// parse a tuple with `(` and `)`.
     /// If `lift_singleton` is true, then a singleton tuple is lifted to a tuple of length 1,
     /// otherwise it will fail. If `allow_empty` is true, then an empty tuple is allowed.
-    fn parse_tuple(
-        &mut self,
-        lift_singleton: bool,
-        allow_empty: bool,
-    ) -> Result<Vec<Expr>, ParserError> {
+    fn parse_tuple(&mut self, lift_singleton: bool, allow_empty: bool) -> PResult<Vec<Expr>> {
         if lift_singleton {
             if self.consume_token(&Token::LParen) {
                 let result = if allow_empty && self.consume_token(&Token::RParen) {
@@ -1041,153 +1011,49 @@ impl Parser {
         }
     }
 
-    pub fn parse_case_expr(&mut self) -> Result<Expr, ParserError> {
-        let mut operand = None;
-        if !self.parse_keyword(Keyword::WHEN) {
-            operand = Some(Box::new(self.parse_expr()?));
-            self.expect_keyword(Keyword::WHEN)?;
-        }
-        let mut conditions = vec![];
-        let mut results = vec![];
-        loop {
-            conditions.push(self.parse_expr()?);
-            self.expect_keyword(Keyword::THEN)?;
-            results.push(self.parse_expr()?);
-            if !self.parse_keyword(Keyword::WHEN) {
-                break;
-            }
-        }
-        let else_result = if self.parse_keyword(Keyword::ELSE) {
-            Some(Box::new(self.parse_expr()?))
-        } else {
-            None
-        };
-        self.expect_keyword(Keyword::END)?;
-        Ok(Expr::Case {
-            operand,
-            conditions,
-            results,
-            else_result,
-        })
+    pub fn parse_case_expr(&mut self) -> PResult<Expr> {
+        parser_v2::expr_case(self)
     }
 
     /// Parse a SQL CAST function e.g. `CAST(expr AS FLOAT)`
-    pub fn parse_cast_expr(&mut self) -> Result<Expr, ParserError> {
-        self.expect_token(&Token::LParen)?;
-        let expr = self.parse_expr()?;
-        self.expect_keyword(Keyword::AS)?;
-        let data_type = self.parse_data_type()?;
-        self.expect_token(&Token::RParen)?;
-        Ok(Expr::Cast {
-            expr: Box::new(expr),
-            data_type,
-        })
+    pub fn parse_cast_expr(&mut self) -> PResult<Expr> {
+        parser_v2::expr_cast(self)
     }
 
     /// Parse a SQL TRY_CAST function e.g. `TRY_CAST(expr AS FLOAT)`
-    pub fn parse_try_cast_expr(&mut self) -> Result<Expr, ParserError> {
-        self.expect_token(&Token::LParen)?;
-        let expr = self.parse_expr()?;
-        self.expect_keyword(Keyword::AS)?;
-        let data_type = self.parse_data_type()?;
-        self.expect_token(&Token::RParen)?;
-        Ok(Expr::TryCast {
-            expr: Box::new(expr),
-            data_type,
-        })
+    pub fn parse_try_cast_expr(&mut self) -> PResult<Expr> {
+        parser_v2::expr_try_cast(self)
     }
 
     /// Parse a SQL EXISTS expression e.g. `WHERE EXISTS(SELECT ...)`.
-    pub fn parse_exists_expr(&mut self) -> Result<Expr, ParserError> {
+    pub fn parse_exists_expr(&mut self) -> PResult<Expr> {
         self.expect_token(&Token::LParen)?;
         let exists_node = Expr::Exists(Box::new(self.parse_query()?));
         self.expect_token(&Token::RParen)?;
         Ok(exists_node)
     }
 
-    pub fn parse_extract_expr(&mut self) -> Result<Expr, ParserError> {
-        self.expect_token(&Token::LParen)?;
-        let field = self.parse_date_time_field_in_extract()?;
-        self.expect_keyword(Keyword::FROM)?;
-        let expr = self.parse_expr()?;
-        self.expect_token(&Token::RParen)?;
-        Ok(Expr::Extract {
-            field,
-            expr: Box::new(expr),
-        })
+    pub fn parse_extract_expr(&mut self) -> PResult<Expr> {
+        parser_v2::expr_extract(self)
     }
 
-    pub fn parse_substring_expr(&mut self) -> Result<Expr, ParserError> {
-        // PARSE SUBSTRING (EXPR [FROM 1] [FOR 3])
-        self.expect_token(&Token::LParen)?;
-        let expr = self.parse_expr()?;
-        let mut from_expr = None;
-        if self.parse_keyword(Keyword::FROM) || self.consume_token(&Token::Comma) {
-            from_expr = Some(self.parse_expr()?);
-        }
-
-        let mut to_expr = None;
-        if self.parse_keyword(Keyword::FOR) || self.consume_token(&Token::Comma) {
-            to_expr = Some(self.parse_expr()?);
-        }
-        self.expect_token(&Token::RParen)?;
-
-        Ok(Expr::Substring {
-            expr: Box::new(expr),
-            substring_from: from_expr.map(Box::new),
-            substring_for: to_expr.map(Box::new),
-        })
+    pub fn parse_substring_expr(&mut self) -> PResult<Expr> {
+        parser_v2::expr_substring(self)
     }
 
     /// `POSITION(<expr> IN <expr>)`
-    pub fn parse_position_expr(&mut self) -> Result<Expr, ParserError> {
-        self.expect_token(&Token::LParen)?;
-
-        // Logically `parse_expr`, but limited to those with precedence higher than `BETWEEN`/`IN`,
-        // to avoid conflict with general IN operator, for example `position(a IN (b) IN (c))`.
-        // https://github.com/postgres/postgres/blob/REL_15_2/src/backend/parser/gram.y#L16012
-        let substring = self.parse_subexpr(Precedence::Between)?;
-        self.expect_keyword(Keyword::IN)?;
-        let string = self.parse_subexpr(Precedence::Between)?;
-
-        self.expect_token(&Token::RParen)?;
-
-        Ok(Expr::Position {
-            substring: Box::new(substring),
-            string: Box::new(string),
-        })
+    pub fn parse_position_expr(&mut self) -> PResult<Expr> {
+        parser_v2::expr_position(self)
     }
 
     /// `OVERLAY(<expr> PLACING <expr> FROM <expr> [ FOR <expr> ])`
-    pub fn parse_overlay_expr(&mut self) -> Result<Expr, ParserError> {
-        self.expect_token(&Token::LParen)?;
-
-        let expr = self.parse_expr()?;
-
-        self.expect_keyword(Keyword::PLACING)?;
-        let new_substring = self.parse_expr()?;
-
-        self.expect_keyword(Keyword::FROM)?;
-        let start = self.parse_expr()?;
-
-        let mut count = None;
-        if self.parse_keyword(Keyword::FOR) {
-            count = Some(self.parse_expr()?);
-        }
-
-        self.expect_token(&Token::RParen)?;
-
-        Ok(Expr::Overlay {
-            expr: Box::new(expr),
-            new_substring: Box::new(new_substring),
-            start: Box::new(start),
-            count: count.map(Box::new),
-        })
+    pub fn parse_overlay_expr(&mut self) -> PResult<Expr> {
+        parser_v2::expr_overlay(self)
     }
 
     /// `TRIM ([WHERE] ['text'] FROM 'text')`\
     /// `TRIM ([WHERE] [FROM] 'text' [, 'text'])`
-    pub fn parse_trim_expr(&mut self) -> Result<Expr, ParserError> {
+    pub fn parse_trim_expr(&mut self) -> PResult<Expr> {
         self.expect_token(&Token::LParen)?;
         let mut trim_where = None;
         if let Token::Word(word) = self.peek_token().token {
@@ -1223,87 +1089,77 @@ impl Parser {
         })
     }
 
-    pub fn parse_trim_where(&mut self) -> Result<TrimWhereField, ParserError> {
-        let token = self.next_token();
-        match token.token {
-            Token::Word(w) => match w.keyword {
-                Keyword::BOTH => Ok(TrimWhereField::Both),
-                Keyword::LEADING => Ok(TrimWhereField::Leading),
-                Keyword::TRAILING => Ok(TrimWhereField::Trailing),
-                _ => self.expected(
-                    "trim_where field",
-                    Token::Word(w).with_location(token.location),
-                )?,
-            },
-            unexpected => {
-                self.expected("trim_where field", unexpected.with_location(token.location))
-            }
+    pub fn parse_trim_where(&mut self) -> PResult<TrimWhereField> {
+        dispatch! { peek(keyword);
+            Keyword::BOTH => keyword.value(TrimWhereField::Both),
+            Keyword::LEADING => keyword.value(TrimWhereField::Leading),
+            Keyword::TRAILING => keyword.value(TrimWhereField::Trailing),
+            _ => fail
         }
+        .expect("BOTH, LEADING, or TRAILING")
+        .parse_next(self)
     }
 
     /// Parses an array expression `[ex1, ex2, ..]`
-    /// if `named` is `true`, came from an expression like  `ARRAY[ex1, ex2]`
-    pub fn parse_array_expr(&mut self, named: bool) -> Result<Expr, ParserError> {
-        self.increase_array_depth(1);
-        if self.array_named_stack.len() < self.peek_array_depth() {
-            self.array_named_stack.push(named);
-        } else if let Err(parse_err) = self.check_same_named_array(named) {
-            Err(parse_err)?
-        }
+    pub fn parse_array_expr(&mut self) -> PResult<Expr> {
+        let mut expected_depth = None;
+        let exprs = self.parse_array_inner(0, &mut expected_depth)?;
+        Ok(Expr::Array(Array {
+            elem: exprs,
+            // Top-level array is named.
+            named: true,
+        }))
+    }
 
-        if self.peek_token() == Token::RBracket {
-            let _ = self.next_token(); // consume ]
-            self.decrease_array_depth(1);
-            Ok(Expr::Array(Array {
-                elem: vec![],
-                named,
-            }))
+    fn parse_array_inner(
+        &mut self,
+        depth: usize,
+        expected_depth: &mut Option<usize>,
+    ) -> PResult<Vec<Expr>> {
+        self.expect_token(&Token::LBracket)?;
+        if let Some(expected_depth) = *expected_depth
+            && depth > expected_depth
+        {
+            return self.expected("]");
+        }
+        let exprs = if self.peek_token() == Token::LBracket {
+            self.parse_comma_separated(|parser| {
+                let exprs = parser.parse_array_inner(depth + 1, expected_depth)?;
+                Ok(Expr::Array(Array {
+                    elem: exprs,
+                    named: false,
+                }))
+            })?
         } else {
-            let exprs = self.parse_comma_separated(Parser::parse_expr)?;
-            self.expect_token(&Token::RBracket)?;
-            if self.array_named_stack.len() > self.peek_array_depth() {
-                self.array_named_stack.pop();
+            if let Some(expected_depth) = *expected_depth {
+                if depth < expected_depth {
+                    return self.expected("[");
+                }
+            } else {
+                *expected_depth = Some(depth);
             }
-            self.decrease_array_depth(1);
-            Ok(Expr::Array(Array { elem: exprs, named }))
-        }
-    }
-
-    fn check_same_named_array(&mut self, current_named: bool) -> Result<(), ParserError> {
-        let previous_named = self.array_named_stack.last().unwrap();
-        if current_named != *previous_named {
-            // for '['
-            self.prev_token();
-            if current_named {
-                // for keyword 'array'
-                self.prev_token();
+            if self.consume_token(&Token::RBracket) {
+                return Ok(vec![]);
             }
-            parser_err!(format!("syntax error at or near {}", self.peek_token()))?
-        } else {
-            Ok(())
-        }
+            self.parse_comma_separated(Self::parse_expr)?
+        };
+        self.expect_token(&Token::RBracket)?;
+        Ok(exprs)
     }
 
     // This function parses date/time fields for interval qualifiers.
-    pub fn parse_date_time_field(&mut self) -> Result<DateTimeField, ParserError> {
-        let token = self.next_token();
-        match token.token {
-            Token::Word(w) => match w.keyword {
-                Keyword::YEAR => Ok(DateTimeField::Year),
-                Keyword::MONTH => Ok(DateTimeField::Month),
-                Keyword::DAY => Ok(DateTimeField::Day),
-                Keyword::HOUR => Ok(DateTimeField::Hour),
-                Keyword::MINUTE => Ok(DateTimeField::Minute),
-                Keyword::SECOND => Ok(DateTimeField::Second),
-                _ => self.expected(
-                    "date/time field",
-                    Token::Word(w).with_location(token.location),
-                )?,
-            },
-            unexpected => {
-                self.expected("date/time field", unexpected.with_location(token.location))
-            }
+    pub fn parse_date_time_field(&mut self) -> PResult<DateTimeField> {
+        dispatch! { peek(keyword);
+            Keyword::YEAR => keyword.value(DateTimeField::Year),
+            Keyword::MONTH => keyword.value(DateTimeField::Month),
+            Keyword::DAY => keyword.value(DateTimeField::Day),
+            Keyword::HOUR => keyword.value(DateTimeField::Hour),
+            Keyword::MINUTE => keyword.value(DateTimeField::Minute),
+            Keyword::SECOND => keyword.value(DateTimeField::Second),
+            _ => fail,
         }
+        .expect("date/time field")
+        .parse_next(self)
     }
 
     // This function parses date/time fields for the EXTRACT function-like operator. PostgreSQL
@@ -1315,13 +1171,15 @@ impl Parser {
     //   select extract("invaLId" from null::date);
     //   select extract('invaLId' from null::date);
     // ```
-    pub fn parse_date_time_field_in_extract(&mut self) -> Result<String, ParserError> {
+    pub fn parse_date_time_field_in_extract(&mut self) -> PResult<String> {
+        let checkpoint = *self;
         let token = self.next_token();
         match token.token {
             Token::Word(w) => Ok(w.value.to_uppercase()),
             Token::SingleQuotedString(s) => Ok(s.to_uppercase()),
-            unexpected => {
-                self.expected("date/time field", unexpected.with_location(token.location))
+            _ => {
+                *self = checkpoint;
+                self.expected("date/time field")
             }
         }
     }
@@ -1338,7 +1196,7 @@ impl Parser {
     ///   6. `INTERVAL '1:1' HOUR (5) TO MINUTE (5)`
     ///
     /// Note that we do not currently attempt to parse the quoted value.
-    pub fn parse_literal_interval(&mut self) -> Result<Expr, ParserError> {
+    pub fn parse_literal_interval(&mut self) -> PResult<Expr> {
         // The SQL standard allows an optional sign before the value string, but
         // it is not clear if any implementations support that syntax, so we
         // don't currently try to parse it. (The sign can instead be included
@@ -1406,7 +1264,8 @@ impl Parser {
     }
 
     /// Parse an operator following an expression
-    pub fn parse_infix(&mut self, expr: Expr, precedence: Precedence) -> Result<Expr, ParserError> {
+    pub fn parse_infix(&mut self, expr: Expr, precedence: Precedence) -> PResult<Expr> {
+        let checkpoint = *self;
         let tok = self.next_token();
         debug!("parsing infix {:?}", tok.token);
         let regular_binary_operator = match &tok.token {
@@ -1476,7 +1335,7 @@ impl Parser {
 
             //     // TODO: support `all/any/some(subquery)`.
             //     if let Expr::Subquery(_) = &right {
-            //         parser_err!("ANY/SOME/ALL(Subquery) is not implemented")?;
+            //         parser_err!("ANY/SOME/ALL(Subquery) is not implemented");
             //     }
 
             //     let right = match keyword {
@@ -1532,27 +1391,20 @@ impl Parser {
                         } else {
                             self.expected(
                                 "[NOT] { TRUE | FALSE | UNKNOWN | NULL | DISTINCT FROM | JSON } after IS",
-                                self.peek_token(),
                             )
                         }
                     }
                 }
                 Keyword::AT => {
-                    if self.parse_keywords(&[Keyword::TIME, Keyword::ZONE]) {
-                        let token = self.next_token();
-                        match token.token {
-                            Token::SingleQuotedString(time_zone) => Ok(Expr::AtTimeZone {
-                                timestamp: Box::new(expr),
-                                time_zone,
-                            }),
-                            unexpected => self.expected(
-                                "Expected Token::SingleQuotedString after AT TIME ZONE",
-                                unexpected.with_location(token.location),
-                            ),
-                        }
-                    } else {
-                        self.expected("Expected Token::Word after AT", tok)
-                    }
+                    let time_zone = preceded(
+                        (Keyword::TIME, Keyword::ZONE),
+                        cut_err(Self::parse_literal_string),
+                    )
+                    .parse_next(self)?;
+                    Ok(Expr::AtTimeZone {
+                        timestamp: Box::new(expr),
+                        time_zone,
+                    })
                 }
                 keyword @ (Keyword::ALL | Keyword::ANY | Keyword::SOME) => {
                     self.expect_token(&Token::LParen)?;
@@ -1563,7 +1415,7 @@ impl Parser {
 
                     // TODO: support `all/any/some(subquery)`.
                     if let Expr::Subquery(_) = &sub {
-                        parser_err!("ANY/SOME/ALL(Subquery) is not implemented")?;
+                        parser_err!("ANY/SOME/ALL(Subquery) is not implemented");
                     }
 
                     Ok(match keyword {
@@ -1579,7 +1431,7 @@ impl Parser {
                 | Keyword::LIKE
                 | Keyword::ILIKE
                 | Keyword::SIMILAR => {
-                    self.prev_token();
+                    *self = checkpoint;
                     let negated = self.parse_keyword(Keyword::NOT);
                     if self.parse_keyword(Keyword::IN) {
                         self.parse_in(expr, negated)
@@ -1607,11 +1459,11 @@ impl Parser {
                             escape_char: self.parse_escape()?,
                         })
                     } else {
-                        self.expected("IN, BETWEEN or SIMILAR TO after NOT", self.peek_token())
+                        self.expected("IN, BETWEEN or SIMILAR TO after NOT")
                     }
                 }
                 // Can only happen if `get_next_precedence` got out of sync with this function
-                _ => parser_err!(format!("No infix parser for token {:?}", tok)),
+                _ => parser_err!("No infix parser for token {:?}", tok),
             }
         } else if Token::DoubleColon == tok {
             self.parse_pg_cast(expr)
@@ -1625,20 +1477,18 @@ impl Parser {
             self.parse_array_index(expr)
         } else {
             // Can only happen if `get_next_precedence` got out of sync with this function
-            parser_err!(format!("No infix parser for token {:?}", tok))
+            parser_err!("No infix parser for token {:?}", tok)
         }
     }
 
     /// parse the ESCAPE CHAR portion of LIKE, ILIKE, and SIMILAR TO
-    pub fn parse_escape(&mut self) -> Result<Option<EscapeChar>, ParserError> {
+    pub fn parse_escape(&mut self) -> PResult<Option<EscapeChar>> {
         if self.parse_keyword(Keyword::ESCAPE) {
             let s = self.parse_literal_string()?;
             let mut chs = s.chars();
             if let Some(ch) = chs.next() {
                 if chs.next().is_some() {
-                    parser_err!(format!(
-                        "Escape string must be empty or one character, found {s:?}"
-                    ))
+                    parser_err!("Escape string must be empty or one character, found {s:?}")
                 } else {
                     Ok(Some(EscapeChar::escape(ch)))
                 }
@@ -1652,7 +1502,7 @@ impl Parser {
 
     /// We parse both `array[1,9][1]`, `array[1,9][1:2]`, `array[1,9][:2]`, `array[1,9][1:]` and
     /// `array[1,9][:]` in this function.
-    pub fn parse_array_index(&mut self, expr: Expr) -> Result<Expr, ParserError> {
+    pub fn parse_array_index(&mut self, expr: Expr) -> PResult<Expr> {
         let new_expr = match self.peek_token().token {
             Token::Colon => {
                 // [:] or [:N]
@@ -1717,7 +1567,7 @@ impl Parser {
     }
 
     /// Parses the optional constraints following the `IS [NOT] JSON` predicate
-    pub fn parse_is_json(&mut self, expr: Expr, negated: bool) -> Result<Expr, ParserError> {
+    pub fn parse_is_json(&mut self, expr: Expr, negated: bool) -> PResult<Expr> {
         let item_type = match self.peek_token().token {
             Token::Word(w) => match w.keyword {
                 Keyword::VALUE => Some(JsonPredicateType::Value),
@@ -1749,10 +1599,10 @@ impl Parser {
     }
 
     /// Parses the parens following the `[ NOT ] IN` operator
-    pub fn parse_in(&mut self, expr: Expr, negated: bool) -> Result<Expr, ParserError> {
+    pub fn parse_in(&mut self, expr: Expr, negated: bool) -> PResult<Expr> {
         self.expect_token(&Token::LParen)?;
-        let in_op = if self.parse_keyword(Keyword::SELECT) || self.parse_keyword(Keyword::WITH) {
-            self.prev_token();
+        let in_op = if matches!(self.peek_token().token, Token::Word(w) if w.keyword == Keyword::SELECT || w.keyword == Keyword::WITH)
+        {
             Expr::InSubquery {
                 expr: Box::new(expr),
                 subquery: Box::new(self.parse_query()?),
@@ -1770,7 +1620,7 @@ impl Parser {
     }
 
     /// Parses `BETWEEN <low> AND <high>`, assuming the `BETWEEN` keyword was already consumed
-    pub fn parse_between(&mut self, expr: Expr, negated: bool) -> Result<Expr, ParserError> {
+    pub fn parse_between(&mut self, expr: Expr, negated: bool) -> PResult<Expr> {
         // Stop parsing subexpressions for <low> and <high> on tokens with
         // precedence lower than that of `BETWEEN`, such as `AND`, `IS`, etc.
         let low = self.parse_subexpr(Precedence::Between)?;
@@ -1785,7 +1635,7 @@ impl Parser {
     }
 
     /// Parse a postgresql casting style which is in the form of `expr::datatype`
-    pub fn parse_pg_cast(&mut self, expr: Expr) -> Result<Expr, ParserError> {
+    pub fn parse_pg_cast(&mut self, expr: Expr) -> PResult<Expr> {
         Ok(Expr::Cast {
             expr: Box::new(expr),
             data_type: self.parse_data_type()?,
@@ -1793,7 +1643,7 @@ impl Parser {
     }
 
     /// Get the precedence of the next token
-    pub fn get_next_precedence(&self) -> Result<Precedence, ParserError> {
+    pub fn get_next_precedence(&self) -> PResult<Precedence> {
         use Precedence as P;
 
         let token = self.peek_token();
@@ -1898,17 +1748,15 @@ impl Parser {
 
     /// Return nth non-whitespace token that has not yet been processed
     pub fn peek_nth_token(&self, mut n: usize) -> TokenWithLocation {
-        let mut index = self.index;
+        let mut index = 0;
         loop {
+            let token = self.0.get(index);
             index += 1;
-            let token = self.tokens.get(index - 1);
             match token.map(|x| &x.token) {
                 Some(Token::Whitespace(_)) => continue,
                 _ => {
                     if n == 0 {
-                        return token
-                            .cloned()
-                            .unwrap_or(TokenWithLocation::wrap(Token::EOF));
+                        return token.cloned().unwrap_or(TokenWithLocation::eof());
                     }
                     n -= 1;
                 }
@@ -1921,61 +1769,37 @@ impl Parser {
     /// repeatedly after reaching EOF.
     pub fn next_token(&mut self) -> TokenWithLocation {
         loop {
-            self.index += 1;
-            let token = self.tokens.get(self.index - 1);
-            match token.map(|x| &x.token) {
-                Some(Token::Whitespace(_)) => continue,
-                _ => {
-                    return token
-                        .cloned()
-                        .unwrap_or(TokenWithLocation::wrap(Token::EOF))
-                }
+            let Some(token) = self.0.first() else {
+                return TokenWithLocation::eof();
+            };
+            self.0 = &self.0[1..];
+            match token.token {
+                Token::Whitespace(_) => continue,
+                _ => return token.clone(),
             }
         }
     }
 
     /// Return the first unprocessed token, possibly whitespace.
     pub fn next_token_no_skip(&mut self) -> Option<&TokenWithLocation> {
-        self.index += 1;
-        self.tokens.get(self.index - 1)
+        if self.0.is_empty() {
+            None
+        } else {
+            let (first, rest) = self.0.split_at(1);
+            self.0 = rest;
+            Some(&first[0])
+        }
     }
 
-    /// Push back the last one non-whitespace token. Must be called after
-    /// `next_token()`, otherwise might panic. OK to call after
-    /// `next_token()` indicates an EOF.
-    pub fn prev_token(&mut self) {
-        loop {
-            assert!(self.index > 0);
-            self.index -= 1;
-            if let Some(token) = self.tokens.get(self.index)
-                && let Token::Whitespace(_) = token.token
-            {
-                continue;
-            }
-            return;
-        }
+    /// Report an expected error at the current position.
+    pub fn expected<T>(&self, expected: &str) -> PResult<T> {
+        parser_err!("expected {}, found: {}", expected, self.peek_token().token)
     }
 
-    /// Report unexpected token
-    pub fn expected<T>(&self, expected: &str, found: TokenWithLocation) -> Result<T, ParserError> {
-        let start_off = self.index.saturating_sub(10);
-        let end_off = self.index.min(self.tokens.len());
-        let near_tokens = &self.tokens[start_off..end_off];
-        struct TokensDisplay<'a>(&'a [TokenWithLocation]);
-        impl<'a> fmt::Display for TokensDisplay<'a> {
-            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-                for token in self.0 {
-                    write!(f, "{}", token.token)?;
-                }
-                Ok(())
-            }
-        }
-        parser_err!(format!(
-            "Expected {}, found: {}\nNear \"{}\"",
-            expected,
-            found,
-            TokensDisplay(near_tokens),
-        ))
+    /// Revert the parser to a previous position and report an expected error.
+    pub fn expected_at<T>(&mut self, checkpoint: Self, expected: &str) -> PResult<T> {
+        *self = checkpoint;
+        self.expected(expected)
     }
 
     /// Look for an expected keyword and consume it if it exists
@@ -1993,12 +1817,12 @@ impl Parser {
     /// Look for an expected sequence of keywords and consume them if they exist
     #[must_use]
     pub fn parse_keywords(&mut self, keywords: &[Keyword]) -> bool {
-        let index = self.index;
+        let checkpoint = *self;
         for &keyword in keywords {
             if !self.parse_keyword(keyword) {
                 // println!("parse_keywords aborting .. did not find {:?}", keyword);
                 // reset index and return immediately
-                self.index = index;
+                *self = checkpoint;
                 return false;
             }
         }
@@ -2030,30 +1854,27 @@ impl Parser {
     }
 
     /// Bail out if the current token is not one of the expected keywords, or consume it if it is
-    pub fn expect_one_of_keywords(&mut self, keywords: &[Keyword]) -> Result<Keyword, ParserError> {
+    pub fn expect_one_of_keywords(&mut self, keywords: &[Keyword]) -> PResult<Keyword> {
         if let Some(keyword) = self.parse_one_of_keywords(keywords) {
             Ok(keyword)
         } else {
             let keywords: Vec<String> = keywords.iter().map(|x| format!("{:?}", x)).collect();
-            self.expected(
-                &format!("one of {}", keywords.join(" or ")),
-                self.peek_token(),
-            )
+            self.expected(&format!("one of {}", keywords.join(" or ")))
         }
     }
 
     /// Bail out if the current token is not an expected keyword, or consume it if it is
-    pub fn expect_keyword(&mut self, expected: Keyword) -> Result<(), ParserError> {
+    pub fn expect_keyword(&mut self, expected: Keyword) -> PResult<()> {
         if self.parse_keyword(expected) {
             Ok(())
         } else {
-            self.expected(format!("{:?}", &expected).as_str(), self.peek_token())
+            self.expected(format!("{:?}", &expected).as_str())
         }
     }
 
     /// Bail out if the following tokens are not the expected sequence of
     /// keywords, or consume them if they are.
-    pub fn expect_keywords(&mut self, expected: &[Keyword]) -> Result<(), ParserError> {
+    pub fn expect_keywords(&mut self, expected: &[Keyword]) -> PResult<()> {
         for &kw in expected {
             self.expect_keyword(kw)?;
         }
@@ -2072,18 +1893,18 @@ impl Parser {
     }
 
     /// Bail out if the current token is not an expected keyword, or consume it if it is
-    pub fn expect_token(&mut self, expected: &Token) -> Result<(), ParserError> {
+    pub fn expect_token(&mut self, expected: &Token) -> PResult<()> {
         if self.consume_token(expected) {
             Ok(())
         } else {
-            self.expected(&expected.to_string(), self.peek_token())
+            self.expected(&expected.to_string())
         }
     }
 
     /// Parse a comma-separated list of 1+ items accepted by `F`
-    pub fn parse_comma_separated<T, F>(&mut self, mut f: F) -> Result<Vec<T>, ParserError>
+    pub fn parse_comma_separated<T, F>(&mut self, mut f: F) -> PResult<Vec<T>>
     where
-        F: FnMut(&mut Parser) -> Result<T, ParserError>,
+        F: FnMut(&mut Self) -> PResult<T>,
     {
         let mut values = vec![];
         loop {
@@ -2100,31 +1921,31 @@ impl Parser {
     #[must_use]
     fn maybe_parse<T, F>(&mut self, mut f: F) -> Option<T>
     where
-        F: FnMut(&mut Parser) -> Result<T, ParserError>,
+        F: FnMut(&mut Self) -> PResult<T>,
     {
-        let index = self.index;
+        let checkpoint = *self;
         if let Ok(t) = f(self) {
             Some(t)
         } else {
-            self.index = index;
+            *self = checkpoint;
             None
         }
     }
 
     /// Parse either `ALL` or `DISTINCT`. Returns `true` if `DISTINCT` is parsed and results in a
     /// `ParserError` if both `ALL` and `DISTINCT` are fround.
-    pub fn parse_all_or_distinct(&mut self) -> Result<bool, ParserError> {
+    pub fn parse_all_or_distinct(&mut self) -> PResult<bool> {
         let all = self.parse_keyword(Keyword::ALL);
         let distinct = self.parse_keyword(Keyword::DISTINCT);
         if all && distinct {
-            parser_err!("Cannot specify both ALL and DISTINCT".to_string())
+            parser_err!("Cannot specify both ALL and DISTINCT")
         } else {
             Ok(distinct)
         }
     }
 
     /// Parse either `ALL` or `DISTINCT` or `DISTINCT ON (<expr>)`.
-    pub fn parse_all_or_distinct_on(&mut self) -> Result<Distinct, ParserError> {
+    pub fn parse_all_or_distinct_on(&mut self) -> PResult<Distinct> {
         if self.parse_keywords(&[Keyword::DISTINCT, Keyword::ON]) {
             self.expect_token(&Token::LParen)?;
             let exprs = self.parse_comma_separated(Parser::parse_expr)?;
@@ -2138,7 +1959,7 @@ impl Parser {
     }
 
     /// Parse a SQL CREATE statement
-    pub fn parse_create(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_create(&mut self) -> PResult<Statement> {
         let or_replace = self.parse_keywords(&[Keyword::OR, Keyword::REPLACE]);
         let temporary = self
             .parse_one_of_keywords(&[Keyword::TEMP, Keyword::TEMPORARY])
@@ -2166,7 +1987,6 @@ impl Parser {
         } else if or_replace {
             self.expected(
                 "[EXTERNAL] TABLE or [MATERIALIZED] VIEW or [MATERIALIZED] SOURCE or SINK or FUNCTION after CREATE OR REPLACE",
-                self.peek_token(),
             )
         } else if self.parse_keyword(Keyword::INDEX) {
             self.parse_create_index(false)
@@ -2178,21 +1998,35 @@ impl Parser {
             self.parse_create_database()
         } else if self.parse_keyword(Keyword::USER) {
             self.parse_create_user()
+        } else if self.parse_keyword(Keyword::SECRET) {
+            self.parse_create_secret()
         } else {
-            self.expected("an object type after CREATE", self.peek_token())
+            self.expected("an object type after CREATE")
         }
     }
 
-    pub fn parse_create_schema(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_create_schema(&mut self) -> PResult<Statement> {
         let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]);
-        let schema_name = self.parse_object_name()?;
+        let (schema_name, user_specified) = if self.parse_keyword(Keyword::AUTHORIZATION) {
+            let user_specified = self.parse_object_name()?;
+            (user_specified.clone(), Some(user_specified))
+        } else {
+            let schema_name = self.parse_object_name()?;
+            let user_specified = if self.parse_keyword(Keyword::AUTHORIZATION) {
+                Some(self.parse_object_name()?)
+            } else {
+                None
+            };
+            (schema_name, user_specified)
+        };
         Ok(Statement::CreateSchema {
             schema_name,
             if_not_exists,
+            user_specified,
         })
     }
 
-    pub fn parse_create_database(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_create_database(&mut self) -> PResult<Statement> {
         let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]);
         let db_name = self.parse_object_name()?;
         Ok(Statement::CreateDatabase {
@@ -2205,7 +2039,7 @@ impl Parser {
         &mut self,
         materialized: bool,
         or_replace: bool,
-    ) -> Result<Statement, ParserError> {
+    ) -> PResult<Statement> {
         let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]);
         // Many dialects support `OR ALTER` right after `CREATE`, but we don't (yet).
         // ANSI SQL and Postgres support RECURSIVE here, but we don't support it either.
@@ -2240,7 +2074,7 @@ impl Parser {
     // [WITH (properties)]?
     // ROW FORMAT <row_format: Ident>
     // [ROW SCHEMA LOCATION <row_schema_location: String>]?
-    pub fn parse_create_source(&mut self, _or_replace: bool) -> Result<Statement, ParserError> {
+    pub fn parse_create_source(&mut self, _or_replace: bool) -> PResult<Statement> {
         Ok(Statement::CreateSource {
             stmt: CreateSourceStatement::parse_to(self)?,
         })
@@ -2253,7 +2087,7 @@ impl Parser {
     // FROM
     // <materialized_view: Ident>
     // [WITH (properties)]?
-    pub fn parse_create_sink(&mut self, _or_replace: bool) -> Result<Statement, ParserError> {
+    pub fn parse_create_sink(&mut self, _or_replace: bool) -> PResult<Statement> {
         Ok(Statement::CreateSink {
             stmt: CreateSinkStatement::parse_to(self)?,
         })
@@ -2266,10 +2100,7 @@ impl Parser {
     // FROM
     // <materialized_view: Ident>
     // [WITH (properties)]?
-    pub fn parse_create_subscription(
-        &mut self,
-        _or_replace: bool,
-    ) -> Result<Statement, ParserError> {
+    pub fn parse_create_subscription(&mut self, _or_replace: bool) -> PResult<Statement> {
         Ok(Statement::CreateSubscription {
             stmt: CreateSubscriptionStatement::parse_to(self)?,
         })
@@ -2280,7 +2111,7 @@ impl Parser {
     // [IF NOT EXISTS]?
     // <connection_name: Ident>
     // [WITH (properties)]?
-    pub fn parse_create_connection(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_create_connection(&mut self) -> PResult<Statement> {
         Ok(Statement::CreateConnection {
             stmt: CreateConnectionStatement::parse_to(self)?,
         })
@@ -2290,11 +2121,10 @@ impl Parser {
         &mut self,
         or_replace: bool,
         temporary: bool,
-    ) -> Result<Statement, ParserError> {
+    ) -> PResult<Statement> {
         let name = self.parse_object_name()?;
         self.expect_token(&Token::LParen)?;
-        let args = if self.consume_token(&Token::RParen) {
-            self.prev_token();
+        let args = if self.peek_token().token == Token::RParen {
             None
         } else {
             Some(self.parse_comma_separated(Parser::parse_function_arg)?)
@@ -2313,7 +2143,7 @@ impl Parser {
                         // allow a trailing comma, even though it's not in standard
                         break;
                     } else if !comma {
-                        return self.expected("',' or ')'", self.peek_token());
+                        return self.expected("',' or ')'");
                     }
                 }
                 Some(CreateFunctionReturns::Table(values))
@@ -2338,17 +2168,14 @@ impl Parser {
         })
     }
 
-    fn parse_create_aggregate(&mut self, or_replace: bool) -> Result<Statement, ParserError> {
+    fn parse_create_aggregate(&mut self, or_replace: bool) -> PResult<Statement> {
         let name = self.parse_object_name()?;
         self.expect_token(&Token::LParen)?;
         let args = self.parse_comma_separated(Parser::parse_function_arg)?;
         self.expect_token(&Token::RParen)?;
 
-        let return_type = if self.parse_keyword(Keyword::RETURNS) {
-            Some(self.parse_data_type()?)
-        } else {
-            None
-        };
+        self.expect_keyword(Keyword::RETURNS)?;
+        let returns = self.parse_data_type()?;
 
         let append_only = self.parse_keywords(&[Keyword::APPEND, Keyword::ONLY]);
         let params = self.parse_create_function_body()?;
@@ -2357,38 +2184,38 @@ impl Parser {
             or_replace,
             name,
             args,
-            returns: return_type,
+            returns,
             append_only,
             params,
         })
     }
 
-    pub fn parse_declare(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_declare(&mut self) -> PResult<Statement> {
         Ok(Statement::DeclareCursor {
             stmt: DeclareCursorStatement::parse_to(self)?,
         })
     }
 
-    pub fn parse_fetch_cursor(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_fetch_cursor(&mut self) -> PResult<Statement> {
         Ok(Statement::FetchCursor {
             stmt: FetchCursorStatement::parse_to(self)?,
         })
     }
 
-    pub fn parse_close_cursor(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_close_cursor(&mut self) -> PResult<Statement> {
         Ok(Statement::CloseCursor {
             stmt: CloseCursorStatement::parse_to(self)?,
         })
     }
 
-    fn parse_table_column_def(&mut self) -> Result<TableColumnDef, ParserError> {
+    fn parse_table_column_def(&mut self) -> PResult<TableColumnDef> {
         Ok(TableColumnDef {
             name: self.parse_identifier_non_reserved()?,
             data_type: self.parse_data_type()?,
         })
     }
 
-    fn parse_function_arg(&mut self) -> Result<OperateFunctionArg, ParserError> {
+    fn parse_function_arg(&mut self) -> PResult<OperateFunctionArg> {
         let mode = if self.parse_keyword(Keyword::IN) {
             Some(ArgMode::In)
         } else if self.parse_keyword(Keyword::OUT) {
@@ -2424,14 +2251,12 @@ impl Parser {
         })
     }
 
-    fn parse_create_function_body(&mut self) -> Result<CreateFunctionBody, ParserError> {
+    fn parse_create_function_body(&mut self) -> PResult<CreateFunctionBody> {
         let mut body = CreateFunctionBody::default();
         loop {
-            fn ensure_not_set<T>(field: &Option<T>, name: &str) -> Result<(), ParserError> {
+            fn ensure_not_set<T>(field: &Option<T>, name: &str) -> PResult<()> {
                 if field.is_some() {
-                    return Err(ParserError::ParserError(format!(
-                        "{name} specified more than once",
-                    )));
+                    parser_err!("{name} specified more than once");
                 }
                 Ok(())
             }
@@ -2443,7 +2268,7 @@ impl Parser {
                 body.language = Some(self.parse_identifier()?);
             } else if self.parse_keyword(Keyword::RUNTIME) {
                 ensure_not_set(&body.runtime, "RUNTIME")?;
-                body.runtime = Some(self.parse_function_runtime()?);
+                body.runtime = Some(self.parse_identifier()?);
             } else if self.parse_keyword(Keyword::IMMUTABLE) {
                 ensure_not_set(&body.behavior, "IMMUTABLE | STABLE | VOLATILE")?;
                 body.behavior = Some(FunctionBehavior::Immutable);
@@ -2474,7 +2299,7 @@ impl Parser {
         }
     }
 
-    fn parse_create_function_using(&mut self) -> Result<CreateFunctionUsing, ParserError> {
+    fn parse_create_function_using(&mut self) -> PResult<CreateFunctionUsing> {
         let keyword = self.expect_one_of_keywords(&[Keyword::LINK, Keyword::BASE64])?;
 
         match keyword {
@@ -2490,22 +2315,11 @@ impl Parser {
         }
     }
 
-    fn parse_function_runtime(&mut self) -> Result<FunctionRuntime, ParserError> {
-        let ident = self.parse_identifier()?;
-        match ident.value.to_lowercase().as_str() {
-            "deno" => Ok(FunctionRuntime::Deno),
-            "quickjs" => Ok(FunctionRuntime::QuickJs),
-            r => Err(ParserError::ParserError(format!(
-                "Unsupported runtime: {r}"
-            ))),
-        }
-    }
-
     fn parse_function_type(
         &mut self,
         is_async: bool,
         is_generator: bool,
-    ) -> Result<CreateFunctionType, ParserError> {
+    ) -> PResult<CreateFunctionType> {
         let is_generator = if is_generator {
             true
         } else {
@@ -2527,25 +2341,32 @@ impl Parser {
     //     | CREATEUSER | NOCREATEUSER
     //     | LOGIN | NOLOGIN
     //     | [ ENCRYPTED ] PASSWORD 'password' | PASSWORD NULL | OAUTH
-    fn parse_create_user(&mut self) -> Result<Statement, ParserError> {
+    fn parse_create_user(&mut self) -> PResult<Statement> {
         Ok(Statement::CreateUser(CreateUserStatement::parse_to(self)?))
     }
 
-    pub fn parse_with_properties(&mut self) -> Result<Vec<SqlOption>, ParserError> {
+    fn parse_create_secret(&mut self) -> PResult<Statement> {
+        Ok(Statement::CreateSecret {
+            stmt: CreateSecretStatement::parse_to(self)?,
+        })
+    }
+
+    pub fn parse_with_properties(&mut self) -> PResult<Vec<SqlOption>> {
         Ok(self
             .parse_options_with_preceding_keyword(Keyword::WITH)?
             .to_vec())
     }
 
-    pub fn parse_discard(&mut self) -> Result<Statement, ParserError> {
-        self.expect_keyword(Keyword::ALL)
-            .map_err(|_| ParserError::ParserError("only DISCARD ALL is supported".to_string()))?;
+    pub fn parse_discard(&mut self) -> PResult<Statement> {
+        self.expect_keyword(Keyword::ALL)?;
         Ok(Statement::Discard(DiscardType::All))
     }
 
-    pub fn parse_drop(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_drop(&mut self) -> PResult<Statement> {
         if self.parse_keyword(Keyword::FUNCTION) {
             return self.parse_drop_function();
+        } else if self.parse_keyword(Keyword::AGGREGATE) {
+            return self.parse_drop_aggregate();
         }
         Ok(Statement::Drop(DropStatement::parse_to(self)?))
     }
@@ -2554,7 +2375,7 @@ impl Parser {
     /// DROP FUNCTION [ IF EXISTS ] name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] [, ...]
     /// [ CASCADE | RESTRICT ]
     /// ```
-    fn parse_drop_function(&mut self) -> Result<Statement, ParserError> {
+    fn parse_drop_function(&mut self) -> PResult<Statement> {
         let if_exists = self.parse_keywords(&[Keyword::IF, Keyword::EXISTS]);
         let func_desc = self.parse_comma_separated(Parser::parse_function_desc)?;
         let option = match self.parse_one_of_keywords(&[Keyword::CASCADE, Keyword::RESTRICT]) {
@@ -2569,7 +2390,26 @@ impl Parser {
         })
     }
 
-    fn parse_function_desc(&mut self) -> Result<FunctionDesc, ParserError> {
+    /// ```sql
+    /// DROP AGGREGATE [ IF EXISTS ] name [ ( [ [ argmode ] [ argname ] argtype [, ...] ] ) ] [, ...]
+    /// [ CASCADE | RESTRICT ]
+    /// ```
+    fn parse_drop_aggregate(&mut self) -> PResult<Statement> {
+        let if_exists = self.parse_keywords(&[Keyword::IF, Keyword::EXISTS]);
+        let func_desc = self.parse_comma_separated(Parser::parse_function_desc)?;
+        let option = match self.parse_one_of_keywords(&[Keyword::CASCADE, Keyword::RESTRICT]) {
+            Some(Keyword::CASCADE) => Some(ReferentialAction::Cascade),
+            Some(Keyword::RESTRICT) => Some(ReferentialAction::Restrict),
+            _ => None,
+        };
+        Ok(Statement::DropAggregate {
+            if_exists,
+            func_desc,
+            option,
+        })
+    }
+
+    fn parse_function_desc(&mut self) -> PResult<FunctionDesc> {
         let name = self.parse_object_name()?;
 
         let args = if self.consume_token(&Token::LParen) {
@@ -2587,7 +2427,7 @@ impl Parser {
         Ok(FunctionDesc { name, args })
     }
 
-    pub fn parse_create_index(&mut self, unique: bool) -> Result<Statement, ParserError> {
+    pub fn parse_create_index(&mut self, unique: bool) -> PResult<Statement> {
         let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]);
         let index_name = self.parse_object_name()?;
         self.expect_keyword(Keyword::ON)?;
@@ -2618,7 +2458,7 @@ impl Parser {
         })
     }
 
-    pub fn parse_with_version_column(&mut self) -> Result<Option<String>, ParserError> {
+    pub fn parse_with_version_column(&mut self) -> PResult<Option<String>> {
         if self.parse_keywords(&[Keyword::WITH, Keyword::VERSION, Keyword::COLUMN]) {
             self.expect_token(&Token::LParen)?;
             let name = self.parse_identifier_non_reserved()?;
@@ -2629,7 +2469,7 @@ impl Parser {
         }
     }
 
-    pub fn parse_on_conflict(&mut self) -> Result<Option<OnConflict>, ParserError> {
+    pub fn parse_on_conflict(&mut self) -> PResult<Option<OnConflict>> {
         if self.parse_keywords(&[Keyword::ON, Keyword::CONFLICT]) {
             self.parse_handle_conflict_behavior()
         } else {
@@ -2637,11 +2477,7 @@ impl Parser {
         }
     }
 
-    pub fn parse_create_table(
-        &mut self,
-        or_replace: bool,
-        temporary: bool,
-    ) -> Result<Statement, ParserError> {
+    pub fn parse_create_table(&mut self, or_replace: bool, temporary: bool) -> PResult<Statement> {
         let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]);
         let table_name = self.parse_object_name()?;
         // parse optional column list (schema) and watermarks on source.
@@ -2676,9 +2512,7 @@ impl Parser {
         // Parse optional `AS ( query )`
         let query = if self.parse_keyword(Keyword::AS) {
             if !source_watermarks.is_empty() {
-                return Err(ParserError::ParserError(
-                    "Watermarks can't be defined on table created by CREATE TABLE AS".to_string(),
-                ));
+                parser_err!("Watermarks can't be defined on table created by CREATE TABLE AS");
             }
             Some(Box::new(self.parse_query()?))
         } else {
@@ -2717,7 +2551,7 @@ impl Parser {
         })
     }
 
-    pub fn parse_include_options(&mut self) -> Result<IncludeOption, ParserError> {
+    pub fn parse_include_options(&mut self) -> PResult<IncludeOption> {
         let mut options = vec![];
         while self.parse_keyword(Keyword::INCLUDE) {
             let column_type = self.parse_identifier()?;
@@ -2761,7 +2595,7 @@ impl Parser {
         Ok(options)
     }
 
-    pub fn parse_columns_with_watermark(&mut self) -> Result<ColumnsDefTuple, ParserError> {
+    pub fn parse_columns_with_watermark(&mut self) -> PResult<ColumnsDefTuple> {
         let mut columns = vec![];
         let mut constraints = vec![];
         let mut watermarks = vec![];
@@ -2775,9 +2609,7 @@ impl Parser {
                 if wildcard_idx.is_none() {
                     wildcard_idx = Some(columns.len());
                 } else {
-                    return Err(ParserError::ParserError(
-                        "At most 1 wildcard is allowed in source definetion".to_string(),
-                    ));
+                    parser_err!("At most 1 wildcard is allowed in source definetion");
                 }
             } else if let Some(constraint) = self.parse_optional_table_constraint()? {
                 constraints.push(constraint);
@@ -2785,28 +2617,26 @@ impl Parser {
                 watermarks.push(watermark);
                 if watermarks.len() > 1 {
                     // TODO(yuhao): allow multiple watermark on source.
-                    return Err(ParserError::ParserError(
-                        "Only 1 watermark is allowed to be defined on source.".to_string(),
-                    ));
+                    parser_err!("Only 1 watermark is allowed to be defined on source.");
                 }
             } else if let Token::Word(_) = self.peek_token().token {
                 columns.push(self.parse_column_def()?);
             } else {
-                return self.expected("column name or constraint definition", self.peek_token());
+                return self.expected("column name or constraint definition");
             }
             let comma = self.consume_token(&Token::Comma);
             if self.consume_token(&Token::RParen) {
                 // allow a trailing comma, even though it's not in standard
                 break;
             } else if !comma {
-                return self.expected("',' or ')' after column definition", self.peek_token());
+                return self.expected("',' or ')' after column definition");
             }
         }
 
         Ok((columns, constraints, watermarks, wildcard_idx))
     }
 
-    fn parse_column_def(&mut self) -> Result<ColumnDef, ParserError> {
+    fn parse_column_def(&mut self) -> PResult<ColumnDef> {
         let name = self.parse_identifier_non_reserved()?;
         let data_type = if let Token::Word(_) = self.peek_token().token {
             Some(self.parse_data_type()?)
@@ -2826,10 +2656,7 @@ impl Parser {
                 if let Some(option) = self.parse_optional_column_option()? {
                     options.push(ColumnOptionDef { name, option });
                 } else {
-                    return self.expected(
-                        "constraint details after CONSTRAINT <name>",
-                        self.peek_token(),
-                    );
+                    return self.expected("constraint details after CONSTRAINT <name>");
                 }
             } else if let Some(option) = self.parse_optional_column_option()? {
                 options.push(ColumnOptionDef { name: None, option });
@@ -2845,7 +2672,7 @@ impl Parser {
         })
     }
 
-    pub fn parse_optional_column_option(&mut self) -> Result<Option<ColumnOption>, ParserError> {
+    pub fn parse_optional_column_option(&mut self) -> PResult<Option<ColumnOption>> {
         if self.parse_keywords(&[Keyword::NOT, Keyword::NULL]) {
             Ok(Some(ColumnOption::NotNull))
         } else if self.parse_keyword(Keyword::NULL) {
@@ -2892,7 +2719,7 @@ impl Parser {
         }
     }
 
-    pub fn parse_handle_conflict_behavior(&mut self) -> Result<Option<OnConflict>, ParserError> {
+    pub fn parse_handle_conflict_behavior(&mut self) -> PResult<Option<OnConflict>> {
         if self.parse_keyword(Keyword::OVERWRITE) {
             Ok(Some(OnConflict::OverWrite))
         } else if self.parse_keyword(Keyword::IGNORE) {
@@ -2910,7 +2737,7 @@ impl Parser {
         }
     }
 
-    pub fn parse_referential_action(&mut self) -> Result<ReferentialAction, ParserError> {
+    pub fn parse_referential_action(&mut self) -> PResult<ReferentialAction> {
         if self.parse_keyword(Keyword::RESTRICT) {
             Ok(ReferentialAction::Restrict)
         } else if self.parse_keyword(Keyword::CASCADE) {
@@ -2922,14 +2749,11 @@ impl Parser {
         } else if self.parse_keywords(&[Keyword::SET, Keyword::DEFAULT]) {
             Ok(ReferentialAction::SetDefault)
         } else {
-            self.expected(
-                "one of RESTRICT, CASCADE, SET NULL, NO ACTION or SET DEFAULT",
-                self.peek_token(),
-            )
+            self.expected("one of RESTRICT, CASCADE, SET NULL, NO ACTION or SET DEFAULT")
         }
     }
 
-    pub fn parse_optional_watermark(&mut self) -> Result<Option<SourceWatermark>, ParserError> {
+    pub fn parse_optional_watermark(&mut self) -> PResult<Option<SourceWatermark>> {
         if self.parse_keyword(Keyword::WATERMARK) {
             self.expect_keyword(Keyword::FOR)?;
             let column = self.parse_identifier_non_reserved()?;
@@ -2941,14 +2765,13 @@ impl Parser {
         }
     }
 
-    pub fn parse_optional_table_constraint(
-        &mut self,
-    ) -> Result<Option<TableConstraint>, ParserError> {
+    pub fn parse_optional_table_constraint(&mut self) -> PResult<Option<TableConstraint>> {
         let name = if self.parse_keyword(Keyword::CONSTRAINT) {
             Some(self.parse_identifier_non_reserved()?)
         } else {
             None
         };
+        let checkpoint = *self;
         let token = self.next_token();
         match token.token {
             Token::Word(w) if w.keyword == Keyword::PRIMARY || w.keyword == Keyword::UNIQUE => {
@@ -2997,14 +2820,11 @@ impl Parser {
                 self.expect_token(&Token::RParen)?;
                 Ok(Some(TableConstraint::Check { name, expr }))
             }
-            unexpected => {
+            _ => {
+                *self = checkpoint;
                 if name.is_some() {
-                    self.expected(
-                        "PRIMARY, UNIQUE, FOREIGN, or CHECK",
-                        unexpected.with_location(token.location),
-                    )
+                    self.expected("PRIMARY, UNIQUE, FOREIGN, or CHECK")
                 } else {
-                    self.prev_token();
                     Ok(None)
                 }
             }
@@ -3014,7 +2834,7 @@ impl Parser {
     pub fn parse_options_with_preceding_keyword(
         &mut self,
         keyword: Keyword,
-    ) -> Result<Vec<SqlOption>, ParserError> {
+    ) -> PResult<Vec<SqlOption>> {
         if self.parse_keyword(keyword) {
             self.expect_token(&Token::LParen)?;
             self.parse_options_inner()
@@ -3023,7 +2843,7 @@ impl Parser {
         }
     }
 
-    pub fn parse_options(&mut self) -> Result<Vec<SqlOption>, ParserError> {
+    pub fn parse_options(&mut self) -> PResult<Vec<SqlOption>> {
         if self.peek_token() == Token::LParen {
             self.next_token();
             self.parse_options_inner()
@@ -3033,7 +2853,7 @@ impl Parser {
     }
 
     // has parsed a LParen
-    pub fn parse_options_inner(&mut self) -> Result<Vec<SqlOption>, ParserError> {
+    pub fn parse_options_inner(&mut self) -> PResult<Vec<SqlOption>> {
         let mut values = vec![];
         loop {
             values.push(Parser::parse_sql_option(self)?);
@@ -3042,21 +2862,22 @@ impl Parser {
                 // allow a trailing comma, even though it's not in standard
                 break;
             } else if !comma {
-                return self.expected("',' or ')' after option definition", self.peek_token());
+                return self.expected("',' or ')' after option definition");
             }
         }
         Ok(values)
     }
 
-    pub fn parse_sql_option(&mut self) -> Result<SqlOption, ParserError> {
+    pub fn parse_sql_option(&mut self) -> PResult<SqlOption> {
         let name = self.parse_object_name()?;
         self.expect_token(&Token::Eq)?;
         let value = self.parse_value()?;
         Ok(SqlOption { name, value })
     }
 
-    pub fn parse_since(&mut self) -> Result<Option<Since>, ParserError> {
+    pub fn parse_since(&mut self) -> PResult<Option<Since>> {
         if self.parse_keyword(Keyword::SINCE) {
+            let checkpoint = *self;
             let token = self.next_token();
             match token.token {
                 Token::Word(w) => {
@@ -3071,29 +2892,26 @@ impl Parser {
                         self.expect_token(&Token::RParen)?;
                         Ok(Some(Since::Begin))
                     } else {
-                        parser_err!(format!(
+                        parser_err!(
                             "Expected proctime(), begin() or now(), found: {}",
                             ident.real_value()
-                        ))
+                        )
                     }
                 }
                 Token::Number(s) => {
-                    let num = s.parse::<u64>().map_err(|e| {
-                        ParserError::ParserError(format!("Could not parse '{}' as u64: {}", s, e))
-                    });
-                    Ok(Some(Since::TimestampMsNum(num?)))
+                    let num = s
+                        .parse::<u64>()
+                        .map_err(|e| StrError(format!("Could not parse '{}' as u64: {}", s, e)))?;
+                    Ok(Some(Since::TimestampMsNum(num)))
                 }
-                unexpected => self.expected(
-                    "proctime(), begin() , now(), Number",
-                    unexpected.with_location(token.location),
-                ),
+                _ => self.expected_at(checkpoint, "proctime(), begin() , now(), Number"),
             }
         } else {
             Ok(None)
         }
     }
 
-    pub fn parse_emit_mode(&mut self) -> Result<Option<EmitMode>, ParserError> {
+    pub fn parse_emit_mode(&mut self) -> PResult<Option<EmitMode>> {
         if self.parse_keyword(Keyword::EMIT) {
             match self.parse_one_of_keywords(&[Keyword::IMMEDIATELY, Keyword::ON]) {
                 Some(Keyword::IMMEDIATELY) => Ok(Some(EmitMode::Immediately)),
@@ -3102,17 +2920,14 @@ impl Parser {
                     Ok(Some(EmitMode::OnWindowClose))
                 }
                 Some(_) => unreachable!(),
-                None => self.expected(
-                    "IMMEDIATELY or ON WINDOW CLOSE after EMIT",
-                    self.peek_token(),
-                ),
+                None => self.expected("IMMEDIATELY or ON WINDOW CLOSE after EMIT"),
             }
         } else {
             Ok(None)
         }
     }
 
-    pub fn parse_alter(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_alter(&mut self) -> PResult<Statement> {
         if self.parse_keyword(Keyword::DATABASE) {
             self.parse_alter_database()
         } else if self.parse_keyword(Keyword::SCHEMA) {
@@ -3141,13 +2956,12 @@ impl Parser {
             self.parse_alter_subscription()
         } else {
             self.expected(
-                "DATABASE, SCHEMA, TABLE, INDEX, MATERIALIZED, VIEW, SINK, SUBSCRIPTION, SOURCE, FUNCTION, USER or SYSTEM after ALTER",
-                self.peek_token(),
+                "DATABASE, SCHEMA, TABLE, INDEX, MATERIALIZED, VIEW, SINK, SUBSCRIPTION, SOURCE, FUNCTION, USER or SYSTEM after ALTER"
             )
         }
     }
 
-    pub fn parse_alter_database(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_alter_database(&mut self) -> PResult<Statement> {
         let database_name = self.parse_object_name()?;
         let operation = if self.parse_keywords(&[Keyword::OWNER, Keyword::TO]) {
             let owner_name: Ident = self.parse_identifier()?;
@@ -3159,10 +2973,10 @@ impl Parser {
                 let database_name = self.parse_object_name()?;
                 AlterDatabaseOperation::RenameDatabase { database_name }
             } else {
-                return self.expected("TO after RENAME", self.peek_token());
+                return self.expected("TO after RENAME");
             }
         } else {
-            return self.expected("OWNER TO after ALTER DATABASE", self.peek_token());
+            return self.expected("OWNER TO after ALTER DATABASE");
         };
 
         Ok(Statement::AlterDatabase {
@@ -3171,7 +2985,7 @@ impl Parser {
         })
     }
 
-    pub fn parse_alter_schema(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_alter_schema(&mut self) -> PResult<Statement> {
         let schema_name = self.parse_object_name()?;
         let operation = if self.parse_keywords(&[Keyword::OWNER, Keyword::TO]) {
             let owner_name: Ident = self.parse_identifier()?;
@@ -3179,14 +2993,11 @@ impl Parser {
                 new_owner_name: owner_name,
             }
         } else if self.parse_keyword(Keyword::RENAME) {
-            if self.parse_keyword(Keyword::TO) {
-                let schema_name = self.parse_object_name()?;
-                AlterSchemaOperation::RenameSchema { schema_name }
-            } else {
-                return self.expected("TO after RENAME", self.peek_token());
-            }
+            self.expect_keyword(Keyword::TO)?;
+            let schema_name = self.parse_object_name()?;
+            AlterSchemaOperation::RenameSchema { schema_name }
         } else {
-            return self.expected("RENAME OR OWNER TO after ALTER SCHEMA", self.peek_token());
+            return self.expected("RENAME OR OWNER TO after ALTER SCHEMA");
         };
 
         Ok(Statement::AlterSchema {
@@ -3195,11 +3006,11 @@ impl Parser {
         })
     }
 
-    pub fn parse_alter_user(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_alter_user(&mut self) -> PResult<Statement> {
         Ok(Statement::AlterUser(AlterUserStatement::parse_to(self)?))
     }
 
-    pub fn parse_alter_table(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_alter_table(&mut self) -> PResult<Statement> {
         let _ = self.parse_keyword(Keyword::ONLY);
         let table_name = self.parse_object_name()?;
         let operation = if self.parse_keyword(Keyword::ADD) {
@@ -3246,10 +3057,7 @@ impl Parser {
                 if self.expect_keyword(Keyword::TO).is_err()
                     && self.expect_token(&Token::Eq).is_err()
                 {
-                    return self.expected(
-                        "TO or = after ALTER TABLE SET PARALLELISM",
-                        self.peek_token(),
-                    );
+                    return self.expected("TO or = after ALTER TABLE SET PARALLELISM");
                 }
 
                 let value = self.parse_set_variable()?;
@@ -3260,8 +3068,10 @@ impl Parser {
                     parallelism: value,
                     deferred,
                 }
+            } else if let Some(rate_limit) = self.parse_alter_streaming_rate_limit()? {
+                AlterTableOperation::SetStreamingRateLimit { rate_limit }
             } else {
-                return self.expected("SCHEMA/PARALLELISM after SET", self.peek_token());
+                return self.expected("SCHEMA/PARALLELISM/STREAMING_RATE_LIMIT after SET");
             }
         } else if self.parse_keyword(Keyword::DROP) {
             let _ = self.parse_keyword(Keyword::COLUMN);
@@ -3298,19 +3108,14 @@ impl Parser {
                 };
                 AlterColumnOperation::SetDataType { data_type, using }
             } else {
-                return self.expected(
-                    "SET/DROP NOT NULL, SET DEFAULT, SET DATA TYPE after ALTER COLUMN",
-                    self.peek_token(),
-                );
+                return self
+                    .expected("SET/DROP NOT NULL, SET DEFAULT, SET DATA TYPE after ALTER COLUMN");
             };
             AlterTableOperation::AlterColumn { column_name, op }
         } else if self.parse_keywords(&[Keyword::REFRESH, Keyword::SCHEMA]) {
             AlterTableOperation::RefreshSchema
         } else {
-            return self.expected(
-                "ADD or RENAME or OWNER TO or SET or DROP after ALTER TABLE",
-                self.peek_token(),
-            );
+            return self.expected("ADD or RENAME or OWNER TO or SET or DROP after ALTER TABLE");
         };
         Ok(Statement::AlterTable {
             name: table_name,
@@ -3318,24 +3123,43 @@ impl Parser {
         })
     }
 
-    pub fn parse_alter_index(&mut self) -> Result<Statement, ParserError> {
+    /// STREAMING_RATE_LIMIT = default | NUMBER
+    /// STREAMING_RATE_LIMIT TO default | NUMBER
+    pub fn parse_alter_streaming_rate_limit(&mut self) -> PResult<Option<i32>> {
+        if !self.parse_keyword(Keyword::STREAMING_RATE_LIMIT) {
+            return Ok(None);
+        }
+        if self.expect_keyword(Keyword::TO).is_err() && self.expect_token(&Token::Eq).is_err() {
+            return self.expected("TO or = after ALTER TABLE SET STREAMING_RATE_LIMIT");
+        }
+        let rate_limit = if self.parse_keyword(Keyword::DEFAULT) {
+            -1
+        } else {
+            let s = self.parse_number_value()?;
+            if let Ok(n) = s.parse::<i32>() {
+                n
+            } else {
+                return self.expected("number or DEFAULT");
+            }
+        };
+        Ok(Some(rate_limit))
+    }
+
+    pub fn parse_alter_index(&mut self) -> PResult<Statement> {
         let index_name = self.parse_object_name()?;
         let operation = if self.parse_keyword(Keyword::RENAME) {
             if self.parse_keyword(Keyword::TO) {
                 let index_name = self.parse_object_name()?;
                 AlterIndexOperation::RenameIndex { index_name }
             } else {
-                return self.expected("TO after RENAME", self.peek_token());
+                return self.expected("TO after RENAME");
             }
         } else if self.parse_keyword(Keyword::SET) {
             if self.parse_keyword(Keyword::PARALLELISM) {
                 if self.expect_keyword(Keyword::TO).is_err()
                     && self.expect_token(&Token::Eq).is_err()
                 {
-                    return self.expected(
-                        "TO or = after ALTER TABLE SET PARALLELISM",
-                        self.peek_token(),
-                    );
+                    return self.expected("TO or = after ALTER TABLE SET PARALLELISM");
                 }
 
                 let value = self.parse_set_variable()?;
@@ -3347,10 +3171,10 @@ impl Parser {
                     deferred,
                 }
             } else {
-                return self.expected("PARALLELISM after SET", self.peek_token());
+                return self.expected("PARALLELISM after SET");
             }
         } else {
-            return self.expected("RENAME after ALTER INDEX", self.peek_token());
+            return self.expected("RENAME after ALTER INDEX");
         };
 
         Ok(Statement::AlterIndex {
@@ -3359,14 +3183,14 @@ impl Parser {
         })
     }
 
-    pub fn parse_alter_view(&mut self, materialized: bool) -> Result<Statement, ParserError> {
+    pub fn parse_alter_view(&mut self, materialized: bool) -> PResult<Statement> {
         let view_name = self.parse_object_name()?;
         let operation = if self.parse_keyword(Keyword::RENAME) {
             if self.parse_keyword(Keyword::TO) {
                 let view_name = self.parse_object_name()?;
                 AlterViewOperation::RenameView { view_name }
             } else {
-                return self.expected("TO after RENAME", self.peek_token());
+                return self.expected("TO after RENAME");
             }
         } else if self.parse_keywords(&[Keyword::OWNER, Keyword::TO]) {
             let owner_name: Ident = self.parse_identifier()?;
@@ -3383,10 +3207,7 @@ impl Parser {
                 if self.expect_keyword(Keyword::TO).is_err()
                     && self.expect_token(&Token::Eq).is_err()
                 {
-                    return self.expected(
-                        "TO or = after ALTER TABLE SET PARALLELISM",
-                        self.peek_token(),
-                    );
+                    return self.expected("TO or = after ALTER TABLE SET PARALLELISM");
                 }
 
                 let value = self.parse_set_variable()?;
@@ -3397,17 +3218,18 @@ impl Parser {
                     parallelism: value,
                     deferred,
                 }
+            } else if materialized
+                && let Some(rate_limit) = self.parse_alter_streaming_rate_limit()?
+            {
+                AlterViewOperation::SetStreamingRateLimit { rate_limit }
             } else {
-                return self.expected("SCHEMA/PARALLELISM after SET", self.peek_token());
+                return self.expected("SCHEMA/PARALLELISM/STREAMING_RATE_LIMIT after SET");
             }
         } else {
-            return self.expected(
-                &format!(
-                    "RENAME or OWNER TO or SET after ALTER {}VIEW",
-                    if materialized { "MATERIALIZED " } else { "" }
-                ),
-                self.peek_token(),
-            );
+            return self.expected(&format!(
+                "RENAME or OWNER TO or SET after ALTER {}VIEW",
+                if materialized { "MATERIALIZED " } else { "" }
+            ));
         };
 
         Ok(Statement::AlterView {
@@ -3417,14 +3239,14 @@ impl Parser {
         })
     }
 
-    pub fn parse_alter_sink(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_alter_sink(&mut self) -> PResult<Statement> {
         let sink_name = self.parse_object_name()?;
         let operation = if self.parse_keyword(Keyword::RENAME) {
             if self.parse_keyword(Keyword::TO) {
                 let sink_name = self.parse_object_name()?;
                 AlterSinkOperation::RenameSink { sink_name }
             } else {
-                return self.expected("TO after RENAME", self.peek_token());
+                return self.expected("TO after RENAME");
             }
         } else if self.parse_keywords(&[Keyword::OWNER, Keyword::TO]) {
             let owner_name: Ident = self.parse_identifier()?;
@@ -3441,10 +3263,7 @@ impl Parser {
                 if self.expect_keyword(Keyword::TO).is_err()
                     && self.expect_token(&Token::Eq).is_err()
                 {
-                    return self.expected(
-                        "TO or = after ALTER TABLE SET PARALLELISM",
-                        self.peek_token(),
-                    );
+                    return self.expected("TO or = after ALTER TABLE SET PARALLELISM");
                 }
 
                 let value = self.parse_set_variable()?;
@@ -3455,13 +3274,10 @@ impl Parser {
                     deferred,
                 }
             } else {
-                return self.expected("SCHEMA/PARALLELISM after SET", self.peek_token());
+                return self.expected("SCHEMA/PARALLELISM after SET");
             }
         } else {
-            return self.expected(
-                "RENAME or OWNER TO or SET after ALTER SINK",
-                self.peek_token(),
-            );
+            return self.expected("RENAME or OWNER TO or SET after ALTER SINK");
         };
 
         Ok(Statement::AlterSink {
@@ -3470,14 +3286,14 @@ impl Parser {
         })
     }
 
-    pub fn parse_alter_subscription(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_alter_subscription(&mut self) -> PResult<Statement> {
         let subscription_name = self.parse_object_name()?;
         let operation = if self.parse_keyword(Keyword::RENAME) {
             if self.parse_keyword(Keyword::TO) {
                 let subscription_name = self.parse_object_name()?;
                 AlterSubscriptionOperation::RenameSubscription { subscription_name }
             } else {
-                return self.expected("TO after RENAME", self.peek_token());
+                return self.expected("TO after RENAME");
             }
         } else if self.parse_keywords(&[Keyword::OWNER, Keyword::TO]) {
             let owner_name: Ident = self.parse_identifier()?;
@@ -3490,30 +3306,11 @@ impl Parser {
                 AlterSubscriptionOperation::SetSchema {
                     new_schema_name: schema_name,
                 }
-            } else if self.parse_keyword(Keyword::PARALLELISM) {
-                if self.expect_keyword(Keyword::TO).is_err()
-                    && self.expect_token(&Token::Eq).is_err()
-                {
-                    return self.expected(
-                        "TO or = after ALTER TABLE SET PARALLELISM",
-                        self.peek_token(),
-                    );
-                }
-                let value = self.parse_set_variable()?;
-                let deferred = self.parse_keyword(Keyword::DEFERRED);
-
-                AlterSubscriptionOperation::SetParallelism {
-                    parallelism: value,
-                    deferred,
-                }
             } else {
-                return self.expected("SCHEMA/PARALLELISM after SET", self.peek_token());
+                return self.expected("SCHEMA after SET");
             }
         } else {
-            return self.expected(
-                "RENAME or OWNER TO or SET after ALTER SUBSCRIPTION",
-                self.peek_token(),
-            );
+            return self.expected("RENAME or OWNER TO or SET after ALTER SUBSCRIPTION");
         };
 
         Ok(Statement::AlterSubscription {
@@ -3522,14 +3319,14 @@ impl Parser {
         })
     }
 
-    pub fn parse_alter_source(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_alter_source(&mut self) -> PResult<Statement> {
         let source_name = self.parse_object_name()?;
         let operation = if self.parse_keyword(Keyword::RENAME) {
             if self.parse_keyword(Keyword::TO) {
                 let source_name = self.parse_object_name()?;
                 AlterSourceOperation::RenameSource { source_name }
             } else {
-                return self.expected("TO after RENAME", self.peek_token());
+                return self.expected("TO after RENAME");
             }
         } else if self.parse_keyword(Keyword::ADD) {
             let _ = self.parse_keyword(Keyword::COLUMN);
@@ -3547,18 +3344,22 @@ impl Parser {
                 AlterSourceOperation::SetSchema {
                     new_schema_name: schema_name,
                 }
+            } else if let Some(rate_limit) = self.parse_alter_streaming_rate_limit()? {
+                AlterSourceOperation::SetStreamingRateLimit { rate_limit }
             } else {
-                return self.expected("SCHEMA after SET", self.peek_token());
+                return self.expected("SCHEMA after SET");
             }
         } else if self.peek_nth_any_of_keywords(0, &[Keyword::FORMAT]) {
             let connector_schema = self.parse_schema()?.unwrap();
+            if connector_schema.key_encode.is_some() {
+                parser_err!("key encode clause is not supported in source schema");
+            }
             AlterSourceOperation::FormatEncode { connector_schema }
         } else if self.parse_keywords(&[Keyword::REFRESH, Keyword::SCHEMA]) {
             AlterSourceOperation::RefreshSchema
         } else {
             return self.expected(
-                "RENAME, ADD COLUMN or OWNER TO or SET after ALTER SOURCE",
-                self.peek_token(),
+                "RENAME, ADD COLUMN, OWNER TO, SET or STREAMING_RATE_LIMIT after ALTER SOURCE",
             );
         };
 
@@ -3568,7 +3369,7 @@ impl Parser {
         })
     }
 
-    pub fn parse_alter_function(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_alter_function(&mut self) -> PResult<Statement> {
         let FunctionDesc { name, args } = self.parse_function_desc()?;
 
         let operation = if self.parse_keyword(Keyword::SET) {
@@ -3578,10 +3379,10 @@ impl Parser {
                     new_schema_name: schema_name,
                 }
             } else {
-                return self.expected("SCHEMA after SET", self.peek_token());
+                return self.expected("SCHEMA after SET");
             }
         } else {
-            return self.expected("SET after ALTER FUNCTION", self.peek_token());
+            return self.expected("SET after ALTER FUNCTION");
         };
 
         Ok(Statement::AlterFunction {
@@ -3591,7 +3392,7 @@ impl Parser {
         })
     }
 
-    pub fn parse_alter_connection(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_alter_connection(&mut self) -> PResult<Statement> {
         let connection_name = self.parse_object_name()?;
         let operation = if self.parse_keyword(Keyword::SET) {
             if self.parse_keyword(Keyword::SCHEMA) {
@@ -3600,10 +3401,10 @@ impl Parser {
                     new_schema_name: schema_name,
                 }
             } else {
-                return self.expected("SCHEMA after SET", self.peek_token());
+                return self.expected("SCHEMA after SET");
             }
         } else {
-            return self.expected("SET after ALTER CONNECTION", self.peek_token());
+            return self.expected("SET after ALTER CONNECTION");
         };
 
         Ok(Statement::AlterConnection {
@@ -3612,18 +3413,18 @@ impl Parser {
         })
     }
 
-    pub fn parse_alter_system(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_alter_system(&mut self) -> PResult<Statement> {
         self.expect_keyword(Keyword::SET)?;
         let param = self.parse_identifier()?;
         if self.expect_keyword(Keyword::TO).is_err() && self.expect_token(&Token::Eq).is_err() {
-            return self.expected("TO or = after ALTER SYSTEM SET", self.peek_token());
+            return self.expected("TO or = after ALTER SYSTEM SET");
         }
         let value = self.parse_set_variable()?;
         Ok(Statement::AlterSystem { param, value })
     }
 
     /// Parse a copy statement
-    pub fn parse_copy(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_copy(&mut self) -> PResult<Statement> {
         let table_name = self.parse_object_name()?;
         let columns = self.parse_parenthesized_column_list(Optional)?;
         self.expect_keywords(&[Keyword::FROM, Keyword::STDIN])?;
@@ -3674,7 +3475,8 @@ impl Parser {
     }
 
     /// Parse a literal value (numbers, strings, date/time, booleans)
-    fn parse_value(&mut self) -> Result<Value, ParserError> {
+    pub fn parse_value(&mut self) -> PResult<Value> {
+        let checkpoint = *self;
         let token = self.next_token();
         match token.token {
             Token::Word(w) => match w.keyword {
@@ -3684,12 +3486,9 @@ impl Parser {
                 Keyword::NoKeyword if w.quote_style.is_some() => match w.quote_style {
                     Some('"') => Ok(Value::DoubleQuotedString(w.value)),
                     Some('\'') => Ok(Value::SingleQuotedString(w.value)),
-                    _ => self.expected("A value?", Token::Word(w).with_location(token.location))?,
+                    _ => self.expected_at(checkpoint, "A value")?,
                 },
-                _ => self.expected(
-                    "a concrete value",
-                    Token::Word(w).with_location(token.location),
-                ),
+                _ => self.expected_at(checkpoint, "a concrete value"),
             },
             Token::Number(ref n) => Ok(Value::Number(n.clone())),
             Token::SingleQuotedString(ref s) => Ok(Value::SingleQuotedString(s.to_string())),
@@ -3697,67 +3496,55 @@ impl Parser {
             Token::CstyleEscapesString(ref s) => Ok(Value::CstyleEscapedString(s.clone())),
             Token::NationalStringLiteral(ref s) => Ok(Value::NationalStringLiteral(s.to_string())),
             Token::HexStringLiteral(ref s) => Ok(Value::HexStringLiteral(s.to_string())),
-            unexpected => self.expected("a value", unexpected.with_location(token.location)),
-        }
-    }
-
-    fn parse_set_variable(&mut self) -> Result<SetVariableValue, ParserError> {
-        let mut values = vec![];
-        loop {
-            let token = self.peek_token();
-            let value = match (self.parse_value(), token.token) {
-                (Ok(value), _) => SetVariableValueSingle::Literal(value),
-                (Err(_), Token::Word(w)) => {
-                    if w.keyword == Keyword::DEFAULT {
-                        if !values.is_empty() {
-                            self.expected(
-                                "parameter list value",
-                                Token::Word(w).with_location(token.location),
-                            )?
+            _ => self.expected_at(checkpoint, "a value"),
+        }
+    }
+
+    fn parse_set_variable(&mut self) -> PResult<SetVariableValue> {
+        alt((
+            Keyword::DEFAULT.value(SetVariableValue::Default),
+            separated(
+                1..,
+                alt((
+                    Self::parse_value.map(SetVariableValueSingle::Literal),
+                    |parser: &mut Self| {
+                        let checkpoint = *parser;
+                        let ident = parser.parse_identifier()?;
+                        if ident.value == "default" {
+                            *parser = checkpoint;
+                            return parser.expected("parameter list value").map_err(|e| e.cut());
                         }
-                        return Ok(SetVariableValue::Default);
-                    } else {
-                        SetVariableValueSingle::Ident(w.to_ident()?)
-                    }
-                }
-                (Err(_), unexpected) => {
-                    self.expected("parameter value", unexpected.with_location(token.location))?
+                        Ok(SetVariableValueSingle::Ident(ident))
+                    },
+                    fail.expect("parameter value"),
+                )),
+                Token::Comma,
+            )
+            .map(|list: Vec<SetVariableValueSingle>| {
+                if list.len() == 1 {
+                    SetVariableValue::Single(list[0].clone())
+                } else {
+                    SetVariableValue::List(list)
                 }
-            };
-            values.push(value);
-            if !self.consume_token(&Token::Comma) {
-                break;
-            }
-        }
-        if values.len() == 1 {
-            Ok(SetVariableValue::Single(values[0].clone()))
-        } else {
-            Ok(SetVariableValue::List(values))
-        }
+            }),
+        ))
+        .parse_next(self)
     }
 
-    pub fn parse_number_value(&mut self) -> Result<String, ParserError> {
+    pub fn parse_number_value(&mut self) -> PResult<String> {
+        let checkpoint = *self;
         match self.parse_value()? {
             Value::Number(v) => Ok(v),
-            _ => {
-                self.prev_token();
-                self.expected("literal number", self.peek_token())
-            }
+            _ => self.expected_at(checkpoint, "literal number"),
         }
     }
 
     /// Parse an unsigned literal integer/long
-    pub fn parse_literal_uint(&mut self) -> Result<u64, ParserError> {
-        let token = self.next_token();
-        match token.token {
-            Token::Number(s) => s.parse::<u64>().map_err(|e| {
-                ParserError::ParserError(format!("Could not parse '{}' as u64: {}", s, e))
-            }),
-            unexpected => self.expected("literal int", unexpected.with_location(token.location)),
-        }
+    pub fn parse_literal_uint(&mut self) -> PResult<u64> {
+        literal_uint(self)
     }
 
-    pub fn parse_function_definition(&mut self) -> Result<FunctionDefinition, ParserError> {
+    pub fn parse_function_definition(&mut self) -> PResult<FunctionDefinition> {
         let peek_token = self.peek_token();
         match peek_token.token {
             Token::DollarQuotedString(value) => {
@@ -3771,7 +3558,8 @@ impl Parser {
     }
 
     /// Parse a literal string
-    pub fn parse_literal_string(&mut self) -> Result<String, ParserError> {
+    pub fn parse_literal_string(&mut self) -> PResult<String> {
+        let checkpoint = *self;
         let token = self.next_token();
         match token.token {
             Token::Word(Word {
@@ -3780,12 +3568,13 @@ impl Parser {
                 ..
             }) => Ok(value),
             Token::SingleQuotedString(s) => Ok(s),
-            unexpected => self.expected("literal string", unexpected.with_location(token.location)),
+            _ => self.expected_at(checkpoint, "literal string"),
         }
     }
 
     /// Parse a map key string
-    pub fn parse_map_key(&mut self) -> Result<Expr, ParserError> {
+    pub fn parse_map_key(&mut self) -> PResult<Expr> {
+        let checkpoint = *self;
         let token = self.next_token();
         match token.token {
             Token::Word(Word {
@@ -3800,156 +3589,22 @@ impl Parser {
             }
             Token::SingleQuotedString(s) => Ok(Expr::Value(Value::SingleQuotedString(s))),
             Token::Number(s) => Ok(Expr::Value(Value::Number(s))),
-            unexpected => self.expected(
-                "literal string, number or function",
-                unexpected.with_location(token.location),
-            ),
+            _ => self.expected_at(checkpoint, "literal string, number or function"),
         }
     }
 
     /// Parse a SQL datatype (in the context of a CREATE TABLE statement for example) and convert
     /// into an array of that datatype if needed
-    pub fn parse_data_type(&mut self) -> Result<DataType, ParserError> {
-        let mut data_type = self.parse_data_type_inner()?;
-        while self.consume_token(&Token::LBracket) {
-            self.expect_token(&Token::RBracket)?;
-            data_type = DataType::Array(Box::new(data_type));
-        }
-        Ok(data_type)
-    }
-
-    /// Parse struct `data_type` e.g.`<v1 int, v2 int, v3 struct<...>>`.
-    pub fn parse_struct_data_type(&mut self) -> Result<Vec<StructField>, ParserError> {
-        let mut columns = vec![];
-        if !self.consume_token(&Token::Lt) {
-            return self.expected("'<' after struct", self.peek_token());
-        }
-        self.angle_brackets_num += 1;
-
-        loop {
-            if let Token::Word(_) = self.peek_token().token {
-                let name = self.parse_identifier_non_reserved()?;
-                let data_type = self.parse_data_type()?;
-                columns.push(StructField { name, data_type })
-            } else {
-                return self.expected("struct field name", self.peek_token());
-            }
-            if self.angle_brackets_num == 0 {
-                break;
-            } else if self.consume_token(&Token::Gt) {
-                self.angle_brackets_num -= 1;
-                break;
-            } else if self.consume_token(&Token::ShiftRight) {
-                if self.angle_brackets_num >= 1 {
-                    self.angle_brackets_num -= 2;
-                    break;
-                } else {
-                    return parser_err!("too much '>'");
-                }
-            } else if !self.consume_token(&Token::Comma) {
-                return self.expected("',' or '>' after column definition", self.peek_token());
-            }
-        }
-
-        Ok(columns)
-    }
-
-    /// Parse a SQL datatype
-    pub fn parse_data_type_inner(&mut self) -> Result<DataType, ParserError> {
-        let token = self.next_token();
-        match token.token {
-            Token::Word(w) => match w.keyword {
-                Keyword::BOOLEAN | Keyword::BOOL => Ok(DataType::Boolean),
-                Keyword::FLOAT => {
-                    let precision = self.parse_optional_precision()?;
-                    match precision {
-                        Some(0) => Err(ParserError::ParserError(
-                            "precision for type float must be at least 1 bit".to_string(),
-                        )),
-                        Some(54..) => Err(ParserError::ParserError(
-                            "precision for type float must be less than 54 bits".to_string(),
-                        )),
-                        _ => Ok(DataType::Float(precision)),
-                    }
-                }
-                Keyword::REAL => Ok(DataType::Real),
-                Keyword::DOUBLE => {
-                    let _ = self.parse_keyword(Keyword::PRECISION);
-                    Ok(DataType::Double)
-                }
-                Keyword::SMALLINT => Ok(DataType::SmallInt),
-                Keyword::INT | Keyword::INTEGER => Ok(DataType::Int),
-                Keyword::BIGINT => Ok(DataType::BigInt),
-                Keyword::STRING | Keyword::VARCHAR => Ok(DataType::Varchar),
-                Keyword::CHAR | Keyword::CHARACTER => {
-                    if self.parse_keyword(Keyword::VARYING) {
-                        Ok(DataType::Varchar)
-                    } else {
-                        Ok(DataType::Char(self.parse_optional_precision()?))
-                    }
-                }
-                Keyword::UUID => Ok(DataType::Uuid),
-                Keyword::DATE => Ok(DataType::Date),
-                Keyword::TIMESTAMP => {
-                    let with_time_zone = self.parse_keyword(Keyword::WITH);
-                    if with_time_zone || self.parse_keyword(Keyword::WITHOUT) {
-                        self.expect_keywords(&[Keyword::TIME, Keyword::ZONE])?;
-                    }
-                    Ok(DataType::Timestamp(with_time_zone))
-                }
-                Keyword::TIME => {
-                    let with_time_zone = self.parse_keyword(Keyword::WITH);
-                    if with_time_zone || self.parse_keyword(Keyword::WITHOUT) {
-                        self.expect_keywords(&[Keyword::TIME, Keyword::ZONE])?;
-                    }
-                    Ok(DataType::Time(with_time_zone))
-                }
-                // Interval types can be followed by a complicated interval
-                // qualifier that we don't currently support. See
-                // parse_interval_literal for a taste.
-                Keyword::INTERVAL => Ok(DataType::Interval),
-                Keyword::REGCLASS => Ok(DataType::Regclass),
-                Keyword::REGPROC => Ok(DataType::Regproc),
-                Keyword::TEXT => {
-                    if self.consume_token(&Token::LBracket) {
-                        // Note: this is postgresql-specific
-                        self.expect_token(&Token::RBracket)?;
-                        Ok(DataType::Array(Box::new(DataType::Text)))
-                    } else {
-                        Ok(DataType::Text)
-                    }
-                }
-                Keyword::STRUCT => Ok(DataType::Struct(self.parse_struct_data_type()?)),
-                Keyword::BYTEA => Ok(DataType::Bytea),
-                Keyword::NUMERIC | Keyword::DECIMAL | Keyword::DEC => {
-                    let (precision, scale) = self.parse_optional_precision_scale()?;
-                    Ok(DataType::Decimal(precision, scale))
-                }
-                _ => {
-                    self.prev_token();
-                    let type_name = self.parse_object_name()?;
-                    // JSONB is not a keyword
-                    if type_name.to_string().eq_ignore_ascii_case("jsonb") {
-                        Ok(DataType::Jsonb)
-                    } else {
-                        Ok(DataType::Custom(type_name))
-                    }
-                }
-            },
-            unexpected => {
-                self.expected("a data type name", unexpected.with_location(token.location))
-            }
-        }
+    pub fn parse_data_type(&mut self) -> PResult<DataType> {
+        parser_v2::data_type(self)
     }
 
     /// Parse `AS identifier` (or simply `identifier` if it's not a reserved keyword)
     /// Some examples with aliases: `SELECT 1 foo`, `SELECT COUNT(*) AS cnt`,
     /// `SELECT ... FROM t1 foo, t2 bar`, `SELECT ... FROM (...) AS bar`
-    pub fn parse_optional_alias(
-        &mut self,
-        reserved_kwds: &[Keyword],
-    ) -> Result<Option<Ident>, ParserError> {
+    pub fn parse_optional_alias(&mut self, reserved_kwds: &[Keyword]) -> PResult<Option<Ident>> {
         let after_as = self.parse_keyword(Keyword::AS);
+        let checkpoint = *self;
         let token = self.next_token();
         match token.token {
             // Accept any identifier after `AS` (though many dialects have restrictions on
@@ -3960,14 +3615,11 @@ impl Parser {
             Token::Word(w) if after_as || (!reserved_kwds.contains(&w.keyword)) => {
                 Ok(Some(w.to_ident()?))
             }
-            not_an_ident => {
+            _ => {
+                *self = checkpoint;
                 if after_as {
-                    return self.expected(
-                        "an identifier after AS",
-                        not_an_ident.with_location(token.location),
-                    );
+                    return self.expected("an identifier after AS");
                 }
-                self.prev_token();
                 Ok(None) // no alias found
             }
         }
@@ -3980,7 +3632,7 @@ impl Parser {
     pub fn parse_optional_table_alias(
         &mut self,
         reserved_kwds: &[Keyword],
-    ) -> Result<Option<TableAlias>, ParserError> {
+    ) -> PResult<Option<TableAlias>> {
         match self.parse_optional_alias(reserved_kwds)? {
             Some(name) => {
                 let columns = self.parse_parenthesized_column_list(Optional)?;
@@ -3991,67 +3643,44 @@ impl Parser {
     }
 
     /// syntax `FOR SYSTEM_TIME AS OF PROCTIME()` is used for temporal join.
-    pub fn parse_as_of(&mut self) -> Result<Option<AsOf>, ParserError> {
-        let after_for = self.parse_keyword(Keyword::FOR);
-        if after_for {
-            if self.peek_nth_any_of_keywords(0, &[Keyword::SYSTEM_TIME]) {
-                self.expect_keywords(&[Keyword::SYSTEM_TIME, Keyword::AS, Keyword::OF])?;
-                let token = self.next_token();
-                match token.token {
-                    Token::Word(w) => {
-                        let ident = w.to_ident()?;
-                        // Backward compatibility for now.
-                        if ident.real_value() == "proctime" || ident.real_value() == "now" {
-                            self.expect_token(&Token::LParen)?;
-                            self.expect_token(&Token::RParen)?;
-                            Ok(Some(AsOf::ProcessTime))
-                        } else {
-                            parser_err!(format!("Expected proctime, found: {}", ident.real_value()))
-                        }
-                    }
-                    Token::Number(s) => {
-                        let num = s.parse::<i64>().map_err(|e| {
-                            ParserError::ParserError(format!(
-                                "Could not parse '{}' as i64: {}",
-                                s, e
-                            ))
-                        });
-                        Ok(Some(AsOf::TimestampNum(num?)))
-                    }
-                    Token::SingleQuotedString(s) => Ok(Some(AsOf::TimestampString(s))),
-                    unexpected => self.expected(
-                        "Proctime(), Number or SingleQuotedString",
-                        unexpected.with_location(token.location),
-                    ),
-                }
-            } else {
-                self.expect_keywords(&[Keyword::SYSTEM_VERSION, Keyword::AS, Keyword::OF])?;
-                let token = self.next_token();
-                match token.token {
-                    Token::Number(s) => {
-                        let num = s.parse::<i64>().map_err(|e| {
-                            ParserError::ParserError(format!(
-                                "Could not parse '{}' as i64: {}",
-                                s, e
-                            ))
-                        });
-                        Ok(Some(AsOf::VersionNum(num?)))
-                    }
-                    Token::SingleQuotedString(s) => Ok(Some(AsOf::VersionString(s))),
-                    unexpected => self.expected(
-                        "Number or SingleQuotedString",
-                        unexpected.with_location(token.location),
-                    ),
-                }
-            }
-        } else {
-            Ok(None)
-        }
+    pub fn parse_as_of(&mut self) -> PResult<AsOf> {
+        Keyword::FOR.parse_next(self)?;
+        alt((
+            preceded(
+                (Keyword::SYSTEM_TIME, Keyword::AS, Keyword::OF),
+                cut_err(
+                    alt((
+                        (
+                            Self::parse_identifier.verify(|ident| {
+                                ident.real_value() == "proctime" || ident.real_value() == "now"
+                            }),
+                            cut_err(Token::LParen),
+                            cut_err(Token::RParen),
+                        )
+                            .value(AsOf::ProcessTime),
+                        literal_i64.map(AsOf::TimestampNum),
+                        single_quoted_string.map(AsOf::TimestampString),
+                    ))
+                    .expect("proctime(), now(), number or string"),
+                ),
+            ),
+            preceded(
+                (Keyword::SYSTEM_VERSION, Keyword::AS, Keyword::OF),
+                cut_err(
+                    alt((
+                        literal_i64.map(AsOf::VersionNum),
+                        single_quoted_string.map(AsOf::VersionString),
+                    ))
+                    .expect("number or string"),
+                ),
+            ),
+        ))
+        .parse_next(self)
     }
 
     /// Parse a possibly qualified, possibly quoted identifier, e.g.
     /// `foo` or `myschema."table"
-    pub fn parse_object_name(&mut self) -> Result<ObjectName, ParserError> {
+    pub fn parse_object_name(&mut self) -> PResult<ObjectName> {
         let mut idents = vec![];
         loop {
             idents.push(self.parse_identifier()?);
@@ -4063,7 +3692,7 @@ impl Parser {
     }
 
     /// Parse identifiers strictly i.e. don't parse keywords
-    pub fn parse_identifiers_non_keywords(&mut self) -> Result<Vec<Ident>, ParserError> {
+    pub fn parse_identifiers_non_keywords(&mut self) -> PResult<Vec<Ident>> {
         let mut idents = vec![];
         loop {
             match self.peek_token().token {
@@ -4085,7 +3714,7 @@ impl Parser {
     }
 
     /// Parse identifiers
-    pub fn parse_identifiers(&mut self) -> Result<Vec<Ident>, ParserError> {
+    pub fn parse_identifiers(&mut self) -> PResult<Vec<Ident>> {
         let mut idents = vec![];
         loop {
             let token = self.next_token();
@@ -4102,33 +3731,32 @@ impl Parser {
     }
 
     /// Parse a simple one-word identifier (possibly quoted, possibly a keyword)
-    pub fn parse_identifier(&mut self) -> Result<Ident, ParserError> {
+    pub fn parse_identifier(&mut self) -> PResult<Ident> {
+        let checkpoint = *self;
         let token = self.next_token();
         match token.token {
             Token::Word(w) => Ok(w.to_ident()?),
-            unexpected => self.expected("identifier", unexpected.with_location(token.location)),
+            _ => self.expected_at(checkpoint, "identifier"),
         }
     }
 
     /// Parse a simple one-word identifier (possibly quoted, possibly a non-reserved keyword)
-    pub fn parse_identifier_non_reserved(&mut self) -> Result<Ident, ParserError> {
+    pub fn parse_identifier_non_reserved(&mut self) -> PResult<Ident> {
+        let checkpoint = *self;
         let token = self.next_token();
-        match token.token.clone() {
+        match token.token {
             Token::Word(w) => {
                 match keywords::RESERVED_FOR_COLUMN_OR_TABLE_NAME.contains(&w.keyword) {
-                    true => parser_err!(format!("syntax error at or near {token}")),
+                    true => parser_err!("syntax error at or near {w}"),
                     false => Ok(w.to_ident()?),
                 }
             }
-            unexpected => self.expected("identifier", unexpected.with_location(token.location)),
+            _ => self.expected_at(checkpoint, "identifier"),
         }
     }
 
     /// Parse a parenthesized comma-separated list of unqualified, possibly quoted identifiers
-    pub fn parse_parenthesized_column_list(
-        &mut self,
-        optional: IsOptional,
-    ) -> Result<Vec<Ident>, ParserError> {
+    pub fn parse_parenthesized_column_list(&mut self, optional: IsOptional) -> PResult<Vec<Ident>> {
         if self.consume_token(&Token::LParen) {
             let cols = self.parse_comma_separated(Parser::parse_identifier_non_reserved)?;
             self.expect_token(&Token::RParen)?;
@@ -4136,25 +3764,22 @@ impl Parser {
         } else if optional == Optional {
             Ok(vec![])
         } else {
-            self.expected("a list of columns in parentheses", self.peek_token())
+            self.expected("a list of columns in parentheses")
         }
     }
 
-    pub fn parse_returning(
-        &mut self,
-        optional: IsOptional,
-    ) -> Result<Vec<SelectItem>, ParserError> {
+    pub fn parse_returning(&mut self, optional: IsOptional) -> PResult<Vec<SelectItem>> {
         if self.parse_keyword(Keyword::RETURNING) {
             let cols = self.parse_comma_separated(Parser::parse_select_item)?;
             Ok(cols)
         } else if optional == Optional {
             Ok(vec![])
         } else {
-            self.expected("a list of columns or * after returning", self.peek_token())
+            self.expected("a list of columns or * after returning")
         }
     }
 
-    pub fn parse_row_expr(&mut self) -> Result<Expr, ParserError> {
+    pub fn parse_row_expr(&mut self) -> PResult<Expr> {
         Ok(Expr::Row(self.parse_token_wrapped_exprs(
             &Token::LParen,
             &Token::RParen,
@@ -4162,11 +3787,7 @@ impl Parser {
     }
 
     /// Parse a comma-separated list (maybe empty) from a wrapped expression
-    pub fn parse_token_wrapped_exprs(
-        &mut self,
-        left: &Token,
-        right: &Token,
-    ) -> Result<Vec<Expr>, ParserError> {
+    pub fn parse_token_wrapped_exprs(&mut self, left: &Token, right: &Token) -> PResult<Vec<Expr>> {
         if self.consume_token(left) {
             let exprs = if self.consume_token(right) {
                 vec![]
@@ -4177,11 +3798,11 @@ impl Parser {
             };
             Ok(exprs)
         } else {
-            self.expected(left.to_string().as_str(), self.peek_token())
+            self.expected(left.to_string().as_str())
         }
     }
 
-    pub fn parse_optional_precision(&mut self) -> Result<Option<u64>, ParserError> {
+    pub fn parse_optional_precision(&mut self) -> PResult<Option<u64>> {
         if self.consume_token(&Token::LParen) {
             let n = self.parse_literal_uint()?;
             self.expect_token(&Token::RParen)?;
@@ -4191,9 +3812,7 @@ impl Parser {
         }
     }
 
-    pub fn parse_optional_precision_scale(
-        &mut self,
-    ) -> Result<(Option<u64>, Option<u64>), ParserError> {
+    pub fn parse_optional_precision_scale(&mut self) -> PResult<(Option<u64>, Option<u64>)> {
         if self.consume_token(&Token::LParen) {
             let n = self.parse_literal_uint()?;
             let scale = if self.consume_token(&Token::Comma) {
@@ -4208,7 +3827,7 @@ impl Parser {
         }
     }
 
-    pub fn parse_delete(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_delete(&mut self) -> PResult<Statement> {
         self.expect_keyword(Keyword::FROM)?;
         let table_name = self.parse_object_name()?;
         let selection = if self.parse_keyword(Keyword::WHERE) {
@@ -4237,7 +3856,7 @@ impl Parser {
         }
     }
 
-    pub fn parse_explain(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_explain(&mut self) -> PResult<Statement> {
         let mut options = ExplainOptions::default();
 
         let explain_key_words = [
@@ -4249,7 +3868,7 @@ impl Parser {
             Keyword::DISTSQL,
         ];
 
-        let parse_explain_option = |parser: &mut Parser| -> Result<(), ParserError> {
+        let parse_explain_option = |parser: &mut Parser<'_>| -> PResult<()> {
             let keyword = parser.expect_one_of_keywords(&explain_key_words)?;
             match keyword {
                 Keyword::VERBOSE => options.verbose = parser.parse_optional_boolean(true),
@@ -4298,7 +3917,7 @@ impl Parser {
     /// preceded with some `WITH` CTE declarations and optionally followed
     /// by `ORDER BY`. Unlike some other parse_... methods, this one doesn't
     /// expect the initial keyword to be already consumed
-    pub fn parse_query(&mut self) -> Result<Query, ParserError> {
+    pub fn parse_query(&mut self) -> PResult<Query> {
         let with = if self.parse_keyword(Keyword::WITH) {
             Some(With {
                 recursive: self.parse_keyword(Keyword::RECURSIVE),
@@ -4330,13 +3949,11 @@ impl Parser {
 
         let fetch = if self.parse_keyword(Keyword::FETCH) {
             if limit.is_some() {
-                return parser_err!("Cannot specify both LIMIT and FETCH".to_string());
+                parser_err!("Cannot specify both LIMIT and FETCH");
             }
             let fetch = self.parse_fetch()?;
             if fetch.with_ties && order_by.is_empty() {
-                return parser_err!(
-                    "WITH TIES cannot be specified without ORDER BY clause".to_string()
-                );
+                parser_err!("WITH TIES cannot be specified without ORDER BY clause");
             }
             Some(fetch)
         } else {
@@ -4354,7 +3971,7 @@ impl Parser {
     }
 
     /// Parse a CTE (`alias [( col1, col2, ... )] AS (subquery)`)
-    fn parse_cte(&mut self) -> Result<Cte, ParserError> {
+    fn parse_cte(&mut self) -> PResult<Cte> {
         let name = self.parse_identifier_non_reserved()?;
 
         let mut cte = if self.parse_keyword(Keyword::AS) {
@@ -4397,7 +4014,7 @@ impl Parser {
     ///   subquery ::= query_body [ order_by_limit ]
     ///   set_operation ::= query_body { 'UNION' | 'EXCEPT' | 'INTERSECT' } [ 'ALL' ] query_body
     /// ```
-    fn parse_query_body(&mut self, precedence: u8) -> Result<SetExpr, ParserError> {
+    fn parse_query_body(&mut self, precedence: u8) -> PResult<SetExpr> {
         // We parse the expression using a Pratt parser, as in `parse_expr()`.
         // Start by parsing a restricted SELECT or a `(subquery)`:
         let mut expr = if self.parse_keyword(Keyword::SELECT) {
@@ -4410,10 +4027,7 @@ impl Parser {
         } else if self.parse_keyword(Keyword::VALUES) {
             SetExpr::Values(self.parse_values()?)
         } else {
-            return self.expected(
-                "SELECT, VALUES, or a subquery in the query body",
-                self.peek_token(),
-            );
+            return self.expected("SELECT, VALUES, or a subquery in the query body");
         };
 
         loop {
@@ -4453,7 +4067,7 @@ impl Parser {
 
     /// Parse a restricted `SELECT` statement (no CTEs / `UNION` / `ORDER BY`),
     /// assuming the initial `SELECT` was already consumed
-    pub fn parse_select(&mut self) -> Result<Select, ParserError> {
+    pub fn parse_select(&mut self) -> PResult<Select> {
         let distinct = self.parse_all_or_distinct_on()?;
 
         let projection = self.parse_comma_separated(Parser::parse_select_item)?;
@@ -4528,21 +4142,17 @@ impl Parser {
         })
     }
 
-    pub fn parse_set(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_set(&mut self) -> PResult<Statement> {
         let modifier = self.parse_one_of_keywords(&[Keyword::SESSION, Keyword::LOCAL]);
         if self.parse_keywords(&[Keyword::TIME, Keyword::ZONE]) {
-            let token = self.peek_token();
-            let value = match (self.parse_value(), token.token) {
-                (Ok(value), _) => SetTimeZoneValue::Literal(value),
-                (Err(_), Token::Word(w)) if w.keyword == Keyword::DEFAULT => {
-                    SetTimeZoneValue::Default
-                }
-                (Err(_), Token::Word(w)) if w.keyword == Keyword::LOCAL => SetTimeZoneValue::Local,
-                (Err(_), Token::Word(w)) => SetTimeZoneValue::Ident(w.to_ident()?),
-                (Err(_), unexpected) => {
-                    self.expected("variable value", unexpected.with_location(token.location))?
-                }
-            };
+            let value = alt((
+                Keyword::DEFAULT.value(SetTimeZoneValue::Default),
+                Keyword::LOCAL.value(SetTimeZoneValue::Local),
+                Self::parse_identifier.map(SetTimeZoneValue::Ident),
+                Self::parse_value.map(SetTimeZoneValue::Literal),
+            ))
+            .expect("variable")
+            .parse_next(self)?;
 
             Ok(Statement::SetTimeZone {
                 local: modifier == Some(Keyword::LOCAL),
@@ -4581,7 +4191,7 @@ impl Parser {
                     value,
                 })
             } else {
-                self.expected("equals sign or TO", self.peek_token())
+                self.expected("equals sign or TO")
             }
         }
     }
@@ -4589,8 +4199,8 @@ impl Parser {
     /// If have `databases`,`tables`,`columns`,`schemas` and `materialized views` after show,
     /// return `Statement::ShowCommand` or `Statement::ShowColumn`,
     /// otherwise, return `Statement::ShowVariable`.
-    pub fn parse_show(&mut self) -> Result<Statement, ParserError> {
-        let index = self.index;
+    pub fn parse_show(&mut self) -> PResult<Statement> {
+        let checkpoint = *self;
         if let Token::Word(w) = self.next_token().token {
             match w.keyword {
                 Keyword::TABLES => {
@@ -4663,7 +4273,7 @@ impl Parser {
                             filter: self.parse_show_statement_filter()?,
                         });
                     } else {
-                        return self.expected("VIEWS after MATERIALIZED", self.peek_token());
+                        return self.expected("VIEWS after MATERIALIZED");
                     }
                 }
                 Keyword::COLUMNS => {
@@ -4675,9 +4285,17 @@ impl Parser {
                             filter: self.parse_show_statement_filter()?,
                         });
                     } else {
-                        return self.expected("from after columns", self.peek_token());
+                        return self.expected("from after columns");
                     }
                 }
+                Keyword::SECRETS => {
+                    return Ok(Statement::ShowObjects {
+                        object: ShowObject::Secret {
+                            schema: self.parse_from_and_identifier()?,
+                        },
+                        filter: self.parse_show_statement_filter()?,
+                    });
+                }
                 Keyword::CONNECTIONS => {
                     return Ok(Statement::ShowObjects {
                         object: ShowObject::Connection {
@@ -4703,7 +4321,7 @@ impl Parser {
                             filter: self.parse_show_statement_filter()?,
                         });
                     } else {
-                        return self.expected("from after indexes", self.peek_token());
+                        return self.expected("from after indexes");
                     }
                 }
                 Keyword::CLUSTER => {
@@ -4731,19 +4349,19 @@ impl Parser {
                 _ => {}
             }
         }
-        self.index = index;
+        *self = checkpoint;
         Ok(Statement::ShowVariable {
             variable: self.parse_identifiers()?,
         })
     }
 
-    pub fn parse_cancel_job(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_cancel_job(&mut self) -> PResult<Statement> {
         // CANCEL [JOBS|JOB] job_ids
         match self.peek_token().token {
             Token::Word(w) if Keyword::JOBS == w.keyword || Keyword::JOB == w.keyword => {
                 self.next_token();
             }
-            _ => return self.expected("JOBS or JOB after CANCEL", self.peek_token()),
+            _ => return self.expected("JOBS or JOB after CANCEL"),
         }
 
         let mut job_ids = vec![];
@@ -4756,14 +4374,14 @@ impl Parser {
         Ok(Statement::CancelJobs(JobIdents(job_ids)))
     }
 
-    pub fn parse_kill_process(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_kill_process(&mut self) -> PResult<Statement> {
         let process_id = self.parse_literal_uint()? as i32;
         Ok(Statement::Kill(process_id))
     }
 
     /// Parser `from schema` after `show tables` and `show materialized views`, if not conclude
     /// `from` then use default schema name.
-    pub fn parse_from_and_identifier(&mut self) -> Result<Option<Ident>, ParserError> {
+    pub fn parse_from_and_identifier(&mut self) -> PResult<Option<Ident>> {
         if self.parse_keyword(Keyword::FROM) {
             Ok(Some(self.parse_identifier_non_reserved()?))
         } else {
@@ -4772,7 +4390,7 @@ impl Parser {
     }
 
     /// Parse object type and name after `show create`.
-    pub fn parse_show_create(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_show_create(&mut self) -> PResult<Statement> {
         if let Token::Word(w) = self.next_token().token {
             let show_type = match w.keyword {
                 Keyword::TABLE => ShowCreateType::Table,
@@ -4780,7 +4398,7 @@ impl Parser {
                     if self.parse_keyword(Keyword::VIEW) {
                         ShowCreateType::MaterializedView
                     } else {
-                        return self.expected("VIEW after MATERIALIZED", self.peek_token());
+                        return self.expected("VIEW after MATERIALIZED");
                     }
                 }
                 Keyword::VIEW => ShowCreateType::View,
@@ -4791,7 +4409,6 @@ impl Parser {
                 Keyword::FUNCTION => ShowCreateType::Function,
                 _ => return self.expected(
                     "TABLE, MATERIALIZED VIEW, VIEW, INDEX, FUNCTION, SOURCE, SUBSCRIPTION or SINK",
-                    self.peek_token(),
                 ),
             };
             return Ok(Statement::ShowCreateObject {
@@ -4801,13 +4418,10 @@ impl Parser {
         }
         self.expected(
             "TABLE, MATERIALIZED VIEW, VIEW, INDEX, FUNCTION, SOURCE, SUBSCRIPTION or SINK",
-            self.peek_token(),
         )
     }
 
-    pub fn parse_show_statement_filter(
-        &mut self,
-    ) -> Result<Option<ShowStatementFilter>, ParserError> {
+    pub fn parse_show_statement_filter(&mut self) -> PResult<Option<ShowStatementFilter>> {
         if self.parse_keyword(Keyword::LIKE) {
             Ok(Some(ShowStatementFilter::Like(
                 self.parse_literal_string()?,
@@ -4823,7 +4437,7 @@ impl Parser {
         }
     }
 
-    pub fn parse_table_and_joins(&mut self) -> Result<TableWithJoins, ParserError> {
+    pub fn parse_table_and_joins(&mut self) -> PResult<TableWithJoins> {
         let relation = self.parse_table_factor()?;
 
         // Note that for keywords to be properly handled here, they need to be
@@ -4835,7 +4449,7 @@ impl Parser {
                 let join_operator = if self.parse_keyword(Keyword::JOIN) {
                     JoinOperator::CrossJoin
                 } else {
-                    return self.expected("JOIN after CROSS", self.peek_token());
+                    return self.expected("JOIN after CROSS");
                 };
                 Join {
                     relation: self.parse_table_factor()?,
@@ -4867,10 +4481,10 @@ impl Parser {
                         }
                     }
                     Keyword::OUTER => {
-                        return self.expected("LEFT, RIGHT, or FULL", self.peek_token());
+                        return self.expected("LEFT, RIGHT, or FULL");
                     }
                     _ if natural => {
-                        return self.expected("a join type after NATURAL", self.peek_token());
+                        return self.expected("a join type after NATURAL");
                     }
                     _ => break,
                 };
@@ -4878,7 +4492,7 @@ impl Parser {
                 let join_constraint = self.parse_join_constraint(natural)?;
                 let join_operator = join_operator_type(join_constraint);
                 if let JoinOperator::Inner(JoinConstraint::None) = join_operator {
-                    return self.expected("join constraint after INNER JOIN", self.peek_token());
+                    return self.expected("join constraint after INNER JOIN");
                 }
                 Join {
                     relation,
@@ -4891,11 +4505,11 @@ impl Parser {
     }
 
     /// A table name or a parenthesized subquery, followed by optional `[AS] alias`
-    pub fn parse_table_factor(&mut self) -> Result<TableFactor, ParserError> {
+    pub fn parse_table_factor(&mut self) -> PResult<TableFactor> {
         if self.parse_keyword(Keyword::LATERAL) {
             // LATERAL must always be followed by a subquery.
             if !self.consume_token(&Token::LParen) {
-                self.expected("subquery after LATERAL", self.peek_token())?;
+                self.expected("subquery after LATERAL")?;
             }
             self.parse_derived_table_factor(Lateral)
         } else if self.consume_token(&Token::LParen) {
@@ -4955,10 +4569,10 @@ impl Parser {
             } else {
                 // The SQL spec prohibits derived tables and bare tables from
                 // appearing alone in parentheses (e.g. `FROM (mytable)`)
-                parser_err!(format!(
+                parser_err!(
                     "Expected joined table, found: {table_and_joins}, next_token: {}",
                     self.peek_token()
-                ))
+                );
             }
         } else {
             let name = self.parse_object_name()?;
@@ -4968,7 +4582,7 @@ impl Parser {
                 let (args, order_by, _variadic) = self.parse_optional_args()?;
                 // Table-valued functions do not support ORDER BY, should return error if it appears
                 if !order_by.is_empty() {
-                    return parser_err!("Table-valued functions do not support ORDER BY clauses");
+                    parser_err!("Table-valued functions do not support ORDER BY clauses");
                 }
                 let with_ordinality = self.parse_keywords(&[Keyword::WITH, Keyword::ORDINALITY]);
 
@@ -4980,17 +4594,14 @@ impl Parser {
                     with_ordinality,
                 })
             } else {
-                let as_of = self.parse_as_of()?;
+                let as_of = opt(Self::parse_as_of).parse_next(self)?;
                 let alias = self.parse_optional_table_alias(keywords::RESERVED_FOR_TABLE_ALIAS)?;
                 Ok(TableFactor::Table { name, alias, as_of })
             }
         }
     }
 
-    pub fn parse_derived_table_factor(
-        &mut self,
-        lateral: IsLateral,
-    ) -> Result<TableFactor, ParserError> {
+    pub fn parse_derived_table_factor(&mut self, lateral: IsLateral) -> PResult<TableFactor> {
         let subquery = Box::new(self.parse_query()?);
         self.expect_token(&Token::RParen)?;
         let alias = self.parse_optional_table_alias(keywords::RESERVED_FOR_TABLE_ALIAS)?;
@@ -5004,7 +4615,7 @@ impl Parser {
         })
     }
 
-    fn parse_join_constraint(&mut self, natural: bool) -> Result<JoinConstraint, ParserError> {
+    fn parse_join_constraint(&mut self, natural: bool) -> PResult<JoinConstraint> {
         if natural {
             Ok(JoinConstraint::Natural)
         } else if self.parse_keyword(Keyword::ON) {
@@ -5015,12 +4626,12 @@ impl Parser {
             Ok(JoinConstraint::Using(columns))
         } else {
             Ok(JoinConstraint::None)
-            // self.expected("ON, or USING after JOIN", self.peek_token())
+            // self.expected("ON, or USING after JOIN")
         }
     }
 
     /// Parse a GRANT statement.
-    pub fn parse_grant(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_grant(&mut self) -> PResult<Statement> {
         let (privileges, objects) = self.parse_grant_revoke_privileges_objects()?;
 
         self.expect_keyword(Keyword::TO)?;
@@ -5042,9 +4653,7 @@ impl Parser {
         })
     }
 
-    fn parse_grant_revoke_privileges_objects(
-        &mut self,
-    ) -> Result<(Privileges, GrantObjects), ParserError> {
+    fn parse_grant_revoke_privileges_objects(&mut self) -> PResult<(Privileges, GrantObjects)> {
         let privileges = if self.parse_keyword(Keyword::ALL) {
             Privileges::All {
                 with_privileges_keyword: self.parse_keyword(Keyword::PRIVILEGES),
@@ -5135,7 +4744,7 @@ impl Parser {
         Ok((privileges, objects))
     }
 
-    fn parse_grant_permission(&mut self) -> Result<(Keyword, Option<Vec<Ident>>), ParserError> {
+    fn parse_grant_permission(&mut self) -> PResult<(Keyword, Option<Vec<Ident>>)> {
         let kw = self.expect_one_of_keywords(&[
             Keyword::CONNECT,
             Keyword::CREATE,
@@ -5165,7 +4774,7 @@ impl Parser {
     }
 
     /// Parse a REVOKE statement
-    pub fn parse_revoke(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_revoke(&mut self) -> PResult<Statement> {
         let revoke_grant_option =
             self.parse_keywords(&[Keyword::GRANT, Keyword::OPTION, Keyword::FOR]);
         let (privileges, objects) = self.parse_grant_revoke_privileges_objects()?;
@@ -5180,7 +4789,7 @@ impl Parser {
         let cascade = self.parse_keyword(Keyword::CASCADE);
         let restrict = self.parse_keyword(Keyword::RESTRICT);
         if cascade && restrict {
-            return parser_err!("Cannot specify both CASCADE and RESTRICT in REVOKE");
+            parser_err!("Cannot specify both CASCADE and RESTRICT in REVOKE");
         }
 
         Ok(Statement::Revoke {
@@ -5194,7 +4803,7 @@ impl Parser {
     }
 
     /// Parse an INSERT statement
-    pub fn parse_insert(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_insert(&mut self) -> PResult<Statement> {
         self.expect_keyword(Keyword::INTO)?;
 
         let table_name = self.parse_object_name()?;
@@ -5211,7 +4820,7 @@ impl Parser {
         })
     }
 
-    pub fn parse_update(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_update(&mut self) -> PResult<Statement> {
         let table_name = self.parse_object_name()?;
 
         self.expect_keyword(Keyword::SET)?;
@@ -5231,7 +4840,7 @@ impl Parser {
     }
 
     /// Parse a `var = expr` assignment, used in an UPDATE statement
-    pub fn parse_assignment(&mut self) -> Result<Assignment, ParserError> {
+    pub fn parse_assignment(&mut self) -> PResult<Assignment> {
         let id = self.parse_identifiers_non_keywords()?;
         self.expect_token(&Token::Eq)?;
 
@@ -5245,7 +4854,7 @@ impl Parser {
     }
 
     /// Parse a `[VARIADIC] name => expr`.
-    fn parse_function_args(&mut self) -> Result<(bool, FunctionArg), ParserError> {
+    fn parse_function_args(&mut self) -> PResult<(bool, FunctionArg)> {
         let variadic = self.parse_keyword(Keyword::VARIADIC);
         let arg = if self.peek_nth_token(1) == Token::RArrow {
             let name = self.parse_identifier()?;
@@ -5260,9 +4869,7 @@ impl Parser {
         Ok((variadic, arg))
     }
 
-    pub fn parse_optional_args(
-        &mut self,
-    ) -> Result<(Vec<FunctionArg>, Vec<OrderByExpr>, bool), ParserError> {
+    pub fn parse_optional_args(&mut self) -> PResult<(Vec<FunctionArg>, Vec<OrderByExpr>, bool)> {
         if self.consume_token(&Token::RParen) {
             Ok((vec![], vec![], false))
         } else {
@@ -5272,7 +4879,7 @@ impl Parser {
                 .take(args.len() - 1)
                 .any(|(variadic, _)| *variadic)
             {
-                return parser_err!("VARIADIC argument must be last");
+                parser_err!("VARIADIC argument must be last");
             }
             let variadic = args.last().map(|(variadic, _)| *variadic).unwrap_or(false);
             let args = args.into_iter().map(|(_, arg)| arg).collect();
@@ -5288,7 +4895,7 @@ impl Parser {
     }
 
     /// Parse a comma-delimited list of projections after SELECT
-    pub fn parse_select_item(&mut self) -> Result<SelectItem, ParserError> {
+    pub fn parse_select_item(&mut self) -> PResult<SelectItem> {
         match self.parse_wildcard_or_expr()? {
             WildcardOrExpr::Expr(expr) => self
                 .parse_optional_alias(keywords::RESERVED_FOR_COLUMN_ALIAS)
@@ -5307,7 +4914,7 @@ impl Parser {
     }
 
     /// Parse an expression, optionally followed by ASC or DESC (used in ORDER BY)
-    pub fn parse_order_by_expr(&mut self) -> Result<OrderByExpr, ParserError> {
+    pub fn parse_order_by_expr(&mut self) -> PResult<OrderByExpr> {
         let expr = self.parse_expr()?;
 
         let asc = if self.parse_keyword(Keyword::ASC) {
@@ -5334,7 +4941,7 @@ impl Parser {
     }
 
     /// Parse a LIMIT clause
-    pub fn parse_limit(&mut self) -> Result<Option<String>, ParserError> {
+    pub fn parse_limit(&mut self) -> PResult<Option<String>> {
         if self.parse_keyword(Keyword::ALL) {
             Ok(None)
         } else {
@@ -5348,7 +4955,7 @@ impl Parser {
     }
 
     /// Parse an OFFSET clause
-    pub fn parse_offset(&mut self) -> Result<String, ParserError> {
+    pub fn parse_offset(&mut self) -> PResult<String> {
         let value = self.parse_number_value()?;
         // TODO(Kexiang): support LIMIT expr
         if self.consume_token(&Token::DoubleColon) {
@@ -5359,7 +4966,7 @@ impl Parser {
     }
 
     /// Parse a FETCH clause
-    pub fn parse_fetch(&mut self) -> Result<Fetch, ParserError> {
+    pub fn parse_fetch(&mut self) -> PResult<Fetch> {
         self.expect_one_of_keywords(&[Keyword::FIRST, Keyword::NEXT])?;
         let quantity = if self
             .parse_one_of_keywords(&[Keyword::ROW, Keyword::ROWS])
@@ -5376,7 +4983,7 @@ impl Parser {
         } else if self.parse_keywords(&[Keyword::WITH, Keyword::TIES]) {
             true
         } else {
-            return self.expected("one of ONLY or WITH TIES", self.peek_token());
+            return self.expected("one of ONLY or WITH TIES");
         };
         Ok(Fetch {
             with_ties,
@@ -5384,7 +4991,7 @@ impl Parser {
         })
     }
 
-    pub fn parse_values(&mut self) -> Result<Values, ParserError> {
+    pub fn parse_values(&mut self) -> PResult<Values> {
         let values = self.parse_comma_separated(|parser| {
             parser.expect_token(&Token::LParen)?;
             let exprs = parser.parse_comma_separated(Parser::parse_expr)?;
@@ -5394,21 +5001,21 @@ impl Parser {
         Ok(Values(values))
     }
 
-    pub fn parse_start_transaction(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_start_transaction(&mut self) -> PResult<Statement> {
         self.expect_keyword(Keyword::TRANSACTION)?;
         Ok(Statement::StartTransaction {
             modes: self.parse_transaction_modes()?,
         })
     }
 
-    pub fn parse_begin(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_begin(&mut self) -> PResult<Statement> {
         let _ = self.parse_one_of_keywords(&[Keyword::TRANSACTION, Keyword::WORK]);
         Ok(Statement::Begin {
             modes: self.parse_transaction_modes()?,
         })
     }
 
-    pub fn parse_transaction_modes(&mut self) -> Result<Vec<TransactionMode>, ParserError> {
+    pub fn parse_transaction_modes(&mut self) -> PResult<Vec<TransactionMode>> {
         let mut modes = vec![];
         let mut required = false;
         loop {
@@ -5422,7 +5029,7 @@ impl Parser {
                 } else if self.parse_keyword(Keyword::SERIALIZABLE) {
                     TransactionIsolationLevel::Serializable
                 } else {
-                    self.expected("isolation level", self.peek_token())?
+                    self.expected("isolation level")?
                 };
                 TransactionMode::IsolationLevel(iso_level)
             } else if self.parse_keywords(&[Keyword::READ, Keyword::ONLY]) {
@@ -5430,7 +5037,7 @@ impl Parser {
             } else if self.parse_keywords(&[Keyword::READ, Keyword::WRITE]) {
                 TransactionMode::AccessMode(TransactionAccessMode::ReadWrite)
             } else if required {
-                self.expected("transaction mode", self.peek_token())?
+                self.expected("transaction mode")?
             } else {
                 break;
             };
@@ -5444,19 +5051,19 @@ impl Parser {
         Ok(modes)
     }
 
-    pub fn parse_commit(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_commit(&mut self) -> PResult<Statement> {
         Ok(Statement::Commit {
             chain: self.parse_commit_rollback_chain()?,
         })
     }
 
-    pub fn parse_rollback(&mut self) -> Result<Statement, ParserError> {
+    pub fn parse_rollback(&mut self) -> PResult<Statement> {
         Ok(Statement::Rollback {
             chain: self.parse_commit_rollback_chain()?,
         })
     }
 
-    pub fn parse_commit_rollback_chain(&mut self) -> Result<bool, ParserError> {
+    pub fn parse_commit_rollback_chain(&mut self) -> PResult<bool> {
         let _ = self.parse_one_of_keywords(&[Keyword::TRANSACTION, Keyword::WORK]);
         if self.parse_keyword(Keyword::AND) {
             let chain = !self.parse_keyword(Keyword::NO);
@@ -5467,13 +5074,13 @@ impl Parser {
         }
     }
 
-    fn parse_deallocate(&mut self) -> Result<Statement, ParserError> {
+    fn parse_deallocate(&mut self) -> PResult<Statement> {
         let prepare = self.parse_keyword(Keyword::PREPARE);
         let name = self.parse_identifier()?;
         Ok(Statement::Deallocate { name, prepare })
     }
 
-    fn parse_execute(&mut self) -> Result<Statement, ParserError> {
+    fn parse_execute(&mut self) -> PResult<Statement> {
         let name = self.parse_identifier()?;
 
         let mut parameters = vec![];
@@ -5485,7 +5092,7 @@ impl Parser {
         Ok(Statement::Execute { name, parameters })
     }
 
-    fn parse_prepare(&mut self) -> Result<Statement, ParserError> {
+    fn parse_prepare(&mut self) -> PResult<Statement> {
         let name = self.parse_identifier()?;
 
         let mut data_types = vec![];
@@ -5503,8 +5110,9 @@ impl Parser {
         })
     }
 
-    fn parse_comment(&mut self) -> Result<Statement, ParserError> {
+    fn parse_comment(&mut self) -> PResult<Statement> {
         self.expect_keyword(Keyword::ON)?;
+        let checkpoint = *self;
         let token = self.next_token();
 
         let (object_type, object_name) = match token.token {
@@ -5516,7 +5124,7 @@ impl Parser {
                 let object_name = self.parse_object_name()?;
                 (CommentObject::Table, object_name)
             }
-            _ => self.expected("comment object_type", token)?,
+            _ => self.expected_at(checkpoint, "comment object_type")?,
         };
 
         self.expect_keyword(Keyword::IS)?;
@@ -5535,11 +5143,9 @@ impl Parser {
 
 impl Word {
     /// Convert a Word to a Identifier, return ParserError when the Word's value is a empty string.
-    pub fn to_ident(&self) -> Result<Ident, ParserError> {
+    pub fn to_ident(&self) -> PResult<Ident> {
         if self.value.is_empty() {
-            parser_err!(format!(
-                "zero-length delimited identifier at or near \"{self}\""
-            ))
+            parser_err!("zero-length delimited identifier at or near \"{self}\"")
         } else {
             Ok(Ident {
                 value: self.value.clone(),
@@ -5554,27 +5160,6 @@ mod tests {
     use super::*;
     use crate::test_utils::run_parser_method;
 
-    #[test]
-    fn test_prev_index() {
-        let sql = "SELECT version";
-        run_parser_method(sql, |parser| {
-            assert_eq!(parser.peek_token(), Token::make_keyword("SELECT"));
-            assert_eq!(parser.next_token(), Token::make_keyword("SELECT"));
-            parser.prev_token();
-            assert_eq!(parser.next_token(), Token::make_keyword("SELECT"));
-            assert_eq!(parser.next_token(), Token::make_word("version", None));
-            parser.prev_token();
-            assert_eq!(parser.peek_token(), Token::make_word("version", None));
-            assert_eq!(parser.next_token(), Token::make_word("version", None));
-            assert_eq!(parser.peek_token(), Token::EOF);
-            parser.prev_token();
-            assert_eq!(parser.next_token(), Token::make_word("version", None));
-            assert_eq!(parser.next_token(), Token::EOF);
-            assert_eq!(parser.next_token(), Token::EOF);
-            parser.prev_token();
-        });
-    }
-
     #[test]
     fn test_parse_integer_min() {
         let min_bigint = "-9223372036854775808";
diff --git a/src/sqlparser/src/parser_v2/compact.rs b/src/sqlparser/src/parser_v2/compact.rs
new file mode 100644
index 0000000000000..362e2680481aa
--- /dev/null
+++ b/src/sqlparser/src/parser_v2/compact.rs
@@ -0,0 +1,44 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Compatible layer with parser v1
+
+use winnow::{PResult, Stateful};
+
+use crate::parser as parser_v1;
+
+pub trait ParseV1 {
+    fn parse_v1<F, O>(&mut self, f: F) -> PResult<O>
+    where
+        for<'a> F: FnOnce(&mut parser_v1::Parser<'a>) -> PResult<O>;
+}
+
+impl<'a> ParseV1 for parser_v1::Parser<'a> {
+    fn parse_v1<F, O>(&mut self, f: F) -> PResult<O>
+    where
+        F: FnOnce(&mut parser_v1::Parser<'a>) -> PResult<O>,
+    {
+        f(self)
+    }
+}
+
+impl<S, State> ParseV1 for Stateful<S, State>
+where
+    S: ParseV1,
+{
+    fn parse_v1<F, O>(&mut self, f: F) -> PResult<O>
+    where
+        for<'a> F: FnOnce(&mut parser_v1::Parser<'a>) -> PResult<O>,
+    {
+        self.input.parse_v1(f)
+    }
+}
diff --git a/src/sqlparser/src/parser_v2/data_type.rs b/src/sqlparser/src/parser_v2/data_type.rs
new file mode 100644
index 0000000000000..2a66344fb0357
--- /dev/null
+++ b/src/sqlparser/src/parser_v2/data_type.rs
@@ -0,0 +1,236 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Parsers for data types.
+//!
+//! This module contains parsers for data types. To handle the anbiguity of `>>` and `> >` in struct definition,
+//! we need to use a stateful parser here. See [`with_state`] for more information.
+
+use core::cell::RefCell;
+use std::rc::Rc;
+
+use winnow::combinator::{
+    alt, cut_err, delimited, dispatch, empty, fail, opt, preceded, repeat, separated, seq,
+    terminated, trace,
+};
+use winnow::error::{ContextError, ErrMode, ErrorKind, FromExternalError, StrContext};
+use winnow::{PResult, Parser, Stateful};
+
+use super::{
+    identifier_non_reserved, keyword, literal_uint, object_name, precision_in_range, with_state,
+    TokenStream,
+};
+use crate::ast::{DataType, StructField};
+use crate::keywords::Keyword;
+use crate::tokenizer::Token;
+
+#[derive(Default, Debug)]
+struct DataTypeParsingState {
+    /// Since we can't distinguish between `>>` and `> >` in tokenizer, we need to handle this case in the parser.
+    /// When we want a [`>`][Token::Gt] but actually consumed a [`>>`][Token::ShiftRight], we set this to true.
+    /// When the value was true and we want a [`>`][Token::Gt], we just set this to false instead of really consume it.
+    remaining_close: Rc<RefCell<bool>>,
+}
+
+type StatefulStream<S> = Stateful<S, DataTypeParsingState>;
+
+/// Consume struct type definitions
+fn struct_data_type<S>(input: &mut StatefulStream<S>) -> PResult<Vec<StructField>>
+where
+    S: TokenStream,
+{
+    let remaining_close1 = input.state.remaining_close.clone();
+    let remaining_close2 = input.state.remaining_close.clone();
+
+    // Consume an abstract `>`, it may be the `remaining_close1` flag set by previous `>>`.
+    let consume_close = trace(
+        "consume_struct_close",
+        alt((
+            trace(
+                "consume_remaining_close",
+                move |input: &mut StatefulStream<S>| -> PResult<()> {
+                    if *remaining_close1.borrow() {
+                        *remaining_close1.borrow_mut() = false;
+                        Ok(())
+                    } else {
+                        fail(input)
+                    }
+                },
+            )
+            .void(),
+            trace(
+                "produce_remaining_close",
+                (
+                    Token::ShiftRight,
+                    move |_input: &mut StatefulStream<S>| -> PResult<()> {
+                        *remaining_close2.borrow_mut() = true;
+                        Ok(())
+                    },
+                )
+                    .void(),
+            ),
+            Token::Gt.void(),
+        )),
+    );
+
+    // If there is an `over-consumed' `>`, we shouldn't handle `,`.
+    let sep = |input: &mut StatefulStream<S>| -> PResult<()> {
+        if *input.state.remaining_close.borrow() {
+            fail(input)
+        } else {
+            Token::Comma.void().parse_next(input)
+        }
+    };
+
+    delimited(
+        Token::Lt,
+        cut_err(separated(
+            1..,
+            trace(
+                "struct_field",
+                seq! {
+                    StructField {
+                        name: identifier_non_reserved,
+                        data_type: data_type_stateful,
+                    }
+                },
+            ),
+            sep,
+        )),
+        cut_err(consume_close),
+    )
+    .context(StrContext::Label("struct_data_type"))
+    .parse_next(input)
+}
+
+/// Consume a data type definition.
+///
+/// The parser is the main entry point for data type parsing.
+pub fn data_type<S>(input: &mut S) -> PResult<DataType>
+where
+    S: TokenStream,
+{
+    #[derive(Debug, thiserror::Error)]
+    #[error("unconsumed `>>`")]
+    struct UnconsumedShiftRight;
+
+    with_state::<S, DataTypeParsingState, _, _>(terminated(
+        data_type_stateful,
+        trace("data_type_verify_state", |input: &mut StatefulStream<S>| {
+            // If there is remaining `>`, we should fail.
+            if *input.state.remaining_close.borrow() {
+                Err(ErrMode::Cut(ContextError::from_external_error(
+                    input,
+                    ErrorKind::Fail,
+                    UnconsumedShiftRight,
+                )))
+            } else {
+                Ok(())
+            }
+        }),
+    ))
+    .context(StrContext::Label("data_type"))
+    .parse_next(input)
+}
+
+/// Data type parsing with stateful stream.
+fn data_type_stateful<S>(input: &mut StatefulStream<S>) -> PResult<DataType>
+where
+    S: TokenStream,
+{
+    (
+        data_type_stateful_inner,
+        repeat(0.., (Token::LBracket, cut_err(Token::RBracket))),
+    )
+        .map(|(mut dt, depth)| {
+            let depth: usize = depth;
+            for _ in 0..depth {
+                dt = DataType::Array(Box::new(dt));
+            }
+            dt
+        })
+        .parse_next(input)
+}
+
+/// Consume a data type except [`DataType::Array`].
+fn data_type_stateful_inner<S>(input: &mut StatefulStream<S>) -> PResult<DataType>
+where
+    S: TokenStream,
+{
+    let with_time_zone = || {
+        opt(alt((
+            (Keyword::WITH, Keyword::TIME, Keyword::ZONE).value(true),
+            (Keyword::WITHOUT, Keyword::TIME, Keyword::ZONE).value(false),
+        )))
+        .map(|x| x.unwrap_or(false))
+    };
+
+    let precision_and_scale = || {
+        opt(delimited(
+            Token::LParen,
+            (literal_uint, opt(preceded(Token::Comma, literal_uint))),
+            Token::RParen,
+        ))
+        .map(|p| match p {
+            Some((x, y)) => (Some(x), y),
+            None => (None, None),
+        })
+    };
+
+    let keywords = dispatch! {keyword;
+        Keyword::BOOLEAN | Keyword::BOOL => empty.value(DataType::Boolean),
+        Keyword::FLOAT => opt(precision_in_range(1..54)).map(DataType::Float),
+        Keyword::REAL => empty.value(DataType::Real),
+        Keyword::DOUBLE => opt(Keyword::PRECISION).value(DataType::Double),
+        Keyword::SMALLINT => empty.value(DataType::SmallInt),
+        Keyword::INT | Keyword::INTEGER => empty.value(DataType::Int),
+        Keyword::BIGINT => empty.value(DataType::BigInt),
+        Keyword::STRING | Keyword::VARCHAR => empty.value(DataType::Varchar),
+        Keyword::CHAR | Keyword::CHARACTER => dispatch! {keyword;
+            Keyword::VARYING => empty.value(DataType::Varchar),
+            _ => opt(precision_in_range(..)).map(DataType::Char),
+        },
+        Keyword::UUID => empty.value(DataType::Uuid),
+        Keyword::DATE => empty.value(DataType::Date),
+        Keyword::TIMESTAMP => with_time_zone().map(DataType::Timestamp),
+        Keyword::TIME => with_time_zone().map(DataType::Time),
+        // TODO: Support complex interval type parsing.
+        Keyword::INTERVAL => empty.value(DataType::Interval),
+        Keyword::TEXT => empty.value(DataType::Text),
+        Keyword::STRUCT => cut_err(struct_data_type).map(DataType::Struct),
+        Keyword::BYTEA => empty.value(DataType::Bytea),
+        Keyword::NUMERIC | Keyword::DECIMAL | Keyword::DEC => cut_err(precision_and_scale()).map(|(precision, scale)| {
+            DataType::Decimal(precision, scale)
+        }),
+        _ =>  fail,
+    };
+
+    trace(
+        "data_type_inner",
+        alt((
+            keywords,
+            trace(
+                "non_keyword_data_type",
+                object_name.map(
+                    |name| match name.to_string().to_ascii_lowercase().as_str() {
+                        // PostgreSQL built-in data types that are not keywords.
+                        "jsonb" => DataType::Jsonb,
+                        "regclass" => DataType::Regclass,
+                        "regproc" => DataType::Regproc,
+                        _ => DataType::Custom(name),
+                    },
+                ),
+            ),
+        )),
+    )
+    .parse_next(input)
+}
diff --git a/src/sqlparser/src/parser_v2/expr.rs b/src/sqlparser/src/parser_v2/expr.rs
new file mode 100644
index 0000000000000..2923919ac83b9
--- /dev/null
+++ b/src/sqlparser/src/parser_v2/expr.rs
@@ -0,0 +1,196 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+use winnow::combinator::{alt, cut_err, opt, preceded, repeat, seq, trace};
+use winnow::error::ContextError;
+use winnow::{PResult, Parser};
+
+use super::{data_type, token, ParserExt, TokenStream};
+use crate::ast::Expr;
+use crate::keywords::Keyword;
+use crate::parser::Precedence;
+use crate::tokenizer::Token;
+
+fn expr_parse<S>(input: &mut S) -> PResult<Expr>
+where
+    S: TokenStream,
+{
+    // TODO: implement this function using combinator style.
+    trace("expr", |input: &mut S| {
+        input.parse_v1(|parser| parser.parse_expr())
+    })
+    .parse_next(input)
+}
+
+fn subexpr<S>(precedence: Precedence) -> impl Parser<S, Expr, ContextError>
+where
+    S: TokenStream,
+{
+    // TODO: implement this function using combinator style.
+    trace("subexpr", move |input: &mut S| {
+        input.parse_v1(|parser| parser.parse_subexpr(precedence))
+    })
+}
+
+pub fn expr_case<S>(input: &mut S) -> PResult<Expr>
+where
+    S: TokenStream,
+{
+    let parse = (
+        opt(expr_parse),
+        repeat(
+            1..,
+            (
+                Keyword::WHEN,
+                cut_err(expr_parse),
+                cut_err(Keyword::THEN),
+                cut_err(expr_parse),
+            ),
+        ),
+        opt(preceded(Keyword::ELSE, cut_err(expr_parse))),
+        cut_err(Keyword::END),
+    )
+        .map(|(operand, branches, else_result, _)| {
+            let branches: Vec<_> = branches;
+            let (conditions, results) = branches.into_iter().map(|(_, c, _, t)| (c, t)).unzip();
+            Expr::Case {
+                operand: operand.map(Box::new),
+                conditions,
+                results,
+                else_result: else_result.map(Box::new),
+            }
+        });
+
+    trace("expr_case", parse).parse_next(input)
+}
+
+/// Consume a SQL CAST function e.g. `CAST(expr AS FLOAT)`
+pub fn expr_cast<S>(input: &mut S) -> PResult<Expr>
+where
+    S: TokenStream,
+{
+    let parse = cut_err(seq! {Expr::Cast {
+        _: Token::LParen,
+        expr: expr_parse.map(Box::new),
+        _: Keyword::AS,
+        data_type: data_type,
+        _: Token::RParen,
+    }});
+
+    trace("expr_cast", parse).parse_next(input)
+}
+
+/// Consume a SQL TRY_CAST function e.g. `TRY_CAST(expr AS FLOAT)`
+pub fn expr_try_cast<S>(input: &mut S) -> PResult<Expr>
+where
+    S: TokenStream,
+{
+    let parse = cut_err(seq! {Expr::TryCast {
+        _: Token::LParen,
+        expr: expr_parse.map(Box::new),
+        _: Keyword::AS,
+        data_type: data_type,
+        _: Token::RParen,
+    }});
+
+    trace("expr_try_cast", parse).parse_next(input)
+}
+
+/// Consume a SQL EXTRACT function e.g. `EXTRACT(YEAR FROM expr)`
+pub fn expr_extract<S>(input: &mut S) -> PResult<Expr>
+where
+    S: TokenStream,
+{
+    let mut date_time_field = token
+        .verify_map(|token| match token.token {
+            Token::Word(w) => Some(w.value.to_uppercase()),
+            Token::SingleQuotedString(s) => Some(s.to_uppercase()),
+            _ => None,
+        })
+        .expect("date/time field");
+
+    let parse = cut_err(seq! {Expr::Extract {
+        _: Token::LParen,
+        field: date_time_field,
+        _: Keyword::FROM,
+        expr: expr_parse.map(Box::new),
+        _: Token::RParen,
+    }});
+
+    trace("expr_extract", parse).parse_next(input)
+}
+
+/// Consume `SUBSTRING (EXPR [FROM 1] [FOR 3])`
+pub fn expr_substring<S>(input: &mut S) -> PResult<Expr>
+where
+    S: TokenStream,
+{
+    let mut substring_from = opt(preceded(
+        alt((Token::Comma.void(), Keyword::FROM.void())),
+        cut_err(expr_parse).map(Box::new),
+    ));
+    let mut substring_for = opt(preceded(
+        alt((Token::Comma.void(), Keyword::FOR.void())),
+        cut_err(expr_parse).map(Box::new),
+    ));
+    let parse = cut_err(seq! {Expr::Substring {
+        _: Token::LParen,
+        expr: expr_parse.map(Box::new),
+        substring_from: substring_from,
+        substring_for: substring_for,
+        _: Token::RParen,
+    }});
+
+    trace("expr_substring", parse).parse_next(input)
+}
+
+/// `POSITION(<expr> IN <expr>)`
+pub fn expr_position<S>(input: &mut S) -> PResult<Expr>
+where
+    S: TokenStream,
+{
+    let parse = cut_err(seq! {Expr::Position {
+        _: Token::LParen,
+        // Logically `parse_expr`, but limited to those with precedence higher than `BETWEEN`/`IN`,
+        // to avoid conflict with general IN operator, for example `position(a IN (b) IN (c))`.
+        // https://github.com/postgres/postgres/blob/REL_15_2/src/backend/parser/gram.y#L16012
+        substring: subexpr(Precedence::Between).map(Box::new),
+        _: Keyword::IN,
+        string: subexpr(Precedence::Between).map(Box::new),
+        _: Token::RParen,
+    }});
+
+    trace("expr_position", parse).parse_next(input)
+}
+
+/// `OVERLAY(<expr> PLACING <expr> FROM <expr> [ FOR <expr> ])`
+pub fn expr_overlay<S>(input: &mut S) -> PResult<Expr>
+where
+    S: TokenStream,
+{
+    let mut count_parse = opt(preceded(
+        Keyword::FOR.void(),
+        cut_err(expr_parse).map(Box::new),
+    ));
+
+    let parse = cut_err(seq! {Expr::Overlay {
+        _: Token::LParen,
+        expr: expr_parse.map(Box::new),
+        _: Keyword::PLACING,
+        new_substring: expr_parse.map(Box::new),
+        _: Keyword::FROM,
+        start: expr_parse.map(Box::new),
+        count: count_parse,
+        _: Token::RParen,
+    }});
+
+    trace("expr_overlay", parse).parse_next(input)
+}
diff --git a/src/sqlparser/src/parser_v2/impl_.rs b/src/sqlparser/src/parser_v2/impl_.rs
new file mode 100644
index 0000000000000..e4b6af2919745
--- /dev/null
+++ b/src/sqlparser/src/parser_v2/impl_.rs
@@ -0,0 +1,143 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use winnow::stream::{Checkpoint, Offset, SliceLen, Stream, StreamIsPartial, UpdateSlice};
+
+use crate::parser::Parser;
+use crate::tokenizer::TokenWithLocation;
+
+#[derive(Copy, Clone, Debug)]
+pub struct CheckpointWrapper<'a>(Checkpoint<&'a [TokenWithLocation], &'a [TokenWithLocation]>);
+
+impl<'a> Offset<CheckpointWrapper<'a>> for CheckpointWrapper<'a> {
+    #[inline(always)]
+    fn offset_from(&self, start: &Self) -> usize {
+        self.0.offset_from(&start.0)
+    }
+}
+
+// Used for diagnostics with `--features winnow/debug`.
+impl<'a> std::fmt::Debug for Parser<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if let Some(token) = self.0.first() {
+            write!(f, "{}", token.token)?;
+        }
+        for token in self.0.iter().skip(1) {
+            write!(f, " {}", token.token)?;
+        }
+        Ok(())
+    }
+}
+
+impl<'a> Offset<Parser<'a>> for Parser<'a> {
+    #[inline(always)]
+    fn offset_from(&self, start: &Self) -> usize {
+        self.0.offset_from(&start.0)
+    }
+}
+
+impl<'a> Offset<CheckpointWrapper<'a>> for Parser<'a> {
+    #[inline(always)]
+    fn offset_from(&self, start: &CheckpointWrapper<'a>) -> usize {
+        self.0.offset_from(&start.0)
+    }
+}
+
+impl<'a> SliceLen for Parser<'a> {
+    #[inline(always)]
+    fn slice_len(&self) -> usize {
+        self.0.len()
+    }
+}
+
+impl<'a> StreamIsPartial for Parser<'a> {
+    type PartialState = <&'a [TokenWithLocation] as StreamIsPartial>::PartialState;
+
+    #[must_use]
+    #[inline(always)]
+    fn complete(&mut self) -> Self::PartialState {
+        self.0.complete()
+    }
+
+    #[inline(always)]
+    fn restore_partial(&mut self, state: Self::PartialState) {
+        self.0.restore_partial(state)
+    }
+
+    #[inline(always)]
+    fn is_partial_supported() -> bool {
+        <&'a [TokenWithLocation] as StreamIsPartial>::is_partial_supported()
+    }
+}
+
+impl<'a> Stream for Parser<'a> {
+    type Checkpoint = CheckpointWrapper<'a>;
+    type IterOffsets = <&'a [TokenWithLocation] as Stream>::IterOffsets;
+    type Slice = Parser<'a>;
+    type Token = <&'a [TokenWithLocation] as Stream>::Token;
+
+    #[inline(always)]
+    fn iter_offsets(&self) -> Self::IterOffsets {
+        self.0.iter_offsets()
+    }
+
+    #[inline(always)]
+    fn eof_offset(&self) -> usize {
+        self.0.eof_offset()
+    }
+
+    #[inline(always)]
+    fn next_token(&mut self) -> Option<Self::Token> {
+        self.0.next_token()
+    }
+
+    #[inline(always)]
+    fn offset_for<P>(&self, predicate: P) -> Option<usize>
+    where
+        P: Fn(Self::Token) -> bool,
+    {
+        self.0.offset_for(predicate)
+    }
+
+    #[inline(always)]
+    fn offset_at(&self, tokens: usize) -> Result<usize, winnow::error::Needed> {
+        self.0.offset_at(tokens)
+    }
+
+    #[inline(always)]
+    fn next_slice(&mut self, offset: usize) -> Self::Slice {
+        Parser(self.0.next_slice(offset))
+    }
+
+    #[inline(always)]
+    fn checkpoint(&self) -> Self::Checkpoint {
+        CheckpointWrapper(self.0.checkpoint())
+    }
+
+    #[inline(always)]
+    fn reset(&mut self, checkpoint: &Self::Checkpoint) {
+        self.0.reset(&checkpoint.0)
+    }
+
+    #[inline(always)]
+    fn raw(&self) -> &dyn std::fmt::Debug {
+        // We customized the `Debug` implementation in the wrapper, so don't return `self.0` here.
+        self
+    }
+}
+
+impl<'a> UpdateSlice for Parser<'a> {
+    #[inline(always)]
+    fn update_slice(self, inner: Self::Slice) -> Self {
+        Parser(self.0.update_slice(inner.0))
+    }
+}
diff --git a/src/sqlparser/src/parser_v2/mod.rs b/src/sqlparser/src/parser_v2/mod.rs
new file mode 100644
index 0000000000000..729b56a51253a
--- /dev/null
+++ b/src/sqlparser/src/parser_v2/mod.rs
@@ -0,0 +1,182 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use winnow::combinator::{separated, trace, Context};
+use winnow::error::{AddContext, ContextError, StrContext};
+use winnow::stream::{Stream, StreamIsPartial};
+use winnow::token::any;
+use winnow::{PResult, Parser, Stateful};
+
+use crate::ast::{Ident, ObjectName};
+use crate::keywords::{self, Keyword};
+use crate::tokenizer::{Token, TokenWithLocation};
+
+mod compact;
+mod data_type;
+mod expr;
+mod impl_;
+mod number;
+
+pub(crate) use data_type::*;
+pub(crate) use expr::*;
+pub(crate) use number::*;
+
+/// Bundle trait requirements from winnow, so that we don't need to write them everywhere.
+///
+/// All combinators should accept a generic `S` that implements `TokenStream`.
+pub trait TokenStream:
+    Stream<Token = TokenWithLocation> + StreamIsPartial + Default + compact::ParseV1
+{
+}
+
+impl<S> TokenStream for S where
+    S: Stream<Token = TokenWithLocation> + StreamIsPartial + Default + compact::ParseV1
+{
+}
+
+/// Consume any token.
+///
+/// If you need to consume a specific token, use [`Token::?`][Token] directly, which already implements [`Parser`].
+fn token<S>(input: &mut S) -> PResult<TokenWithLocation>
+where
+    S: TokenStream,
+{
+    any(input)
+}
+
+/// Consume a keyword.
+///
+/// If you need to consume a specific keyword, use [`Keyword::?`][Keyword] directly, which already implements [`Parser`].
+pub fn keyword<S>(input: &mut S) -> PResult<Keyword>
+where
+    S: TokenStream,
+{
+    trace(
+        "keyword",
+        token.verify_map(|t| match &t.token {
+            Token::Word(w) if w.keyword != Keyword::NoKeyword => Some(w.keyword),
+            _ => None,
+        }),
+    )
+    .context(StrContext::Label("keyword"))
+    .parse_next(input)
+}
+
+impl<I> Parser<I, TokenWithLocation, ContextError> for Token
+where
+    I: TokenStream,
+{
+    fn parse_next(&mut self, input: &mut I) -> PResult<TokenWithLocation, ContextError> {
+        trace(
+            format_args!("token {}", self.clone()),
+            token.verify(move |t: &TokenWithLocation| t.token == *self),
+        )
+        .parse_next(input)
+    }
+}
+
+impl<I> Parser<I, Keyword, ContextError> for Keyword
+where
+    I: TokenStream,
+{
+    fn parse_next(&mut self, input: &mut I) -> PResult<Keyword, ContextError> {
+        trace(
+            format_args!("keyword {}", self.clone()),
+            token.verify_map(move |t| match &t.token {
+                Token::Word(w) if *self == w.keyword => Some(w.keyword),
+                _ => None,
+            }),
+        )
+        .parse_next(input)
+    }
+}
+
+/// Consume an identifier that is not a reserved keyword.
+fn identifier_non_reserved<S>(input: &mut S) -> PResult<Ident>
+where
+    S: TokenStream,
+{
+    // FIXME: Reporting error correctly.
+    token
+        .verify_map(|t| match &t.token {
+            Token::Word(w) if !keywords::RESERVED_FOR_COLUMN_OR_TABLE_NAME.contains(&w.keyword) => {
+                w.to_ident().ok()
+            }
+            _ => None,
+        })
+        .parse_next(input)
+}
+
+/// Consume an 'single-quoted string'.
+pub fn single_quoted_string<S>(input: &mut S) -> PResult<String>
+where
+    S: TokenStream,
+{
+    token
+        .verify_map(|t| match &t.token {
+            Token::SingleQuotedString(s) => Some(s.clone()),
+            _ => None,
+        })
+        .parse_next(input)
+}
+
+/// Consume an object name.
+///
+/// FIXME: Object name is extremely complex, we only handle a subset here.
+fn object_name<S>(input: &mut S) -> PResult<ObjectName>
+where
+    S: TokenStream,
+{
+    separated(1.., identifier_non_reserved, Token::Period)
+        .map(ObjectName)
+        .parse_next(input)
+}
+
+/// Accept a subparser contains a given state.
+///
+/// The state will be constructed using [`Default::default()`].
+fn with_state<S, State, O, ParseNext>(mut parse_next: ParseNext) -> impl Parser<S, O, ContextError>
+where
+    S: TokenStream,
+    State: Default,
+    ParseNext: Parser<Stateful<S, State>, O, ContextError>,
+{
+    move |input: &mut S| -> PResult<O> {
+        let state = State::default();
+        let input2 = std::mem::take(input);
+        let mut stateful = Stateful {
+            input: input2,
+            state,
+        };
+        let output = parse_next.parse_next(&mut stateful);
+        *input = stateful.input;
+        output
+    }
+}
+
+pub trait ParserExt<I, O, E>: Parser<I, O, E> {
+    /// Add a context to the error message.
+    ///
+    /// This is a shorthand for `context(StrContext::Expected(StrContextValue::Description(expected)))`.
+    fn expect(self, expected: &'static str) -> Context<Self, I, O, E, StrContext>
+    where
+        Self: Sized,
+        I: Stream,
+        E: AddContext<I, StrContext>,
+    {
+        self.context(StrContext::Expected(
+            winnow::error::StrContextValue::Description(expected),
+        ))
+    }
+}
+
+impl<I, O, E, T> ParserExt<I, O, E> for T where T: Parser<I, O, E> {}
diff --git a/src/sqlparser/src/parser_v2/number.rs b/src/sqlparser/src/parser_v2/number.rs
new file mode 100644
index 0000000000000..efa5cdb3a314d
--- /dev/null
+++ b/src/sqlparser/src/parser_v2/number.rs
@@ -0,0 +1,96 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use core::ops::RangeBounds;
+
+use winnow::combinator::{cut_err, delimited};
+use winnow::error::{ContextError, StrContext};
+use winnow::{PResult, Parser};
+
+use super::{token, TokenStream};
+use crate::tokenizer::Token;
+
+/// Consume a [number][Token::Number] from token.
+pub fn token_number<S>(input: &mut S) -> PResult<String>
+where
+    S: TokenStream,
+{
+    token
+        .verify_map(|t| {
+            if let Token::Number(number) = t.token {
+                Some(number)
+            } else {
+                None
+            }
+        })
+        .context(StrContext::Label("number"))
+        .parse_next(input)
+}
+
+/// Consume an unsigned literal integer/long
+pub fn literal_uint<S>(input: &mut S) -> PResult<u64>
+where
+    S: TokenStream,
+{
+    token_number
+        .try_map(|s| s.parse::<u64>())
+        .context(StrContext::Label("u64"))
+        .parse_next(input)
+}
+
+/// Consume an unsigned literal integer
+pub fn literal_u32<S>(input: &mut S) -> PResult<u32>
+where
+    S: TokenStream,
+{
+    token_number
+        .try_map(|s| s.parse::<u32>())
+        .context(StrContext::Label("u32"))
+        .parse_next(input)
+}
+
+/// Consume an literal integer
+pub fn literal_i64<S>(input: &mut S) -> PResult<i64>
+where
+    S: TokenStream,
+{
+    token_number
+        .try_map(|s| s.parse::<i64>())
+        .context(StrContext::Label("i64"))
+        .parse_next(input)
+}
+
+/// Consume a precision definition in some types, e.g. `FLOAT(32)`.
+///
+/// The precision must be in the given range.
+pub fn precision_in_range<S>(
+    range: impl RangeBounds<u64> + std::fmt::Debug,
+) -> impl Parser<S, u64, ContextError>
+where
+    S: TokenStream,
+{
+    #[derive(Debug, thiserror::Error)]
+    #[error("precision must be in range {0}")]
+    struct OutOfRange(String);
+
+    delimited(
+        Token::LParen,
+        cut_err(literal_uint.try_map(move |v| {
+            if range.contains(&v) {
+                Ok(v)
+            } else {
+                Err(OutOfRange(format!("{:?}", range)))
+            }
+        })),
+        cut_err(Token::RParen),
+    )
+}
diff --git a/src/sqlparser/src/test_utils.rs b/src/sqlparser/src/test_utils.rs
index 57ff2d0f0efb8..bfdf04dcc2d39 100644
--- a/src/sqlparser/src/test_utils.rs
+++ b/src/sqlparser/src/test_utils.rs
@@ -31,11 +31,11 @@ use crate::tokenizer::Tokenizer;
 
 pub fn run_parser_method<F, T: Debug + PartialEq>(sql: &str, f: F) -> T
 where
-    F: Fn(&mut Parser) -> T,
+    F: Fn(&mut Parser<'_>) -> T,
 {
     let mut tokenizer = Tokenizer::new(sql);
     let tokens = tokenizer.tokenize_with_location().unwrap();
-    f(&mut Parser::new(tokens))
+    f(&mut Parser(&tokens))
 }
 
 pub fn parse_sql_statements(sql: &str) -> Result<Vec<Statement>, ParserError> {
diff --git a/src/sqlparser/src/tokenizer.rs b/src/sqlparser/src/tokenizer.rs
index 03b794b06c62b..2a361a1d5230e 100644
--- a/src/sqlparser/src/tokenizer.rs
+++ b/src/sqlparser/src/tokenizer.rs
@@ -368,8 +368,8 @@ impl TokenWithLocation {
         }
     }
 
-    pub fn wrap(token: Token) -> TokenWithLocation {
-        TokenWithLocation::new(token, 0, 0)
+    pub fn eof() -> TokenWithLocation {
+        TokenWithLocation::new(Token::EOF, 0, 0)
     }
 }
 
@@ -388,11 +388,11 @@ impl PartialEq<TokenWithLocation> for Token {
 impl fmt::Display for TokenWithLocation {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         if self.token == Token::EOF {
-            write!(f, "EOF at the end")
+            write!(f, "end of input")
         } else {
             write!(
                 f,
-                "{} at line:{}, column:{}",
+                "{} at line {}, column {}",
                 self.token, self.location.line, self.location.column
             )
         }
@@ -405,14 +405,15 @@ pub struct TokenizerError {
     pub message: String,
     pub line: u64,
     pub col: u64,
+    pub context: String,
 }
 
 impl fmt::Display for TokenizerError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(
             f,
-            "{} at Line: {}, Column {}",
-            self.message, self.line, self.col
+            "{} at line {}, column {}\n{}",
+            self.message, self.line, self.col, self.context
         )
     }
 }
@@ -422,7 +423,8 @@ impl std::error::Error for TokenizerError {}
 
 /// SQL Tokenizer
 pub struct Tokenizer<'a> {
-    query: &'a str,
+    sql: &'a str,
+    chars: Peekable<Chars<'a>>,
     line: u64,
     col: u64,
 }
@@ -431,88 +433,116 @@ impl<'a> Tokenizer<'a> {
     /// Create a new SQL tokenizer for the specified SQL statement
     pub fn new(query: &'a str) -> Self {
         Self {
-            query,
+            sql: query,
+            chars: query.chars().peekable(),
             line: 1,
             col: 1,
         }
     }
 
-    /// Tokenize the statement and produce a vector of tokens with locations.
-    pub fn tokenize_with_location(&mut self) -> Result<Vec<TokenWithLocation>, TokenizerError> {
-        let mut peekable = self.query.chars().peekable();
-
-        let mut tokens: Vec<TokenWithLocation> = vec![];
-
-        while let Some(token) = self.next_token(&mut peekable)? {
-            match &token {
-                Token::Whitespace(Whitespace::Newline) => {
+    /// Consume the next character.
+    fn next(&mut self) -> Option<char> {
+        let ch = self.chars.next();
+        if let Some(ch) = ch {
+            match ch {
+                '\n' => {
                     self.line += 1;
                     self.col = 1;
                 }
-
-                Token::Whitespace(Whitespace::Tab) => self.col += 4,
-                Token::Word(w) if w.quote_style.is_none() => self.col += w.value.len() as u64,
-                Token::Word(w) if w.quote_style.is_some() => self.col += w.value.len() as u64 + 2,
-                Token::Number(s) => self.col += s.len() as u64,
-                Token::SingleQuotedString(s) => self.col += s.len() as u64,
+                '\t' => self.col += 4,
                 _ => self.col += 1,
             }
+        }
+        ch
+    }
 
-            let token_with_location = TokenWithLocation::new(token, self.line, self.col);
+    /// Return the next character without consuming it.
+    fn peek(&mut self) -> Option<char> {
+        self.chars.peek().cloned()
+    }
+
+    /// Tokenize the statement and produce a vector of tokens with locations.
+    ///
+    /// Whitespaces are skipped.
+    pub fn tokenize_with_location(&mut self) -> Result<Vec<TokenWithLocation>, TokenizerError> {
+        let tokens = self.tokenize()?;
+        Ok(tokens
+            .into_iter()
+            .filter(|token| !matches!(&token.token, Token::Whitespace(_)))
+            .collect())
+    }
+
+    /// Tokenize the statement and produce a vector of tokens.
+    ///
+    /// Whitespaces are included.
+    #[allow(dead_code)]
+    fn tokenize_with_whitespace(&mut self) -> Result<Vec<Token>, TokenizerError> {
+        let tokens = self.tokenize()?;
+        Ok(tokens.into_iter().map(|t| t.token).collect())
+    }
 
-            tokens.push(token_with_location);
+    /// Tokenize the statement and produce a vector of tokens.
+    ///
+    /// Whitespaces are included.
+    fn tokenize(&mut self) -> Result<Vec<TokenWithLocation>, TokenizerError> {
+        let mut tokens = Vec::new();
+        while let Some(token) = self.next_token_with_location()? {
+            tokens.push(token);
         }
         Ok(tokens)
     }
 
-    /// Tokenize the statement and produce a vector of tokens without locations.
-    #[allow(dead_code)]
-    fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError> {
-        self.tokenize_with_location()
-            .map(|v| v.into_iter().map(|t| t.token).collect())
+    /// Get the next token or return None
+    fn next_token_with_location(&mut self) -> Result<Option<TokenWithLocation>, TokenizerError> {
+        let loc = Location {
+            line: self.line,
+            column: self.col,
+        };
+        self.next_token()
+            .map(|t| t.map(|token| token.with_location(loc)))
     }
 
     /// Get the next token or return None
-    fn next_token(&self, chars: &mut Peekable<Chars<'_>>) -> Result<Option<Token>, TokenizerError> {
-        match chars.peek() {
-            Some(&ch) => match ch {
-                ' ' => self.consume_and_return(chars, Token::Whitespace(Whitespace::Space)),
-                '\t' => self.consume_and_return(chars, Token::Whitespace(Whitespace::Tab)),
-                '\n' => self.consume_and_return(chars, Token::Whitespace(Whitespace::Newline)),
+    fn next_token(&mut self) -> Result<Option<Token>, TokenizerError> {
+        match self.peek() {
+            Some(ch) => match ch {
+                ' ' => self.consume_and_return(Token::Whitespace(Whitespace::Space)),
+                '\t' => self.consume_and_return(Token::Whitespace(Whitespace::Tab)),
+                '\n' => self.consume_and_return(Token::Whitespace(Whitespace::Newline)),
                 '\r' => {
                     // Emit a single Whitespace::Newline token for \r and \r\n
-                    chars.next();
-                    if let Some('\n') = chars.peek() {
-                        chars.next();
+                    self.next();
+                    if let Some('\n') = self.peek() {
+                        self.next();
                     }
                     Ok(Some(Token::Whitespace(Whitespace::Newline)))
                 }
                 'N' => {
-                    chars.next(); // consume, to check the next char
-                    match chars.peek() {
+                    self.next(); // consume, to check the next char
+                    match self.peek() {
                         Some('\'') => {
                             // N'...' - a <national character string literal>
-                            let s = self.tokenize_single_quoted_string(chars)?;
+                            let s = self.tokenize_single_quoted_string()?;
                             Ok(Some(Token::NationalStringLiteral(s)))
                         }
                         _ => {
                             // regular identifier starting with an "N"
-                            let s = self.tokenize_word('N', chars);
+                            let s = self.tokenize_word('N');
                             Ok(Some(Token::make_word(&s, None)))
                         }
                     }
                 }
                 x @ 'e' | x @ 'E' => {
-                    chars.next(); // consume, to check the next char
-                    match chars.peek() {
+                    self.next(); // consume, to check the next char
+                    match self.peek() {
                         Some('\'') => {
                             // E'...' - a <character string literal>
-                            let s = self.tokenize_single_quoted_string_with_escape(chars)?;
+                            let s = self.tokenize_single_quoted_string_with_escape()?;
                             Ok(Some(Token::CstyleEscapesString(s)))
                         }
                         _ => {
                             // regular identifier starting with an "E"
-                            let s = self.tokenize_word(x, chars);
+                            let s = self.tokenize_word(x);
                             Ok(Some(Token::make_word(&s, None)))
                         }
                     }
@@ -520,42 +550,42 @@ impl<'a> Tokenizer<'a> {
                 // The spec only allows an uppercase 'X' to introduce a hex
                 // string, but PostgreSQL, at least, allows a lowercase 'x' too.
                 x @ 'x' | x @ 'X' => {
-                    chars.next(); // consume, to check the next char
-                    match chars.peek() {
+                    self.next(); // consume, to check the next char
+                    match self.peek() {
                         Some('\'') => {
                             // X'...' - a <binary string literal>
-                            let s = self.tokenize_single_quoted_string(chars)?;
+                            let s = self.tokenize_single_quoted_string()?;
                             Ok(Some(Token::HexStringLiteral(s)))
                         }
                         _ => {
                             // regular identifier starting with an "X"
-                            let s = self.tokenize_word(x, chars);
+                            let s = self.tokenize_word(x);
                             Ok(Some(Token::make_word(&s, None)))
                         }
                     }
                 }
                 // identifier or keyword
                 ch if is_identifier_start(ch) => {
-                    chars.next(); // consume the first char
-                    let s = self.tokenize_word(ch, chars);
+                    self.next(); // consume the first char
+                    let s = self.tokenize_word(ch);
 
                     Ok(Some(Token::make_word(&s, None)))
                 }
                 // string
                 '\'' => {
-                    let s = self.tokenize_single_quoted_string(chars)?;
+                    let s = self.tokenize_single_quoted_string()?;
 
                     Ok(Some(Token::SingleQuotedString(s)))
                 }
                 // delimited (quoted) identifier
                 quote_start if is_delimited_identifier_start(quote_start) => {
-                    chars.next(); // consume the opening quote
+                    self.next(); // consume the opening quote
                     let quote_end = Word::matching_end_quote(quote_start);
-                    let s = peeking_take_while(chars, |ch| ch != quote_end);
-                    if chars.next() == Some(quote_end) {
+                    let s = self.peeking_take_while(|ch| ch != quote_end);
+                    if self.next() == Some(quote_end) {
                         Ok(Some(Token::make_word(&s, Some(quote_start))))
                     } else {
-                        self.tokenizer_error(format!(
+                        self.error(format!(
                             "Expected close delimiter '{}' before EOF.",
                             quote_end
                         ))
@@ -563,14 +593,14 @@ impl<'a> Tokenizer<'a> {
                 }
                 // numbers and period
                 '0'..='9' | '.' => {
-                    let mut s = peeking_take_while(chars, |ch| ch.is_ascii_digit());
+                    let mut s = self.peeking_take_while(|ch| ch.is_ascii_digit());
 
                     // match binary literal that starts with 0x
                     if s == "0"
-                        && let Some(&radix) = chars.peek()
+                        && let Some(radix) = self.peek()
                         && "xob".contains(radix.to_ascii_lowercase())
                     {
-                        chars.next();
+                        self.next();
                         let radix = radix.to_ascii_lowercase();
                         let base = match radix {
                             'x' => 16,
@@ -578,67 +608,67 @@ impl<'a> Tokenizer<'a> {
                             'b' => 2,
                             _ => unreachable!(),
                         };
-                        let s2 = peeking_take_while(chars, |ch| ch.is_digit(base));
+                        let s2 = self.peeking_take_while(|ch| ch.is_digit(base));
                         if s2.is_empty() {
-                            return self.tokenizer_error("incomplete integer literal");
+                            return self.error("incomplete integer literal");
                         }
-                        self.reject_number_junk(chars)?;
+                        self.reject_number_junk()?;
                         return Ok(Some(Token::Number(format!("0{radix}{s2}"))));
                     }
 
                     // match one period
-                    if let Some('.') = chars.peek() {
+                    if let Some('.') = self.peek() {
                         s.push('.');
-                        chars.next();
+                        self.next();
                     }
-                    s += &peeking_take_while(chars, |ch| ch.is_ascii_digit());
+                    s += &self.peeking_take_while(|ch| ch.is_ascii_digit());
 
                     // No number -> Token::Period
                     if s == "." {
                         return Ok(Some(Token::Period));
                     }
 
-                    match chars.peek() {
+                    match self.peek() {
                         // Number is a scientific number (1e6)
                         Some('e') | Some('E') => {
                             s.push('e');
-                            chars.next();
+                            self.next();
 
-                            if let Some('-') = chars.peek() {
+                            if let Some('-') = self.peek() {
                                 s.push('-');
-                                chars.next();
+                                self.next();
                             }
-                            s += &peeking_take_while(chars, |ch| ch.is_ascii_digit());
-                            self.reject_number_junk(chars)?;
+                            s += &self.peeking_take_while(|ch| ch.is_ascii_digit());
+                            self.reject_number_junk()?;
                             return Ok(Some(Token::Number(s)));
                         }
                         // Not a scientific number
                         _ => {}
                     };
-                    self.reject_number_junk(chars)?;
+                    self.reject_number_junk()?;
                     Ok(Some(Token::Number(s)))
                 }
                 // punctuation
-                '(' => self.consume_and_return(chars, Token::LParen),
-                ')' => self.consume_and_return(chars, Token::RParen),
-                ',' => self.consume_and_return(chars, Token::Comma),
+                '(' => self.consume_and_return(Token::LParen),
+                ')' => self.consume_and_return(Token::RParen),
+                ',' => self.consume_and_return(Token::Comma),
                 // operators
                 '-' => {
-                    chars.next(); // consume the '-'
-                    match chars.peek() {
+                    self.next(); // consume the '-'
+                    match self.peek() {
                         Some('-') => {
-                            chars.next(); // consume the second '-', starting a single-line comment
-                            let comment = self.tokenize_single_line_comment(chars);
+                            self.next(); // consume the second '-', starting a single-line comment
+                            let comment = self.tokenize_single_line_comment();
                             Ok(Some(Token::Whitespace(Whitespace::SingleLineComment {
                                 prefix: "--".to_owned(),
                                 comment,
                             })))
                         }
                         Some('>') => {
-                            chars.next(); // consume first '>'
-                            match chars.peek() {
+                            self.next(); // consume first '>'
+                            match self.peek() {
                                 Some('>') => {
-                                    chars.next(); // consume second '>'
+                                    self.next(); // consume second '>'
                                     Ok(Some(Token::LongArrow))
                                 }
                                 _ => Ok(Some(Token::Arrow)),
@@ -649,27 +679,27 @@ impl<'a> Tokenizer<'a> {
                     }
                 }
                 '/' => {
-                    chars.next(); // consume the '/'
-                    match chars.peek() {
+                    self.next(); // consume the '/'
+                    match self.peek() {
                         Some('*') => {
-                            chars.next(); // consume the '*', starting a multi-line comment
-                            self.tokenize_multiline_comment(chars)
+                            self.next(); // consume the '*', starting a multi-line comment
+                            self.tokenize_multiline_comment()
                         }
                         // a regular '/' operator
                         _ => Ok(Some(Token::Div)),
                     }
                 }
-                '+' => self.consume_and_return(chars, Token::Plus),
-                '*' => self.consume_and_return(chars, Token::Mul),
-                '%' => self.consume_and_return(chars, Token::Mod),
+                '+' => self.consume_and_return(Token::Plus),
+                '*' => self.consume_and_return(Token::Mul),
+                '%' => self.consume_and_return(Token::Mod),
                 '|' => {
-                    chars.next(); // consume the '|'
-                    match chars.peek() {
-                        Some('/') => self.consume_and_return(chars, Token::PGSquareRoot),
+                    self.next(); // consume the '|'
+                    match self.peek() {
+                        Some('/') => self.consume_and_return(Token::PGSquareRoot),
                         Some('|') => {
-                            chars.next(); // consume the second '|'
-                            match chars.peek() {
-                                Some('/') => self.consume_and_return(chars, Token::PGCubeRoot),
+                            self.next(); // consume the second '|'
+                            match self.peek() {
+                                Some('/') => self.consume_and_return(Token::PGCubeRoot),
                                 _ => Ok(Some(Token::Concat)),
                             }
                         }
@@ -678,32 +708,32 @@ impl<'a> Tokenizer<'a> {
                     }
                 }
                 '=' => {
-                    chars.next(); // consume
-                    match chars.peek() {
-                        Some('>') => self.consume_and_return(chars, Token::RArrow),
+                    self.next(); // consume
+                    match self.peek() {
+                        Some('>') => self.consume_and_return(Token::RArrow),
                         _ => Ok(Some(Token::Eq)),
                     }
                 }
                 '!' => {
-                    chars.next(); // consume
-                    match chars.peek() {
-                        Some('=') => self.consume_and_return(chars, Token::Neq),
-                        Some('!') => self.consume_and_return(chars, Token::DoubleExclamationMark),
+                    self.next(); // consume
+                    match self.peek() {
+                        Some('=') => self.consume_and_return(Token::Neq),
+                        Some('!') => self.consume_and_return(Token::DoubleExclamationMark),
                         Some('~') => {
-                            chars.next();
-                            match chars.peek() {
+                            self.next();
+                            match self.peek() {
                                 Some('~') => {
-                                    chars.next();
-                                    match chars.peek() {
+                                    self.next();
+                                    match self.peek() {
                                         Some('*') => self.consume_and_return(
-                                            chars,
                                             Token::ExclamationMarkDoubleTildeAsterisk,
                                         ),
                                         _ => Ok(Some(Token::ExclamationMarkDoubleTilde)),
                                     }
                                 }
-                                Some('*') => self
-                                    .consume_and_return(chars, Token::ExclamationMarkTildeAsterisk),
+                                Some('*') => {
+                                    self.consume_and_return(Token::ExclamationMarkTildeAsterisk)
+                                }
                                 _ => Ok(Some(Token::ExclamationMarkTilde)),
                             }
                         }
@@ -711,76 +741,74 @@ impl<'a> Tokenizer<'a> {
                     }
                 }
                 '<' => {
-                    chars.next(); // consume
-                    match chars.peek() {
+                    self.next(); // consume
+                    match self.peek() {
                         Some('=') => {
-                            chars.next();
-                            match chars.peek() {
-                                Some('>') => self.consume_and_return(chars, Token::Spaceship),
+                            self.next();
+                            match self.peek() {
+                                Some('>') => self.consume_and_return(Token::Spaceship),
                                 _ => Ok(Some(Token::LtEq)),
                             }
                         }
-                        Some('>') => self.consume_and_return(chars, Token::Neq),
-                        Some('<') => self.consume_and_return(chars, Token::ShiftLeft),
-                        Some('@') => self.consume_and_return(chars, Token::ArrowAt),
+                        Some('>') => self.consume_and_return(Token::Neq),
+                        Some('<') => self.consume_and_return(Token::ShiftLeft),
+                        Some('@') => self.consume_and_return(Token::ArrowAt),
                         _ => Ok(Some(Token::Lt)),
                     }
                 }
                 '>' => {
-                    chars.next(); // consume
-                    match chars.peek() {
-                        Some('=') => self.consume_and_return(chars, Token::GtEq),
-                        Some('>') => self.consume_and_return(chars, Token::ShiftRight),
+                    self.next(); // consume
+                    match self.peek() {
+                        Some('=') => self.consume_and_return(Token::GtEq),
+                        Some('>') => self.consume_and_return(Token::ShiftRight),
                         _ => Ok(Some(Token::Gt)),
                     }
                 }
                 ':' => {
-                    chars.next();
-                    match chars.peek() {
-                        Some(':') => self.consume_and_return(chars, Token::DoubleColon),
+                    self.next();
+                    match self.peek() {
+                        Some(':') => self.consume_and_return(Token::DoubleColon),
                         _ => Ok(Some(Token::Colon)),
                     }
                 }
-                '$' => Ok(Some(self.tokenize_dollar_preceded_value(chars)?)),
-                ';' => self.consume_and_return(chars, Token::SemiColon),
-                '\\' => self.consume_and_return(chars, Token::Backslash),
-                '[' => self.consume_and_return(chars, Token::LBracket),
-                ']' => self.consume_and_return(chars, Token::RBracket),
-                '&' => self.consume_and_return(chars, Token::Ampersand),
+                '$' => Ok(Some(self.tokenize_dollar_preceded_value()?)),
+                ';' => self.consume_and_return(Token::SemiColon),
+                '\\' => self.consume_and_return(Token::Backslash),
+                '[' => self.consume_and_return(Token::LBracket),
+                ']' => self.consume_and_return(Token::RBracket),
+                '&' => self.consume_and_return(Token::Ampersand),
                 '^' => {
-                    chars.next();
-                    match chars.peek() {
-                        Some('@') => self.consume_and_return(chars, Token::Prefix),
+                    self.next();
+                    match self.peek() {
+                        Some('@') => self.consume_and_return(Token::Prefix),
                         _ => Ok(Some(Token::Caret)),
                     }
                 }
-                '{' => self.consume_and_return(chars, Token::LBrace),
-                '}' => self.consume_and_return(chars, Token::RBrace),
+                '{' => self.consume_and_return(Token::LBrace),
+                '}' => self.consume_and_return(Token::RBrace),
                 '~' => {
-                    chars.next(); // consume
-                    match chars.peek() {
+                    self.next(); // consume
+                    match self.peek() {
                         Some('~') => {
-                            chars.next();
-                            match chars.peek() {
-                                Some('*') => {
-                                    self.consume_and_return(chars, Token::DoubleTildeAsterisk)
-                                }
+                            self.next();
+                            match self.peek() {
+                                Some('*') => self.consume_and_return(Token::DoubleTildeAsterisk),
                                 _ => Ok(Some(Token::DoubleTilde)),
                             }
                         }
-                        Some('*') => self.consume_and_return(chars, Token::TildeAsterisk),
+                        Some('*') => self.consume_and_return(Token::TildeAsterisk),
                         _ => Ok(Some(Token::Tilde)),
                     }
                 }
                 '#' => {
-                    chars.next(); // consume the '#'
-                    match chars.peek() {
-                        Some('-') => self.consume_and_return(chars, Token::HashMinus),
+                    self.next(); // consume the '#'
+                    match self.peek() {
+                        Some('-') => self.consume_and_return(Token::HashMinus),
                         Some('>') => {
-                            chars.next(); // consume first '>'
-                            match chars.peek() {
+                            self.next(); // consume first '>'
+                            match self.peek() {
                                 Some('>') => {
-                                    chars.next(); // consume second '>'
+                                    self.next(); // consume second '>'
                                     Ok(Some(Token::HashLongArrow))
                                 }
                                 _ => Ok(Some(Token::HashArrow)),
@@ -791,50 +819,47 @@ impl<'a> Tokenizer<'a> {
                     }
                 }
                 '@' => {
-                    chars.next(); // consume the '@'
-                    match chars.peek() {
-                        Some('>') => self.consume_and_return(chars, Token::AtArrow),
-                        Some('?') => self.consume_and_return(chars, Token::AtQuestionMark),
-                        Some('@') => self.consume_and_return(chars, Token::AtAt),
+                    self.next(); // consume the '@'
+                    match self.peek() {
+                        Some('>') => self.consume_and_return(Token::AtArrow),
+                        Some('?') => self.consume_and_return(Token::AtQuestionMark),
+                        Some('@') => self.consume_and_return(Token::AtAt),
                         // a regular '@' operator
                         _ => Ok(Some(Token::AtSign)),
                     }
                 }
                 '?' => {
-                    chars.next(); // consume the '?'
-                    match chars.peek() {
-                        Some('|') => self.consume_and_return(chars, Token::QuestionMarkPipe),
-                        Some('&') => self.consume_and_return(chars, Token::QuestionMarkAmpersand),
+                    self.next(); // consume the '?'
+                    match self.peek() {
+                        Some('|') => self.consume_and_return(Token::QuestionMarkPipe),
+                        Some('&') => self.consume_and_return(Token::QuestionMarkAmpersand),
                         // a regular '?' operator
                         _ => Ok(Some(Token::QuestionMark)),
                     }
                 }
-                other => self.consume_and_return(chars, Token::Char(other)),
+                other => self.consume_and_return(Token::Char(other)),
             },
             None => Ok(None),
         }
     }
 
     /// Tokenize dollar preceded value (i.e: a string/placeholder)
-    fn tokenize_dollar_preceded_value(
-        &self,
-        chars: &mut Peekable<Chars<'_>>,
-    ) -> Result<Token, TokenizerError> {
+    fn tokenize_dollar_preceded_value(&mut self) -> Result<Token, TokenizerError> {
         let mut s = String::new();
         let mut value = String::new();
 
-        chars.next();
+        self.next();
 
-        if let Some('$') = chars.peek() {
-            chars.next();
+        if let Some('$') = self.peek() {
+            self.next();
 
             let mut is_terminated = false;
             let mut prev: Option<char> = None;
 
-            while let Some(&ch) = chars.peek() {
+            while let Some(ch) = self.peek() {
                 if prev == Some('$') {
                     if ch == '$' {
-                        chars.next();
+                        self.next();
                         is_terminated = true;
                         break;
                     } else {
@@ -846,11 +871,11 @@ impl<'a> Tokenizer<'a> {
                 }
 
                 prev = Some(ch);
-                chars.next();
+                self.next();
             }
 
-            return if chars.peek().is_none() && !is_terminated {
-                self.tokenizer_error("Unterminated dollar-quoted string")
+            return if self.peek().is_none() && !is_terminated {
+                self.error("Unterminated dollar-quoted string")
             } else {
                 Ok(Token::DollarQuotedString(DollarQuotedString {
                     value: s,
@@ -858,36 +883,33 @@ impl<'a> Tokenizer<'a> {
                 }))
             };
         } else {
-            value.push_str(&peeking_take_while(chars, |ch| {
-                ch.is_alphanumeric() || ch == '_'
-            }));
+            value.push_str(&self.peeking_take_while(|ch| ch.is_alphanumeric() || ch == '_'));
 
-            if let Some('$') = chars.peek() {
-                chars.next();
-                s.push_str(&peeking_take_while(chars, |ch| ch != '$'));
+            if let Some('$') = self.peek() {
+                self.next();
+                s.push_str(&self.peeking_take_while(|ch| ch != '$'));
 
-                match chars.peek() {
+                match self.peek() {
                     Some('$') => {
-                        chars.next();
+                        self.next();
                         for c in value.chars() {
-                            let next_char = chars.next();
+                            let next_char = self.next();
                             if Some(c) != next_char {
-                                return self.tokenizer_error(format!(
+                                return self.error(format!(
                                     "Unterminated dollar-quoted string at or near \"{}\"",
                                     value
                                 ));
                             }
                         }
 
-                        if let Some('$') = chars.peek() {
-                            chars.next();
+                        if let Some('$') = self.peek() {
+                            self.next();
                         } else {
-                            return self
-                                .tokenizer_error("Unterminated dollar-quoted string, expected $");
+                            return self.error("Unterminated dollar-quoted string, expected $");
                         }
                     }
                     _ => {
-                        return self.tokenizer_error("Unterminated dollar-quoted, expected $");
+                        return self.error("Unterminated dollar-quoted, expected $");
                     }
                 }
             } else {
@@ -901,27 +923,32 @@ impl<'a> Tokenizer<'a> {
         }))
     }
 
-    fn tokenizer_error<R>(&self, message: impl Into<String>) -> Result<R, TokenizerError> {
+    fn error<R>(&self, message: impl Into<String>) -> Result<R, TokenizerError> {
+        let prefix = format!("LINE {}: ", self.line);
+        let sql_line = self.sql.split('\n').nth(self.line as usize - 1).unwrap();
+        let cursor = " ".repeat(prefix.len() + self.col as usize - 1);
+        let context = format!("{}{}\n{}^", prefix, sql_line, cursor);
         Err(TokenizerError {
             message: message.into(),
             col: self.col,
             line: self.line,
+            context,
         })
     }
 
-    fn reject_number_junk(&self, chars: &mut Peekable<Chars<'_>>) -> Result<(), TokenizerError> {
-        if let Some(ch) = chars.peek()
-            && is_identifier_start(*ch)
+    fn reject_number_junk(&mut self) -> Result<(), TokenizerError> {
+        if let Some(ch) = self.peek()
+            && is_identifier_start(ch)
         {
-            return self.tokenizer_error("trailing junk after numeric literal");
+            return self.error("trailing junk after numeric literal");
         }
         Ok(())
     }
 
     // Consume characters until newline
-    fn tokenize_single_line_comment(&self, chars: &mut Peekable<Chars<'_>>) -> String {
-        let mut comment = peeking_take_while(chars, |ch| ch != '\n');
-        if let Some(ch) = chars.next() {
+    fn tokenize_single_line_comment(&mut self) -> String {
+        let mut comment = self.peeking_take_while(|ch| ch != '\n');
+        if let Some(ch) = self.next() {
             assert_eq!(ch, '\n');
             comment.push(ch);
         }
@@ -929,66 +956,62 @@ impl<'a> Tokenizer<'a> {
     }
 
     /// Tokenize an identifier or keyword, after the first char is already consumed.
-    fn tokenize_word(&self, first_char: char, chars: &mut Peekable<Chars<'_>>) -> String {
+    fn tokenize_word(&mut self, first_char: char) -> String {
         let mut s = first_char.to_string();
-        s.push_str(&peeking_take_while(chars, is_identifier_part));
+        s.push_str(&self.peeking_take_while(is_identifier_part));
         s
     }
 
     /// Read a single quoted string, starting with the opening quote.
-    fn tokenize_single_quoted_string(
-        &self,
-        chars: &mut Peekable<Chars<'_>>,
-    ) -> Result<String, TokenizerError> {
+    fn tokenize_single_quoted_string(&mut self) -> Result<String, TokenizerError> {
         let mut s = String::new();
-        chars.next(); // consume the opening quote
+        self.next(); // consume the opening quote
 
         // slash escaping is specific to MySQL dialect
         let mut is_escaped = false;
-        while let Some(&ch) = chars.peek() {
+        while let Some(ch) = self.peek() {
             match ch {
                 '\'' => {
-                    chars.next(); // consume
+                    self.next(); // consume
                     if is_escaped {
                         s.push(ch);
                         is_escaped = false;
-                    } else if chars.peek().map(|c| *c == '\'').unwrap_or(false) {
+                    } else if self.peek().map(|c| c == '\'').unwrap_or(false) {
                         s.push(ch);
-                        chars.next();
+                        self.next();
                     } else {
                         return Ok(s);
                     }
                 }
                 '\\' => {
                     s.push(ch);
-                    chars.next();
+                    self.next();
                 }
                 _ => {
-                    chars.next(); // consume
+                    self.next(); // consume
                     s.push(ch);
                 }
             }
         }
-        self.tokenizer_error("Unterminated string literal")
+        self.error("Unterminated string literal")
     }
 
     /// Read a single qutoed string with escape
     fn tokenize_single_quoted_string_with_escape(
-        &self,
-        chars: &mut Peekable<Chars<'_>>,
+        &mut self,
     ) -> Result<CstyleEscapedString, TokenizerError> {
         let mut terminated = false;
         let mut s = String::new();
-        chars.next(); // consume the opening quote
+        self.next(); // consume the opening quote
 
-        while let Some(&ch) = chars.peek() {
+        while let Some(ch) = self.peek() {
             match ch {
                 '\'' => {
-                    chars.next(); // consume
-                    if chars.peek().map(|c| *c == '\'').unwrap_or(false) {
+                    self.next(); // consume
+                    if self.peek().map(|c| c == '\'').unwrap_or(false) {
                         s.push('\\');
                         s.push(ch);
-                        chars.next();
+                        self.next();
                     } else {
                         terminated = true;
                         break;
@@ -996,29 +1019,25 @@ impl<'a> Tokenizer<'a> {
                 }
                 '\\' => {
                     s.push(ch);
-                    chars.next();
-                    if chars
-                        .peek()
-                        .map(|c| *c == '\'' || *c == '\\')
-                        .unwrap_or(false)
-                    {
-                        s.push(chars.next().unwrap());
+                    self.next();
+                    if self.peek().map(|c| c == '\'' || c == '\\').unwrap_or(false) {
+                        s.push(self.next().unwrap());
                     }
                 }
                 _ => {
-                    chars.next(); // consume
+                    self.next(); // consume
                     s.push(ch);
                 }
             }
         }
 
         if !terminated {
-            return self.tokenizer_error("Unterminated string literal");
+            return self.error("Unterminated string literal");
         }
 
         let unescaped = match Self::unescape_c_style(&s) {
             Ok(unescaped) => unescaped,
-            Err(e) => return self.tokenizer_error(e),
+            Err(e) => return self.error(e),
         };
 
         Ok(CstyleEscapedString {
@@ -1139,17 +1158,14 @@ impl<'a> Tokenizer<'a> {
         Ok(res)
     }
 
-    fn tokenize_multiline_comment(
-        &self,
-        chars: &mut Peekable<Chars<'_>>,
-    ) -> Result<Option<Token>, TokenizerError> {
+    fn tokenize_multiline_comment(&mut self) -> Result<Option<Token>, TokenizerError> {
         let mut s = String::new();
 
         let mut nested = 1;
         let mut last_ch = ' ';
 
         loop {
-            match chars.next() {
+            match self.next() {
                 Some(ch) => {
                     if last_ch == '/' && ch == '*' {
                         nested += 1;
@@ -1163,39 +1179,32 @@ impl<'a> Tokenizer<'a> {
                     s.push(ch);
                     last_ch = ch;
                 }
-                None => break self.tokenizer_error("Unexpected EOF while in a multi-line comment"),
+                None => break self.error("Unexpected EOF while in a multi-line comment"),
             }
         }
     }
 
     #[allow(clippy::unnecessary_wraps)]
-    fn consume_and_return(
-        &self,
-        chars: &mut Peekable<Chars<'_>>,
-        t: Token,
-    ) -> Result<Option<Token>, TokenizerError> {
-        chars.next();
+    fn consume_and_return(&mut self, t: Token) -> Result<Option<Token>, TokenizerError> {
+        self.next();
         Ok(Some(t))
     }
-}
 
-/// Read from `chars` until `predicate` returns `false` or EOF is hit.
-/// Return the characters read as String, and keep the first non-matching
-/// char available as `chars.next()`.
-fn peeking_take_while(
-    chars: &mut Peekable<Chars<'_>>,
-    mut predicate: impl FnMut(char) -> bool,
-) -> String {
-    let mut s = String::new();
-    while let Some(&ch) = chars.peek() {
-        if predicate(ch) {
-            chars.next(); // consume
-            s.push(ch);
-        } else {
-            break;
+    /// Read from `self` until `predicate` returns `false` or EOF is hit.
+    /// Return the characters read as String, and keep the first non-matching
+    /// char available as `self.next()`.
+    fn peeking_take_while(&mut self, mut predicate: impl FnMut(char) -> bool) -> String {
+        let mut s = String::new();
+        while let Some(ch) = self.peek() {
+            if predicate(ch) {
+                self.next(); // consume
+                s.push(ch);
+            } else {
+                break;
+            }
         }
+        s
     }
-    s
 }
 
 /// Determine if a character starts a quoted identifier. The default
@@ -1230,20 +1239,21 @@ mod tests {
             message: "test".into(),
             line: 1,
             col: 1,
+            context: "LINE 1:".to_string(),
         };
         #[cfg(feature = "std")]
         {
             use std::error::Error;
             assert!(err.source().is_none());
         }
-        assert_eq!(err.to_string(), "test at Line: 1, Column 1");
+        assert_eq!(err.to_string(), "test at line 1, column 1\nLINE 1:");
     }
 
     #[test]
     fn tokenize_select_1() {
         let sql = String::from("SELECT 1");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
 
         let expected = vec![
             Token::make_keyword("SELECT"),
@@ -1258,7 +1268,7 @@ mod tests {
     fn tokenize_select_float() {
         let sql = String::from("SELECT .1");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
 
         let expected = vec![
             Token::make_keyword("SELECT"),
@@ -1273,7 +1283,7 @@ mod tests {
     fn tokenize_scalar_function() {
         let sql = String::from("SELECT sqrt(1)");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
 
         let expected = vec![
             Token::make_keyword("SELECT"),
@@ -1291,7 +1301,7 @@ mod tests {
     fn tokenize_string_string_concat() {
         let sql = String::from("SELECT 'a' || 'b'");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
 
         let expected = vec![
             Token::make_keyword("SELECT"),
@@ -1310,7 +1320,7 @@ mod tests {
     fn tokenize_bitwise_op() {
         let sql = String::from("SELECT one | two ^ three");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
 
         let expected = vec![
             Token::make_keyword("SELECT"),
@@ -1333,7 +1343,7 @@ mod tests {
         let sql =
             String::from("SELECT true XOR true, false XOR false, true XOR false, false XOR true");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
 
         let expected = vec![
             Token::make_keyword("SELECT"),
@@ -1372,7 +1382,7 @@ mod tests {
     fn tokenize_simple_select() {
         let sql = String::from("SELECT * FROM customer WHERE id = 1 LIMIT 5");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
 
         let expected = vec![
             Token::make_keyword("SELECT"),
@@ -1403,7 +1413,7 @@ mod tests {
     fn tokenize_explain_select() {
         let sql = String::from("EXPLAIN SELECT * FROM customer WHERE id = 1");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
 
         let expected = vec![
             Token::make_keyword("EXPLAIN"),
@@ -1432,7 +1442,7 @@ mod tests {
     fn tokenize_explain_analyze_select() {
         let sql = String::from("EXPLAIN ANALYZE SELECT * FROM customer WHERE id = 1");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
 
         let expected = vec![
             Token::make_keyword("EXPLAIN"),
@@ -1463,7 +1473,7 @@ mod tests {
     fn tokenize_string_predicate() {
         let sql = String::from("SELECT * FROM customer WHERE salary != 'Not Provided'");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
 
         let expected = vec![
             Token::make_keyword("SELECT"),
@@ -1490,7 +1500,7 @@ mod tests {
     fn tokenize_invalid_string() {
         let sql = String::from("\nمصطفىh");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
         // println!("tokens: {:#?}", tokens);
         let expected = vec![
             Token::Whitespace(Whitespace::Newline),
@@ -1508,7 +1518,7 @@ mod tests {
     fn tokenize_newline_in_string_literal() {
         let sql = String::from("'foo\r\nbar\nbaz'");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
         let expected = vec![Token::SingleQuotedString("foo\r\nbar\nbaz".to_string())];
         compare(expected, tokens);
     }
@@ -1518,11 +1528,12 @@ mod tests {
         let sql = String::from("select 'foo");
         let mut tokenizer = Tokenizer::new(&sql);
         assert_eq!(
-            tokenizer.tokenize(),
+            tokenizer.tokenize_with_whitespace(),
             Err(TokenizerError {
                 message: "Unterminated string literal".to_string(),
                 line: 1,
-                col: 8,
+                col: 12,
+                context: "LINE 1: select 'foo\n                   ^".to_string(),
             })
         );
     }
@@ -1531,7 +1542,7 @@ mod tests {
     fn tokenize_invalid_string_cols() {
         let sql = String::from("\n\nSELECT * FROM table\tمصطفىh");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
         // println!("tokens: {:#?}", tokens);
         let expected = vec![
             Token::Whitespace(Whitespace::Newline),
@@ -1558,7 +1569,7 @@ mod tests {
     fn tokenize_right_arrow() {
         let sql = String::from("FUNCTION(key=>value)");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
         let expected = vec![
             Token::make_word("FUNCTION", None),
             Token::LParen,
@@ -1574,7 +1585,7 @@ mod tests {
     fn tokenize_is_null() {
         let sql = String::from("a IS NULL");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
 
         let expected = vec![
             Token::make_word("a", None),
@@ -1591,7 +1602,7 @@ mod tests {
     fn tokenize_comment() {
         let sql = String::from("0--this is a comment\n1");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
         let expected = vec![
             Token::Number("0".to_string()),
             Token::Whitespace(Whitespace::SingleLineComment {
@@ -1607,7 +1618,7 @@ mod tests {
     fn tokenize_comment_at_eof() {
         let sql = String::from("--this is a comment");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
         let expected = vec![Token::Whitespace(Whitespace::SingleLineComment {
             prefix: "--".to_string(),
             comment: "this is a comment".to_string(),
@@ -1619,7 +1630,7 @@ mod tests {
     fn tokenize_multiline_comment() {
         let sql = String::from("0/*multi-line\n* /comment*/1");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
         let expected = vec![
             Token::Number("0".to_string()),
             Token::Whitespace(Whitespace::MultiLineComment(
@@ -1634,7 +1645,7 @@ mod tests {
     fn tokenize_nested_multiline_comment() {
         let sql = String::from("0/*multi-line\n* \n/* comment \n /*comment*/*/ */ /comment*/1");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
         let expected = vec![
             Token::Number("0".to_string()),
             Token::Whitespace(Whitespace::MultiLineComment(
@@ -1649,7 +1660,7 @@ mod tests {
     fn tokenize_multiline_comment_with_even_asterisks() {
         let sql = String::from("\n/** Comment **/\n");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
         let expected = vec![
             Token::Whitespace(Whitespace::Newline),
             Token::Whitespace(Whitespace::MultiLineComment("* Comment *".to_string())),
@@ -1663,11 +1674,12 @@ mod tests {
         let sql = String::from("\"foo");
         let mut tokenizer = Tokenizer::new(&sql);
         assert_eq!(
-            tokenizer.tokenize(),
+            tokenizer.tokenize_with_whitespace(),
             Err(TokenizerError {
                 message: "Expected close delimiter '\"' before EOF.".to_string(),
                 line: 1,
-                col: 1,
+                col: 5,
+                context: "LINE 1: \"foo\n            ^".to_string(),
             })
         );
     }
@@ -1676,7 +1688,7 @@ mod tests {
     fn tokenize_newlines() {
         let sql = String::from("line1\nline2\rline3\r\nline4\r");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
         let expected = vec![
             Token::make_word("line1", None),
             Token::Whitespace(Whitespace::Newline),
@@ -1694,7 +1706,7 @@ mod tests {
     fn tokenize_pg_regex_match() {
         let sql = "SELECT col ~ '^a', col ~* '^a', col !~ '^a', col !~* '^a'";
         let mut tokenizer = Tokenizer::new(sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
         let expected = vec![
             Token::make_keyword("SELECT"),
             Token::Whitespace(Whitespace::Space),
@@ -1732,7 +1744,7 @@ mod tests {
     fn tokenize_select_array() {
         let sql = String::from("SELECT '{1, 2, 3}'");
         let mut tokenizer = Tokenizer::new(&sql);
-        let tokens = tokenizer.tokenize().unwrap();
+        let tokens = tokenizer.tokenize_with_whitespace().unwrap();
 
         let expected = vec![
             Token::make_keyword("SELECT"),
diff --git a/src/sqlparser/test_runner/Cargo.toml b/src/sqlparser/test_runner/Cargo.toml
deleted file mode 100644
index b7da53b7435b3..0000000000000
--- a/src/sqlparser/test_runner/Cargo.toml
+++ /dev/null
@@ -1,42 +0,0 @@
-[package]
-name = "risingwave_sqlparser_test_runner"
-version = "0.1.0"
-edition = "2021"
-
-[package.metadata.cargo-machete]
-ignored = ["workspace-hack"]
-
-[package.metadata.cargo-udeps.ignore]
-normal = ["workspace-hack"]
-
-[dependencies]
-anyhow = "1"
-console = "0.15"
-futures = { version = "0.3", default-features = false, features = ["alloc"] }
-risingwave_sqlparser = { workspace = true }
-serde = { version = "1", features = ["derive"] }
-serde_with = "3"
-serde_yaml = "0.9"
-tokio = { version = "0.2", package = "madsim-tokio", features = [
-    "rt",
-    "rt-multi-thread",
-    "sync",
-    "macros",
-    "time",
-    "signal",
-    "fs",
-] }
-walkdir = "2"
-
-[[bin]]
-name = "parser-test-apply"
-path = "src/bin/apply.rs"
-
-[target.'cfg(not(madsim))'.dependencies]
-workspace-hack = { path = "../../workspace-hack" }
-
-[build-dependencies]
-walkdir = "2"
-
-[lints]
-workspace = true
diff --git a/src/sqlparser/test_runner/sqlparser_test.toml b/src/sqlparser/test_runner/sqlparser_test.toml
deleted file mode 100644
index f1190b5220e1b..0000000000000
--- a/src/sqlparser/test_runner/sqlparser_test.toml
+++ /dev/null
@@ -1,56 +0,0 @@
-[tasks.update-parser-test]
-description = "Update parser test data"
-private = true
-script = '''
-#!/usr/bin/env bash
-set -e
-cargo run --bin parser-test-apply
-'''
-
-[tasks.apply-parser-test]
-private = true
-description = "Generate parser test data"
-dependencies = [
-  "update-parser-test"
-]
-script = '''
-#!/usr/bin/env bash
-set -e
-cd src/sqlparser/tests/testdata/
-
-for f in *.apply.yaml
-do
-  diff "$f" "$(basename "$f" .apply.yaml).yaml" || true
-done
-
-echo "If you want to apply the parser test data, run: $(tput setaf 2)./risedev do-apply-parser-test$(tput sgr 0)"
-'''
-category = "RiseDev - Test"
-
-[tasks.do-apply-parser-test]
-description = "Apply parser test data"
-dependencies = [
-  "update-parser-test"
-]
-script = '''
-#!/usr/bin/env bash
-set -e
-cd src/sqlparser/tests/testdata/
-
-for f in *.apply.yaml
-do
-  SOURCE="$(basename $f .apply.yaml).yaml"
-  if [ -f "$SOURCE" ]; then
-    cat <<EOF > temp.apply.yaml
-# This file is automatically generated. See \`src/sqlparser/test_runner/src/bin/apply.rs\` for more information.
-EOF
-    cat "$f" >> temp.apply.yaml
-    mv temp.apply.yaml "$SOURCE"
-  fi
-done
-
-rm *.apply.yaml
-
-echo "$(tput setaf 2)Diff applied!$(tput sgr 0)"
-'''
-category = "RiseDev - Test"
diff --git a/src/sqlparser/test_runner/src/bin/apply.rs b/src/sqlparser/test_runner/src/bin/apply.rs
deleted file mode 100644
index 2693afdfcf484..0000000000000
--- a/src/sqlparser/test_runner/src/bin/apply.rs
+++ /dev/null
@@ -1,125 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#![feature(register_tool)]
-#![register_tool(rw)]
-#![allow(rw::format_error)]
-
-use std::path::Path;
-use std::sync::{Arc, Mutex};
-
-use anyhow::Result;
-use console::style;
-use futures::future::try_join_all;
-use risingwave_sqlparser::ast::Statement;
-use risingwave_sqlparser::parser::Parser;
-use risingwave_sqlparser_test_runner::TestCase;
-use walkdir::WalkDir;
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    let manifest_dir = env!("CARGO_MANIFEST_DIR");
-    let dir = Path::new(manifest_dir)
-        .parent()
-        .unwrap()
-        .join("tests")
-        .join("testdata");
-    println!("Using test cases from {:?}", dir);
-
-    let mut futs = vec![];
-
-    let log_lock = Arc::new(Mutex::new(()));
-
-    for entry in WalkDir::new(dir) {
-        let entry = entry.unwrap();
-        let path = entry.path();
-
-        if !path.is_file() {
-            continue;
-        }
-
-        if path.is_file()
-            && path.extension().map_or(false, |p| {
-                p.eq_ignore_ascii_case("yml") || p.eq_ignore_ascii_case("yaml")
-            })
-            && !path
-                .file_name()
-                .unwrap()
-                .to_string_lossy()
-                .ends_with(".apply.yaml")
-        {
-            let target = path.with_extension("apply.yaml");
-
-            let path = path.to_path_buf();
-            let log_lock = Arc::clone(&log_lock);
-            futs.push(async move {
-                let file_content = tokio::fs::read_to_string(&path).await?;
-
-                let cases: Vec<TestCase> = serde_yaml::from_str(&file_content)?;
-
-                let mut new_cases = Vec::with_capacity(cases.len());
-
-                for case in cases {
-                    let input = &case.input;
-                    let ast = Parser::parse_sql(input);
-                    let actual_case = match ast {
-                        Ok(ast) => {
-                            let [ast]: [Statement; 1] = ast
-                                .try_into()
-                                .expect("Only one statement is supported now.");
-
-                            let actual_formatted_sql =
-                                case.formatted_sql.as_ref().map(|_| format!("{}", ast));
-                            let actual_formatted_ast =
-                                case.formatted_ast.as_ref().map(|_| format!("{:?}", ast));
-
-                            TestCase {
-                                input: input.clone(),
-                                formatted_sql: actual_formatted_sql,
-                                formatted_ast: actual_formatted_ast,
-                                error_msg: None,
-                            }
-                        }
-                        Err(err) => {
-                            let actual_error_msg = format!("{}", err);
-                            TestCase {
-                                input: input.clone(),
-                                formatted_sql: None,
-                                formatted_ast: None,
-                                error_msg: Some(actual_error_msg),
-                            }
-                        }
-                    };
-
-                    if actual_case != case {
-                        let _guard = log_lock.lock();
-                        println!("{}\n{}\n", style(&case).red(), style(&actual_case).green())
-                    }
-
-                    new_cases.push(actual_case);
-                }
-
-                let output_content = serde_yaml::to_string(&new_cases)?;
-
-                tokio::fs::write(target, output_content).await?;
-
-                Ok::<_, anyhow::Error>(())
-            });
-        }
-    }
-
-    let _res = try_join_all(futs).await?;
-
-    Ok(())
-}
diff --git a/src/sqlparser/test_runner/src/lib.rs b/src/sqlparser/test_runner/src/lib.rs
deleted file mode 100644
index d3f7a85939fb7..0000000000000
--- a/src/sqlparser/test_runner/src/lib.rs
+++ /dev/null
@@ -1,127 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Data-driven tests.
-
-#![feature(register_tool)]
-#![register_tool(rw)]
-#![allow(rw::format_error)]
-
-use std::fmt::Display;
-
-use anyhow::{anyhow, Result};
-use risingwave_sqlparser::parser::Parser;
-use serde::{Deserialize, Serialize};
-
-/// `TestCase` will be deserialized from yaml.
-#[serde_with::skip_serializing_none]
-#[derive(PartialEq, Eq, Debug, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct TestCase {
-    pub input: String,
-    pub formatted_sql: Option<String>,
-    pub error_msg: Option<String>,
-    pub formatted_ast: Option<String>,
-}
-
-impl Display for TestCase {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_str(&serde_yaml::to_string(self).unwrap())
-    }
-}
-
-fn run_test_case(c: TestCase) -> Result<()> {
-    let result = Parser::parse_sql(&c.input);
-    if let Some(error_msg) = c.error_msg.map(|s| s.trim().to_string()) {
-        if result.is_ok() {
-            return Err(anyhow!("Expected failure:\n  {}", error_msg));
-        }
-        let actual_error_msg = result.unwrap_err().to_string();
-        if actual_error_msg != error_msg {
-            return Err(anyhow!(
-                "Expected error message: {}\n  Actual error message: {}",
-                error_msg,
-                actual_error_msg
-            ));
-        }
-    } else {
-        if c.formatted_sql.is_none() {
-            return Err(anyhow!("Illegal test case without given the formatted sql"));
-        }
-        if let Err(e) = result {
-            return Err(anyhow!("Unexpected failure:\n  {}", e));
-        }
-        let ast = &result.unwrap()[0];
-        let formatted_sql = format!("{}", ast);
-        let expected_formatted_sql = c.formatted_sql.as_ref().unwrap();
-        if &formatted_sql != expected_formatted_sql {
-            return Err(anyhow!(
-                "Expected formatted sql:\n  {}\n  Actual formatted sql:\n  {}",
-                expected_formatted_sql,
-                formatted_sql
-            ));
-        }
-        if let Some(expected_formatted_ast) = c.formatted_ast.map(|s| s.trim().to_string()) {
-            let formatted_ast = format!("{:?}", ast);
-            if formatted_ast != expected_formatted_ast {
-                return Err(anyhow!(
-                    "Expected formatted ast:\n  {}\n  Actual formatted ast:\n  {}",
-                    expected_formatted_ast,
-                    formatted_ast
-                ));
-            }
-        }
-    }
-    Ok(())
-}
-
-fn run_test_file(file_name: &str, file_content: &str) {
-    let cases: Vec<TestCase> = serde_yaml::from_str(file_content)
-        .unwrap_or_else(|_| panic!("Failed to parse {}", file_name));
-    let mut failed_num = 0;
-    for c in cases {
-        let input = c.input.clone();
-        if let Err(e) = run_test_case(c) {
-            println!(
-                "\nThe input SQL from file {}:\n  {}\n{}",
-                file_name, input, e
-            );
-            failed_num += 1;
-        }
-    }
-    if failed_num > 0 {
-        println!("\n");
-        panic!("{} test cases failed", failed_num);
-    }
-}
-
-/// The entry point of `test_runner`.
-pub fn run_all_test_files() {
-    use walkdir::WalkDir;
-    for entry in WalkDir::new("../tests/testdata/") {
-        let entry = entry.unwrap();
-        if !(entry.path().is_file()) {
-            continue;
-        }
-        if !(entry
-            .path()
-            .extension()
-            .map_or(false, |p| p.eq_ignore_ascii_case("yaml")))
-        {
-            continue;
-        }
-        let file_content = std::fs::read_to_string(entry.path()).unwrap();
-        run_test_file(entry.path().to_str().unwrap(), &file_content);
-    }
-}
diff --git a/src/sqlparser/test_runner/tests/test_runner.rs b/src/sqlparser/test_runner/tests/test_runner.rs
deleted file mode 100644
index 501e27f33ab39..0000000000000
--- a/src/sqlparser/test_runner/tests/test_runner.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Traverses the 'testdata/' directory and runs all files.
-#[test]
-fn run_all_test_files() {
-    risingwave_sqlparser_test_runner::run_all_test_files();
-}
diff --git a/src/sqlparser/tests/parser_test.rs b/src/sqlparser/tests/parser_test.rs
new file mode 100644
index 0000000000000..57864ac6643d2
--- /dev/null
+++ b/src/sqlparser/tests/parser_test.rs
@@ -0,0 +1,199 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Run parser tests from yaml files in `tests/testdata`.
+//!
+//! Set `UPDATE_PARSER_TEST=1` to update the yaml files.
+
+#![feature(register_tool)]
+#![register_tool(rw)]
+#![allow(rw::format_error)]
+
+use std::fmt::{Display, Write as _};
+use std::fs::File;
+use std::io::Write;
+
+use anyhow::{anyhow, bail, Context, Result};
+use console::style;
+use libtest_mimic::{Arguments, Failed, Trial};
+use risingwave_sqlparser::ast::Statement;
+use risingwave_sqlparser::parser::Parser;
+use serde::{Deserialize, Serialize};
+
+/// `TestCase` will be deserialized from yaml.
+#[serde_with::skip_serializing_none]
+#[derive(PartialEq, Eq, Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct TestCase {
+    pub input: String,
+    pub formatted_sql: Option<String>,
+    pub error_msg: Option<String>,
+    pub formatted_ast: Option<String>,
+}
+
+impl Display for TestCase {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(&serde_yaml::to_string(self).unwrap())
+    }
+}
+
+fn run_test_case(c: TestCase) -> Result<()> {
+    let result = Parser::parse_sql(&c.input);
+    if let Some(error_msg) = c.error_msg.map(|s| s.trim().to_string()) {
+        if result.is_ok() {
+            return Err(anyhow!("Expected failure:\n  {}", error_msg));
+        }
+        let actual_error_msg = result.unwrap_err().to_string();
+        if actual_error_msg != error_msg {
+            return Err(anyhow!(
+                "Expected error message: {}\n  Actual error message: {}",
+                style(&error_msg).green(),
+                style(&actual_error_msg).red(),
+            ));
+        }
+    } else {
+        if c.formatted_sql.is_none() {
+            return Err(anyhow!("Illegal test case without given the formatted sql"));
+        }
+        if let Err(e) = result {
+            return Err(anyhow!("Unexpected failure:\n  {}", e));
+        }
+        let ast = &result.unwrap()[0];
+        let formatted_sql = format!("{}", ast);
+        let expected_formatted_sql = c.formatted_sql.as_ref().unwrap();
+        if &formatted_sql != expected_formatted_sql {
+            return Err(anyhow!(
+                "Expected formatted sql:\n  {}\nActual:\n  {}",
+                style(&expected_formatted_sql).green(),
+                style(&formatted_sql).red(),
+            ));
+        }
+        if let Some(expected_formatted_ast) = c.formatted_ast.map(|s| s.trim().to_string()) {
+            let formatted_ast = format!("{:?}", ast);
+            if formatted_ast != expected_formatted_ast {
+                return Err(anyhow!(
+                    "Expected formatted ast:\n  {}\nActual:\n  {}",
+                    style(&expected_formatted_ast).green(),
+                    style(&formatted_ast).red(),
+                ));
+            }
+        }
+    }
+    Ok(())
+}
+
+fn run_test_file(file_name: &str) -> Result<()> {
+    let file_content = std::fs::read_to_string(file_name).unwrap();
+    let cases: Vec<TestCase> =
+        serde_yaml::from_str(&file_content).context("failed to parse yaml")?;
+    let mut error_msg = String::new();
+    for c in cases {
+        let input = c.input.clone();
+        if let Err(e) = run_test_case(c) {
+            write!(&mut error_msg, "With input:\n  {}\n{}\n", input, e)?;
+        }
+    }
+    if !error_msg.is_empty() {
+        bail!("{}", error_msg);
+    }
+    Ok(())
+}
+
+fn update_test_file(file_name: &str) -> Result<()> {
+    let file_content = std::fs::read_to_string(file_name)?;
+
+    let cases: Vec<TestCase> = serde_yaml::from_str(&file_content)?;
+
+    let mut new_cases = Vec::with_capacity(cases.len());
+
+    for case in cases {
+        let input = &case.input;
+        let ast = Parser::parse_sql(input);
+        let actual_case = match ast {
+            Ok(ast) => {
+                let [ast]: [Statement; 1] = ast
+                    .try_into()
+                    .expect("Only one statement is supported now.");
+
+                let actual_formatted_sql = case.formatted_sql.as_ref().map(|_| format!("{}", ast));
+                let actual_formatted_ast =
+                    case.formatted_ast.as_ref().map(|_| format!("{:?}", ast));
+
+                TestCase {
+                    input: input.clone(),
+                    formatted_sql: actual_formatted_sql,
+                    formatted_ast: actual_formatted_ast,
+                    error_msg: None,
+                }
+            }
+            Err(err) => {
+                let actual_error_msg = format!("{}", err);
+                TestCase {
+                    input: input.clone(),
+                    formatted_sql: None,
+                    formatted_ast: None,
+                    error_msg: Some(actual_error_msg),
+                }
+            }
+        };
+
+        if actual_case != case {
+            println!("{}\n{}\n", style(&case).red(), style(&actual_case).green())
+        }
+
+        new_cases.push(actual_case);
+    }
+
+    let mut output_file = File::create(file_name)?;
+    writeln!(
+        output_file,
+        "# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`."
+    )?;
+    write!(output_file, "{}", serde_yaml::to_string(&new_cases)?)?;
+    Ok(())
+}
+
+/// The entry point of `test_runner`.
+fn main() {
+    let run_tests_args = &Arguments::from_args();
+    let mut tests = vec![];
+    let update = std::env::var("UPDATE_PARSER_TEST").is_ok();
+    std::env::set_current_dir("tests/testdata").unwrap();
+
+    use walkdir::WalkDir;
+    for entry in WalkDir::new(".") {
+        let entry = entry.unwrap();
+        if !(entry.path().is_file()) {
+            continue;
+        }
+        if !(entry
+            .path()
+            .extension()
+            .map_or(false, |p| p.eq_ignore_ascii_case("yaml")))
+        {
+            continue;
+        }
+        let file_name = entry.file_name().to_string_lossy().to_string();
+        tests.push(Trial::test(file_name.clone(), move || {
+            let f = if update {
+                update_test_file
+            } else {
+                run_test_file
+            };
+            f(&file_name).map_err(|e| Failed::from(e.to_string()))
+        }));
+    }
+
+    libtest_mimic::run(run_tests_args, tests).exit();
+}
diff --git a/src/sqlparser/tests/sqlparser_common.rs b/src/sqlparser/tests/sqlparser_common.rs
index ff0e9d2e53c40..a3159400b88fe 100644
--- a/src/sqlparser/tests/sqlparser_common.rs
+++ b/src/sqlparser/tests/sqlparser_common.rs
@@ -25,7 +25,6 @@ use risingwave_sqlparser::ast::*;
 use risingwave_sqlparser::keywords::ALL_KEYWORDS;
 use risingwave_sqlparser::parser::ParserError;
 use risingwave_sqlparser::test_utils::*;
-use test_utils::{expr_from_projection, join, number, only, table, table_alias};
 
 #[test]
 fn parse_insert_values() {
@@ -131,12 +130,12 @@ fn parse_update() {
 
     let sql = "UPDATE t WHERE 1";
     let res = parse_sql_statements(sql);
-    assert!(format!("{}", res.unwrap_err()).contains("Expected SET, found: WHERE"));
+    assert!(format!("{}", res.unwrap_err()).contains("expected SET, found: WHERE"));
 
     let sql = "UPDATE t SET a = 1 extrabadstuff";
     let res = parse_sql_statements(sql);
     assert!(
-        format!("{}", res.unwrap_err()).contains("Expected end of statement, found: extrabadstuff")
+        format!("{}", res.unwrap_err()).contains("expected end of statement, found: extrabadstuff")
     );
 }
 
@@ -253,8 +252,10 @@ fn parse_select_all() {
 #[test]
 fn parse_select_all_distinct() {
     let result = parse_sql_statements("SELECT ALL DISTINCT name FROM customer");
-    assert!(format!("{}", result.unwrap_err())
-        .contains("syntax error at or near DISTINCT at line:1, column:20"));
+    assert!(result
+        .unwrap_err()
+        .to_string()
+        .contains("syntax error at or near DISTINCT at line 1, column 12"));
 }
 
 #[test]
@@ -285,7 +286,7 @@ fn parse_select_wildcard() {
 
     let sql = "SELECT * + * FROM foo;";
     let result = parse_sql_statements(sql);
-    assert!(format!("{}", result.unwrap_err()).contains("Expected end of statement, found: +"));
+    assert!(format!("{}", result.unwrap_err()).contains("expected end of statement, found: +"));
 }
 
 #[test]
@@ -322,7 +323,7 @@ fn parse_column_aliases() {
         assert_eq!(&Expr::Value(number("1")), right.as_ref());
         assert_eq!(&Ident::new_unchecked("newname"), alias);
     } else {
-        panic!("Expected ExprWithAlias")
+        panic!("expected ExprWithAlias")
     }
 
     // alias without AS is parsed correctly:
@@ -332,10 +333,10 @@ fn parse_column_aliases() {
 #[test]
 fn test_eof_after_as() {
     let res = parse_sql_statements("SELECT foo AS");
-    assert!(format!("{}", res.unwrap_err()).contains("Expected an identifier after AS, found: EOF"));
+    assert!(format!("{}", res.unwrap_err()).contains("expected an identifier after AS, found: EOF"));
 
     let res = parse_sql_statements("SELECT 1 FROM foo AS");
-    assert!(format!("{}", res.unwrap_err()).contains("Expected an identifier after AS, found: EOF"));
+    assert!(format!("{}", res.unwrap_err()).contains("expected an identifier after AS, found: EOF"));
 }
 
 #[test]
@@ -391,7 +392,7 @@ fn parse_select_count_distinct() {
 #[test]
 fn parse_invalid_infix_not() {
     let res = parse_sql_statements("SELECT c FROM t WHERE c NOT (");
-    assert!(format!("{}", res.unwrap_err(),).contains("Expected end of statement, found: NOT"));
+    assert!(format!("{}", res.unwrap_err(),).contains("expected end of statement, found: NOT"));
 }
 
 #[test]
@@ -1267,9 +1268,6 @@ fn parse_extract() {
     verified_stmt("SELECT EXTRACT(HOUR FROM d)");
     verified_stmt("SELECT EXTRACT(MINUTE FROM d)");
     verified_stmt("SELECT EXTRACT(SECOND FROM d)");
-
-    let res = parse_sql_statements("SELECT EXTRACT(0 FROM d)");
-    assert!(format!("{}", res.unwrap_err()).contains("Expected date/time field, found: 0"));
 }
 
 #[test]
@@ -1435,13 +1433,13 @@ fn parse_create_table() {
     assert!(res
         .unwrap_err()
         .to_string()
-        .contains("Expected \',\' or \')\' after column definition, found: GARBAGE"));
+        .contains("expected \',\' or \')\' after column definition, found: GARBAGE"));
 
     let res = parse_sql_statements("CREATE TABLE t (a int NOT NULL CONSTRAINT foo)");
     assert!(res
         .unwrap_err()
         .to_string()
-        .contains("Expected constraint details after CONSTRAINT <name>"));
+        .contains("expected constraint details after CONSTRAINT <name>"));
 }
 
 #[test]
@@ -1782,12 +1780,12 @@ fn parse_alter_table_alter_column_type() {
 #[test]
 fn parse_bad_constraint() {
     let res = parse_sql_statements("ALTER TABLE tab ADD");
-    assert!(format!("{}", res.unwrap_err()).contains("Expected identifier, found: EOF"));
+    assert!(format!("{}", res.unwrap_err()).contains("expected identifier, found: EOF"));
 
     let res = parse_sql_statements("CREATE TABLE tab (foo int,");
 
     assert!(format!("{}", res.unwrap_err())
-        .contains("Expected column name or constraint definition, found: EOF"));
+        .contains("expected column name or constraint definition, found: EOF"));
 }
 
 fn run_explain_analyze(query: &str, expected_analyze: bool, expected_options: ExplainOptions) {
@@ -1886,7 +1884,7 @@ fn parse_explain_with_invalid_options() {
     assert!(res.is_err());
 
     let res = parse_sql_statements("EXPLAIN (VERBOSE TRACE) SELECT sqrt(id) FROM foo");
-    assert!(format!("{}", res.unwrap_err()).contains("Expected ), found: TRACE"));
+    assert!(format!("{}", res.unwrap_err()).contains("expected ), found: TRACE"));
 
     let res = parse_sql_statements("EXPLAIN () SELECT sqrt(id) FROM foo");
     assert!(res.is_err());
@@ -1894,7 +1892,7 @@ fn parse_explain_with_invalid_options() {
     let res = parse_sql_statements("EXPLAIN (VERBOSE, ) SELECT sqrt(id) FROM foo");
 
     let err_msg =
-        "Expected one of VERBOSE or TRACE or TYPE or LOGICAL or PHYSICAL or DISTSQL, found: )";
+        "expected one of VERBOSE or TRACE or TYPE or LOGICAL or PHYSICAL or DISTSQL, found: )";
     assert!(format!("{}", res.unwrap_err()).contains(err_msg));
 }
 
@@ -2208,10 +2206,10 @@ fn parse_literal_interval() {
     );
 
     let result = parse_sql_statements("SELECT INTERVAL '1' SECOND TO SECOND");
-    assert!(format!("{}", result.unwrap_err()).contains("Expected end of statement, found: SECOND"));
+    assert!(format!("{}", result.unwrap_err()).contains("expected end of statement, found: SECOND"));
 
     let result = parse_sql_statements("SELECT INTERVAL '10' HOUR (1) TO HOUR (2)");
-    assert!(format!("{}", result.unwrap_err()).contains("Expected end of statement, found: ("));
+    assert!(format!("{}", result.unwrap_err()).contains("expected end of statement, found: ("));
 
     verified_only_select("SELECT INTERVAL '1' YEAR");
     verified_only_select("SELECT INTERVAL '1' MONTH");
@@ -2294,7 +2292,7 @@ fn parse_delimited_identifiers() {
             );
             assert_eq!(&Ident::with_quote_unchecked('"', "column alias"), alias);
         }
-        _ => panic!("Expected ExprWithAlias"),
+        _ => panic!("expected ExprWithAlias"),
     }
 
     verified_stmt(r#"CREATE TABLE "foo" ("bar" "int")"#);
@@ -2613,7 +2611,7 @@ fn parse_natural_join() {
 
     let sql = "SELECT * FROM t1 natural";
     assert!(format!("{}", parse_sql_statements(sql).unwrap_err(),)
-        .contains("Expected a join type after NATURAL, found: EOF"));
+        .contains("expected a join type after NATURAL, found: EOF"));
 }
 
 #[test]
@@ -2677,7 +2675,7 @@ fn parse_join_syntax_variants() {
     );
 
     let res = parse_sql_statements("SELECT * FROM a OUTER JOIN b ON 1");
-    assert!(format!("{}", res.unwrap_err()).contains("Expected LEFT, RIGHT, or FULL, found: OUTER"));
+    assert!(format!("{}", res.unwrap_err()).contains("expected LEFT, RIGHT, or FULL, found: OUTER"));
 }
 
 #[test]
@@ -2713,7 +2711,7 @@ fn parse_ctes() {
         Expr::Subquery(ref subquery) => {
             assert_ctes_in_select(&cte_sqls, subquery.as_ref());
         }
-        _ => panic!("Expected subquery"),
+        _ => panic!("expected subquery"),
     }
     // CTE in a derived table
     let sql = &format!("SELECT * FROM ({})", with);
@@ -2722,13 +2720,13 @@ fn parse_ctes() {
         TableFactor::Derived { subquery, .. } => {
             assert_ctes_in_select(&cte_sqls, subquery.as_ref())
         }
-        _ => panic!("Expected derived table"),
+        _ => panic!("expected derived table"),
     }
     // CTE in a view
     let sql = &format!("CREATE VIEW v AS {}", with);
     match verified_stmt(sql) {
         Statement::CreateView { query, .. } => assert_ctes_in_select(&cte_sqls, &query),
-        _ => panic!("Expected CREATE VIEW"),
+        _ => panic!("expected CREATE VIEW"),
     }
     // CTE in a CTE...
     let sql = &format!("WITH outer_cte AS ({}) SELECT * FROM outer_cte", with);
@@ -2867,7 +2865,7 @@ fn parse_multiple_statements() {
         one_statement_parses_to(&(sql1.to_owned() + ";"), sql1);
         // Check that forgetting the semicolon results in an error:
         let res = parse_sql_statements(&(sql1.to_owned() + " " + sql2_kw + sql2_rest));
-        let err_msg = "Expected end of statement, found: ";
+        let err_msg = "expected end of statement, found: ";
         assert!(format!("{}", res.unwrap_err()).contains(err_msg));
     }
     test_with("SELECT foo", "SELECT", " bar");
@@ -2918,38 +2916,6 @@ fn parse_substring() {
     one_statement_parses_to("SELECT SUBSTRING('1' FOR 3)", "SELECT SUBSTRING('1' FOR 3)");
 }
 
-#[test]
-fn parse_overlay() {
-    one_statement_parses_to(
-        "SELECT OVERLAY('abc' PLACING 'xyz' FROM 1)",
-        "SELECT OVERLAY('abc' PLACING 'xyz' FROM 1)",
-    );
-
-    one_statement_parses_to(
-        "SELECT OVERLAY('abc' PLACING 'xyz' FROM 1 FOR 2)",
-        "SELECT OVERLAY('abc' PLACING 'xyz' FROM 1 FOR 2)",
-    );
-
-    for (sql, err_msg) in [
-        ("SELECT OVERLAY('abc', 'xyz')", "Expected PLACING, found: ,"),
-        (
-            "SELECT OVERLAY('abc' PLACING 'xyz')",
-            "Expected FROM, found: )",
-        ),
-        (
-            "SELECT OVERLAY('abc' PLACING 'xyz' FOR 2)",
-            "Expected FROM, found: FOR",
-        ),
-        (
-            "SELECT OVERLAY('abc' PLACING 'xyz' FOR 2 FROM 1)",
-            "Expected FROM, found: FOR",
-        ),
-    ] {
-        let res = parse_sql_statements(sql);
-        assert!(format!("{}", res.unwrap_err()).contains(err_msg));
-    }
-}
-
 #[test]
 fn parse_trim() {
     one_statement_parses_to(
@@ -2971,7 +2937,7 @@ fn parse_trim() {
 
     let res = parse_sql_statements("SELECT TRIM(FOO 'xyz' FROM 'xyzfooxyz')");
 
-    let err_msg = "Expected ), found: 'xyz'";
+    let err_msg = "expected ), found: 'xyz'";
     assert!(format!("{}", res.unwrap_err()).contains(err_msg));
 }
 
@@ -2999,12 +2965,12 @@ fn parse_exists_subquery() {
     verified_stmt("SELECT EXISTS (SELECT 1)");
 
     let res = parse_sql_statements("SELECT EXISTS (");
-    let err_msg = "Expected SELECT, VALUES, or a subquery in the query body, found: EOF";
+    let err_msg = "expected SELECT, VALUES, or a subquery in the query body, found: EOF";
     assert!(format!("{}", res.unwrap_err()).contains(err_msg));
 
     let res = parse_sql_statements("SELECT EXISTS (NULL)");
 
-    let err_msg = "Expected SELECT, VALUES, or a subquery in the query body, found: NULL";
+    let err_msg = "expected SELECT, VALUES, or a subquery in the query body, found: NULL";
     assert!(format!("{}", res.unwrap_err()).contains(err_msg));
 }
 
@@ -3338,11 +3304,11 @@ fn parse_drop_table() {
 
     let sql = "DROP TABLE";
     assert!(format!("{}", parse_sql_statements(sql).unwrap_err(),)
-        .contains("Expected identifier, found: EOF"));
+        .contains("expected identifier, found: EOF"));
 
     let sql = "DROP TABLE IF EXISTS foo CASCADE RESTRICT";
     assert!(format!("{}", parse_sql_statements(sql).unwrap_err(),)
-        .contains("Expected end of statement, found: RESTRICT"));
+        .contains("expected end of statement, found: RESTRICT"));
 }
 
 #[test]
@@ -3403,7 +3369,7 @@ fn parse_create_user() {
 #[test]
 fn parse_invalid_subquery_without_parens() {
     let res = parse_sql_statements("SELECT SELECT 1 FROM bar WHERE 1=1 FROM baz");
-    assert!(format!("{}", res.unwrap_err()).contains("Expected end of statement, found: 1"));
+    assert!(format!("{}", res.unwrap_err()).contains("expected end of statement, found: 1"));
 }
 
 #[test]
@@ -3565,12 +3531,12 @@ fn lateral_derived() {
     let sql = "SELECT * FROM customer LEFT JOIN LATERAL generate_series(1, customer.id)";
     let res = parse_sql_statements(sql);
     assert!(format!("{}", res.unwrap_err())
-        .contains("Expected subquery after LATERAL, found: generate_series"));
+        .contains("expected subquery after LATERAL, found: generate_series"));
 
     let sql = "SELECT * FROM a LEFT JOIN LATERAL (b CROSS JOIN c)";
     let res = parse_sql_statements(sql);
     assert!(format!("{}", res.unwrap_err())
-        .contains("Expected SELECT, VALUES, or a subquery in the query body, found: b"));
+        .contains("expected SELECT, VALUES, or a subquery in the query body, found: b"));
 }
 
 #[test]
@@ -3623,13 +3589,13 @@ fn parse_start_transaction() {
     );
 
     let res = parse_sql_statements("START TRANSACTION ISOLATION LEVEL BAD");
-    assert!(format!("{}", res.unwrap_err()).contains("Expected isolation level, found: BAD"));
+    assert!(format!("{}", res.unwrap_err()).contains("expected isolation level, found: BAD"));
 
     let res = parse_sql_statements("START TRANSACTION BAD");
-    assert!(format!("{}", res.unwrap_err()).contains("Expected end of statement, found: BAD"));
+    assert!(format!("{}", res.unwrap_err()).contains("expected end of statement, found: BAD"));
 
     let res = parse_sql_statements("START TRANSACTION READ ONLY,");
-    assert!(format!("{}", res.unwrap_err()).contains("Expected transaction mode, found: EOF"));
+    assert!(format!("{}", res.unwrap_err()).contains("expected transaction mode, found: EOF"));
 }
 
 #[test]
diff --git a/src/sqlparser/tests/sqlparser_postgres.rs b/src/sqlparser/tests/sqlparser_postgres.rs
index d70625c33d770..f3088f47750c4 100644
--- a/src/sqlparser/tests/sqlparser_postgres.rs
+++ b/src/sqlparser/tests/sqlparser_postgres.rs
@@ -16,7 +16,6 @@
 #[macro_use]
 mod test_utils;
 use risingwave_sqlparser::ast::*;
-use risingwave_sqlparser::parser::ParserError;
 use test_utils::*;
 
 #[test]
@@ -314,19 +313,19 @@ fn parse_bad_if_not_exists() {
     for (sql, err_msg) in [
         (
             "CREATE TABLE NOT EXISTS uk_cities ()",
-            "Expected end of statement, found: EXISTS",
+            "expected end of statement, found: EXISTS",
         ),
         (
             "CREATE TABLE IF EXISTS uk_cities ()",
-            "Expected end of statement, found: EXISTS",
+            "expected end of statement, found: EXISTS",
         ),
         (
             "CREATE TABLE IF uk_cities ()",
-            "Expected end of statement, found: uk_cities",
+            "expected end of statement, found: uk_cities",
         ),
         (
             "CREATE TABLE IF NOT uk_cities ()",
-            "Expected end of statement, found: NOT",
+            "expected end of statement, found: NOT",
         ),
     ] {
         let res = parse_sql_statements(sql);
@@ -342,6 +341,7 @@ fn parse_create_schema_if_not_exists() {
         Statement::CreateSchema {
             if_not_exists: true,
             schema_name,
+            ..
         } => assert_eq!("schema_name", schema_name.to_string()),
         _ => unreachable!(),
     }
@@ -439,12 +439,14 @@ fn parse_set() {
     one_statement_parses_to("SET a TO b", "SET a = b");
     one_statement_parses_to("SET SESSION a = b", "SET a = b");
     for (sql, err_msg) in [
-        ("SET", "Expected identifier, found: EOF"),
-        ("SET a b", "Expected equals sign or TO, found: b"),
-        ("SET a =", "Expected parameter value, found: EOF"),
+        ("SET", "expected identifier, found: EOF"),
+        ("SET a b", "expected equals sign or TO, found: b"),
+        ("SET a =", "expected parameter value"),
     ] {
-        let res = parse_sql_statements(sql);
-        assert!(format!("{}", res.unwrap_err()).contains(err_msg));
+        let error = parse_sql_statements(sql).unwrap_err().to_string();
+        if !error.contains(err_msg) {
+            panic!("expected error '{}' not found in '{}'", err_msg, error);
+        }
     }
 }
 
@@ -909,7 +911,7 @@ fn parse_create_aggregate() {
             or_replace: true,
             name: ObjectName(vec![Ident::new_unchecked("sum")]),
             args: vec![OperateFunctionArg::unnamed(DataType::Int)],
-            returns: Some(DataType::BigInt),
+            returns: DataType::BigInt,
             append_only: true,
             params: CreateFunctionBody {
                 language: Some("python".into()),
@@ -1119,41 +1121,19 @@ fn parse_array() {
     );
 
     let sql = "SELECT ARRAY[ARRAY[1, 2], [3, 4]]";
-    assert_eq!(
-        parse_sql_statements(sql),
-        Err(ParserError::ParserError(
-            "syntax error at or near [ at line:1, column:28".to_string()
-        ))
-    );
+    assert!(parse_sql_statements(sql).is_err());
 
     let sql = "SELECT ARRAY[ARRAY[], []]";
-    assert_eq!(
-        parse_sql_statements(sql),
-        Err(ParserError::ParserError(
-            "syntax error at or near [ at line:1, column:24".to_string()
-        ))
-    );
+    assert!(parse_sql_statements(sql).is_err());
 
     let sql = "SELECT ARRAY[[1, 2], ARRAY[3, 4]]";
-    assert_eq!(
-        parse_sql_statements(sql),
-        Err(ParserError::ParserError(
-            "syntax error at or near ARRAY at line:1, column:27".to_string()
-        ))
-    );
+    assert!(parse_sql_statements(sql).is_err());
 
     let sql = "SELECT ARRAY[[], ARRAY[]]";
-    assert_eq!(
-        parse_sql_statements(sql),
-        Err(ParserError::ParserError(
-            "syntax error at or near ARRAY at line:1, column:23".to_string()
-        ))
-    );
+    assert!(parse_sql_statements(sql).is_err());
 
     let sql = "SELECT [[1, 2], [3, 4]]";
-    let res = parse_sql_statements(sql);
-    let err_msg = "Expected an expression:, found: [";
-    assert!(format!("{}", res.unwrap_err()).contains(err_msg));
+    assert!(parse_sql_statements(sql).is_err());
 }
 
 #[test]
@@ -1308,10 +1288,8 @@ fn parse_variadic_argument() {
     _ = verified_stmt(sql);
 
     let sql = "SELECT foo(VARIADIC a, b, VARIADIC c)";
-    assert_eq!(
-        parse_sql_statements(sql),
-        Err(ParserError::ParserError(
-            "VARIADIC argument must be last".to_string()
-        ))
-    );
+    assert!(parse_sql_statements(sql)
+        .unwrap_err()
+        .to_string()
+        .contains("VARIADIC argument must be last"),);
 }
diff --git a/src/sqlparser/tests/test_utils/mod.rs b/src/sqlparser/tests/test_utils/mod.rs
index 6df188a90ff29..c72dbc33d0635 100644
--- a/src/sqlparser/tests/test_utils/mod.rs
+++ b/src/sqlparser/tests/test_utils/mod.rs
@@ -11,6 +11,7 @@
 // limitations under the License.
 
 // Re-export everything from `src/test_utils.rs`.
+#[allow(unused_imports)]
 pub use risingwave_sqlparser::test_utils::*;
 
 // For the test-only macros we take a different approach of keeping them here
diff --git a/src/sqlparser/tests/testdata/alter.yaml b/src/sqlparser/tests/testdata/alter.yaml
index ad8afff801418..e23086e5f29b2 100644
--- a/src/sqlparser/tests/testdata/alter.yaml
+++ b/src/sqlparser/tests/testdata/alter.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: ALTER USER user WITH SUPERUSER CREATEDB PASSWORD 'password'
   formatted_sql: ALTER USER user WITH SUPERUSER CREATEDB PASSWORD 'password'
 - input: ALTER USER user RENAME TO another
diff --git a/src/sqlparser/tests/testdata/array.yaml b/src/sqlparser/tests/testdata/array.yaml
index aa655652b2b7f..20ed73ab9d910 100644
--- a/src/sqlparser/tests/testdata/array.yaml
+++ b/src/sqlparser/tests/testdata/array.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: CREATE TABLE t(a int[]);
   formatted_sql: CREATE TABLE t (a INT[])
 - input: CREATE TABLE t(a int[][]);
@@ -7,16 +7,19 @@
   formatted_sql: CREATE TABLE t (a INT[][][])
 - input: CREATE TABLE t(a int[);
   error_msg: |-
-    sql parser error: Expected ], found: ) at line:1, column:23
-    Near "CREATE TABLE t(a int["
+    sql parser error: invalid data_type
+    LINE 1: CREATE TABLE t(a int[);
+                                 ^
 - input: CREATE TABLE t(a int[[]);
   error_msg: |-
-    sql parser error: Expected ], found: [ at line:1, column:23
-    Near "CREATE TABLE t(a int["
+    sql parser error: invalid data_type
+    LINE 1: CREATE TABLE t(a int[[]);
+                                 ^
 - input: CREATE TABLE t(a int]);
   error_msg: |-
-    sql parser error: Expected ',' or ')' after column definition, found: ] at line:1, column:22
-    Near "CREATE TABLE t(a int"
+    sql parser error: expected ',' or ')' after column definition, found: ]
+    LINE 1: CREATE TABLE t(a int]);
+                                ^
 - input: SELECT foo[0] FROM foos
   formatted_sql: SELECT foo[0] FROM foos
 - input: SELECT foo[0][0] FROM foos
@@ -25,3 +28,49 @@
   formatted_sql: SELECT (CAST(ARRAY[ARRAY[2, 3]] AS INT[][]))[1][2]
 - input: SELECT ARRAY[]
   formatted_sql: SELECT ARRAY[]
+- input: SELECT ARRAY[[1,2],[3,4]]
+  formatted_sql: SELECT ARRAY[[1, 2], [3, 4]]
+- input: SELECT ARRAY[ARRAY[1,2],ARRAY[3,4]]
+  formatted_sql: SELECT ARRAY[ARRAY[1, 2], ARRAY[3, 4]]
+- input: SELECT ARRAY[[],[]]
+  formatted_sql: SELECT ARRAY[[], []]
+- input: SELECT ARRAY[ARRAY[],[]]
+  error_msg: |-
+    sql parser error: expected an expression, found: [
+    LINE 1: SELECT ARRAY[ARRAY[],[]]
+                                 ^
+- input: SELECT ARRAY[[],ARRAY[]]
+  error_msg: |-
+    sql parser error: expected [, found: ARRAY
+    LINE 1: SELECT ARRAY[[],ARRAY[]]
+                            ^
+- input: SELECT ARRAY[[1,2],3]
+  error_msg: |-
+    sql parser error: expected [, found: 3
+    LINE 1: SELECT ARRAY[[1,2],3]
+                               ^
+- input: SELECT ARRAY[1,[2,3]]
+  error_msg: |-
+    sql parser error: expected an expression, found: [
+    LINE 1: SELECT ARRAY[1,[2,3]]
+                           ^
+- input: SELECT ARRAY[ARRAY[1,2],[3,4]]
+  error_msg: |-
+    sql parser error: expected an expression, found: [
+    LINE 1: SELECT ARRAY[ARRAY[1,2],[3,4]]
+                                    ^
+- input: SELECT ARRAY[[1,2],ARRAY[3,4]]
+  error_msg: |-
+    sql parser error: expected [, found: ARRAY
+    LINE 1: SELECT ARRAY[[1,2],ARRAY[3,4]]
+                               ^
+- input: SELECT ARRAY[[1,2],[3] || [4]]
+  error_msg: |-
+    sql parser error: expected ], found: ||
+    LINE 1: SELECT ARRAY[[1,2],[3] || [4]]
+                                   ^
+- input: SELECT [1,2]
+  error_msg: |-
+    sql parser error: expected an expression, found: [
+    LINE 1: SELECT [1,2]
+                   ^
diff --git a/src/sqlparser/tests/testdata/as_of.yaml b/src/sqlparser/tests/testdata/as_of.yaml
new file mode 100644
index 0000000000000..2d7c81881988b
--- /dev/null
+++ b/src/sqlparser/tests/testdata/as_of.yaml
@@ -0,0 +1,23 @@
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
+- input: select * from t1 left join t2 FOR SYSTEM_TIME AS OF PROCTIME() on a1 = a2;
+  formatted_sql: SELECT * FROM t1 LEFT JOIN t2 FOR SYSTEM_TIME AS OF PROCTIME() ON a1 = a2
+- input: select * from t1 left join t2 FOR SYSTEM_TIME AS OF NOW() on a1 = a2;
+  formatted_sql: SELECT * FROM t1 LEFT JOIN t2 FOR SYSTEM_TIME AS OF PROCTIME() ON a1 = a2
+- input: select * from t1 left join t2 FOR SYSTEM_TIME AS OF 1 on a1 = a2;
+  formatted_sql: SELECT * FROM t1 LEFT JOIN t2 FOR SYSTEM_TIME AS OF 1 ON a1 = a2
+- input: select * from t1 left join t2 FOR SYSTEM_TIME AS OF 'string' on a1 = a2;
+  formatted_sql: SELECT * FROM t1 LEFT JOIN t2 FOR SYSTEM_TIME AS OF 'string' ON a1 = a2
+- input: select * from t1 left join t2 FOR SYSTEM_TIME AS OF PROCTIME on a1 = a2;
+  error_msg: |-
+    sql parser error: expected proctime(), now(), number or string
+    LINE 1: select * from t1 left join t2 FOR SYSTEM_TIME AS OF PROCTIME on a1 = a2;
+                                                                         ^
+- input: select * from t1 left join t2 FOR SYSTEM_VERSION AS OF 1 on a1 = a2;
+  formatted_sql: SELECT * FROM t1 LEFT JOIN t2 FOR SYSTEM_VERSION AS OF 1 ON a1 = a2
+- input: select * from t1 left join t2 FOR SYSTEM_VERSION AS OF 'string' on a1 = a2;
+  formatted_sql: SELECT * FROM t1 LEFT JOIN t2 FOR SYSTEM_VERSION AS OF 'string' ON a1 = a2
+- input: select * from t1 left join t2 FOR SYSTEM_VERSION AS OF PROCTIME() on a1 = a2;
+  error_msg: |-
+    sql parser error: expected number or string
+    LINE 1: select * from t1 left join t2 FOR SYSTEM_VERSION AS OF PROCTIME() on a1 = a2;
+                                                                   ^
diff --git a/src/sqlparser/tests/testdata/create.yaml b/src/sqlparser/tests/testdata/create.yaml
index 1fa2833267f81..4add7cde71a07 100644
--- a/src/sqlparser/tests/testdata/create.yaml
+++ b/src/sqlparser/tests/testdata/create.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: CREATE DATABASE t
   formatted_sql: CREATE DATABASE t
   formatted_ast: 'CreateDatabase { db_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: false }'
@@ -7,22 +7,24 @@
   formatted_ast: 'CreateDatabase { db_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: true }'
 - input: CREATE SCHEMA t
   formatted_sql: CREATE SCHEMA t
-  formatted_ast: 'CreateSchema { schema_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: false }'
+  formatted_ast: 'CreateSchema { schema_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: false, user_specified: None }'
 - input: CREATE SCHEMA IF NOT EXISTS t
   formatted_sql: CREATE SCHEMA IF NOT EXISTS t
-  formatted_ast: 'CreateSchema { schema_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: true }'
+  formatted_ast: 'CreateSchema { schema_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: true, user_specified: None }'
 - input: CREATE OR REPLACE TABLE t (a INT)
   formatted_sql: CREATE OR REPLACE TABLE t (a INT)
 - input: CREATE TABLE t (a INT, b INT) AS SELECT 1 AS b, 2 AS a
   formatted_sql: CREATE TABLE t (a INT, b INT) AS SELECT 1 AS b, 2 AS a
 - input: CREATE SOURCE src
   error_msg: |-
-    sql parser error: Expected description of the format, found: EOF at the end
-    Near "CREATE SOURCE src"
+    sql parser error: expected description of the format, found: EOF
+    LINE 1: CREATE SOURCE src
+                             ^
 - input: CREATE SOURCE src-a FORMAT PLAIN ENCODE JSON
   error_msg: |-
-    sql parser error: Expected description of the format, found: - at line:1, column:19
-    Near "CREATE SOURCE src"
+    sql parser error: expected description of the format, found: -
+    LINE 1: CREATE SOURCE src-a FORMAT PLAIN ENCODE JSON
+                             ^
 - input: CREATE SOURCE src FORMAT PLAIN ENCODE JSON
   formatted_sql: CREATE SOURCE src FORMAT PLAIN ENCODE JSON
 - input: CREATE SOURCE mysql_src with ( connector = 'mysql-cdc', hostname = 'localhost', port = '3306', database.name = 'mytest', server.id = '5601' )
@@ -31,17 +33,18 @@
   formatted_sql: CREATE TABLE sbtest10 (id INT PRIMARY KEY, k INT, c CHARACTER VARYING, pad CHARACTER VARYING) FROM sbtest TABLE 'mydb.sbtest10'
 - input: CREATE TABLE sbtest10 (id INT PRIMARY KEY, k INT, c CHARACTER VARYING, pad CHARACTER VARYING) FROM sbtest
   error_msg: |-
-    sql parser error: Expected TABLE, found: EOF at the end
-    Near "pad CHARACTER VARYING) FROM sbtest"
+    sql parser error: expected TABLE, found: EOF
+    LINE 1: CREATE TABLE sbtest10 (id INT PRIMARY KEY, k INT, c CHARACTER VARYING, pad CHARACTER VARYING) FROM sbtest
+                                                                                                                     ^
 - input: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.servers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.location = 'file://')
   formatted_sql: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.servers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.location = 'file://')
-  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: true, columns: [], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "servers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "location", quote_style: None }]), value: SingleQuotedString("file://") }] }), source_watermarks: [], include_column_options: [] } }'
+  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: true, columns: [], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "servers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "location", quote_style: None }]), value: SingleQuotedString("file://") }], key_encode: None }), source_watermarks: [], include_column_options: [] } }'
 - input: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.servers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.registry = 'http://')
   formatted_sql: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.servers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.registry = 'http://')
-  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: true, columns: [], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "servers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "registry", quote_style: None }]), value: SingleQuotedString("http://") }] }), source_watermarks: [], include_column_options: [] } }'
+  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: true, columns: [], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "servers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "registry", quote_style: None }]), value: SingleQuotedString("http://") }], key_encode: None }), source_watermarks: [], include_column_options: [] } }'
 - input: CREATE SOURCE bid (auction INTEGER, bidder INTEGER, price INTEGER, WATERMARK FOR auction AS auction - 1, "date_time" TIMESTAMP) with (connector = 'nexmark', nexmark.table.type = 'Bid', nexmark.split.num = '12',  nexmark.min.event.gap.in.ns = '0')
   formatted_sql: CREATE SOURCE bid (auction INT, bidder INT, price INT, "date_time" TIMESTAMP, WATERMARK FOR auction AS auction - 1) WITH (connector = 'nexmark', nexmark.table.type = 'Bid', nexmark.split.num = '12', nexmark.min.event.gap.in.ns = '0') FORMAT NATIVE ENCODE NATIVE
-  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: false, columns: [ColumnDef { name: Ident { value: "auction", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "bidder", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "price", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "date_time", quote_style: Some(''"'') }, data_type: Some(Timestamp(false)), collation: None, options: [] }], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "bid", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "connector", quote_style: None }]), value: SingleQuotedString("nexmark") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "table", quote_style: None }, Ident { value: "type", quote_style: None }]), value: SingleQuotedString("Bid") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "split", quote_style: None }, Ident { value: "num", quote_style: None }]), value: SingleQuotedString("12") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "min", quote_style: None }, Ident { value: "event", quote_style: None }, Ident { value: "gap", quote_style: None }, Ident { value: "in", quote_style: None }, Ident { value: "ns", quote_style: None }]), value: SingleQuotedString("0") }]), source_schema: V2(ConnectorSchema { format: Native, row_encode: Native, row_options: [] }), source_watermarks: [SourceWatermark { column: Ident { value: "auction", quote_style: None }, expr: BinaryOp { left: Identifier(Ident { value: "auction", quote_style: None }), op: Minus, right: Value(Number("1")) } }], include_column_options: [] } }'
+  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: false, columns: [ColumnDef { name: Ident { value: "auction", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "bidder", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "price", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "date_time", quote_style: Some(''"'') }, data_type: Some(Timestamp(false)), collation: None, options: [] }], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "bid", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "connector", quote_style: None }]), value: SingleQuotedString("nexmark") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "table", quote_style: None }, Ident { value: "type", quote_style: None }]), value: SingleQuotedString("Bid") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "split", quote_style: None }, Ident { value: "num", quote_style: None }]), value: SingleQuotedString("12") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "min", quote_style: None }, Ident { value: "event", quote_style: None }, Ident { value: "gap", quote_style: None }, Ident { value: "in", quote_style: None }, Ident { value: "ns", quote_style: None }]), value: SingleQuotedString("0") }]), source_schema: V2(ConnectorSchema { format: Native, row_encode: Native, row_options: [], key_encode: None }), source_watermarks: [SourceWatermark { column: Ident { value: "auction", quote_style: None }, expr: BinaryOp { left: Identifier(Ident { value: "auction", quote_style: None }), op: Minus, right: Value(Number("1")) } }], include_column_options: [] } }'
 - input: CREATE TABLE T (v1 INT, v2 STRUCT<v1 INT, v2 INT>)
   formatted_sql: CREATE TABLE T (v1 INT, v2 STRUCT<v1 INT, v2 INT>)
 - input: CREATE TABLE T (v1 INT, v2 STRUCT<v1 INT, v2 INT, v3 STRUCT<v1 INT, v2 INT>>)
@@ -49,7 +52,10 @@
 - input: CREATE TABLE T (a STRUCT<v1 INT>)
   formatted_sql: CREATE TABLE T (a STRUCT<v1 INT>)
 - input: CREATE TABLE T (FULL INT)
-  error_msg: 'sql parser error: syntax error at or near FULL at line:1, column:21'
+  error_msg: |-
+    sql parser error: syntax error at or near FULL
+    LINE 1: CREATE TABLE T (FULL INT)
+                                 ^
 - input: CREATE TABLE T ("FULL" INT)
   formatted_sql: CREATE TABLE T ("FULL" INT)
 - input: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMN("FULL")
@@ -58,22 +64,28 @@
   formatted_sql: CREATE TABLE T ("FULL" INT) ON CONFLICT IGNORE
 - input: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMN
   error_msg: |-
-    sql parser error: Expected (, found: EOF at the end
-    Near " CONFLICT OVERWRITE WITH VERSION COLUMN"
+    sql parser error: expected (, found: EOF
+    LINE 1: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMN
+                                                                                 ^
 - input: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMN(FULL
-  error_msg: 'sql parser error: syntax error at or near FULL at line:1, column:75'
+  error_msg: |-
+    sql parser error: syntax error at or near FULL
+    LINE 1: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMN(FULL
+                                                                                      ^
 - input: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMNFULL)
   error_msg: |-
-    sql parser error: Expected (, found: VERSION at line:1, column:63
-    Near "INT) ON CONFLICT OVERWRITE WITH"
+    sql parser error: expected (, found: VERSION
+    LINE 1: CREATE TABLE T ("FULL" INT) ON CONFLICT OVERWRITE WITH VERSION COLUMNFULL)
+                                                                   ^
 - input: CREATE TABLE T ("FULL" INT) ON CONFLICT DO UPDATE IF NOT NULL
   formatted_sql: CREATE TABLE T ("FULL" INT) ON CONFLICT DO UPDATE IF NOT NULL
 - input: CREATE USER user WITH SUPERUSER CREATEDB PASSWORD 'password'
   formatted_sql: CREATE USER user WITH SUPERUSER CREATEDB PASSWORD 'password'
 - input: CREATE SINK snk
   error_msg: |-
-    sql parser error: Expected FROM or AS after CREATE SINK sink_name, found: EOF at the end
-    Near "CREATE SINK snk"
+    sql parser error: expected FROM or AS after CREATE SINK sink_name, found: EOF
+    LINE 1: CREATE SINK snk
+                           ^
 - input: CREATE SINK IF NOT EXISTS snk FROM mv WITH (connector = 'mysql', mysql.endpoint = '127.0.0.1:3306', mysql.table = '<table_name>', mysql.database = '<database_name>', mysql.user = '<user_name>', mysql.password = '<password>')
   formatted_sql: CREATE SINK IF NOT EXISTS snk FROM mv WITH (connector = 'mysql', mysql.endpoint = '127.0.0.1:3306', mysql.table = '<table_name>', mysql.database = '<database_name>', mysql.user = '<user_name>', mysql.password = '<password>')
 - input: CREATE SINK IF NOT EXISTS snk AS SELECT count(*) AS cnt FROM mv WITH (connector = 'mysql', mysql.endpoint = '127.0.0.1:3306', mysql.table = '<table_name>', mysql.database = '<database_name>', mysql.user = '<user_name>', mysql.password = '<password>')
@@ -90,29 +102,53 @@
   formatted_sql: CREATE SINK snk INTO t AS SELECT * FROM t
 - input: CREATE SINK snk FROM mv WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:9092', topic = 'test_topic') format;
   error_msg: |-
-    sql parser error: Expected identifier, found: ; at line:1, column:123
-    Near " topic = 'test_topic') format;"
+    sql parser error: expected identifier, found: ;
+    LINE 1: CREATE SINK snk FROM mv WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:9092', topic = 'test_topic') format;
+                                                                                                                                           ^
 - input: create sink sk1 from tt where v1 % 10 = 0 with (connector='blackhole')
   error_msg: |-
-    sql parser error: Expected WITH, found: where at line:1, column:30
-    Near "create sink sk1 from tt"
+    sql parser error: expected WITH, found: where
+    LINE 1: create sink sk1 from tt where v1 % 10 = 0 with (connector='blackhole')
+                                    ^
 - input: CREATE SINK snk FROM mv WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:9092', topic = 'test_topic') format debezium;
   error_msg: |-
-    sql parser error: Expected ENCODE, found: ; at line:1, column:132
-    Near "topic = 'test_topic') format debezium"
+    sql parser error: expected ENCODE, found: ;
+    LINE 1: CREATE SINK snk FROM mv WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:9092', topic = 'test_topic') format debezium;
+                                                                                                                                                    ^
 - input: CREATE SINK snk FROM mv WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:9092', topic = 'test_topic') format debezium encode;
   error_msg: |-
-    sql parser error: Expected identifier, found: ; at line:1, column:139
-    Near " 'test_topic') format debezium encode;"
+    sql parser error: expected identifier, found: ;
+    LINE 1: CREATE SINK snk FROM mv WITH (connector = 'kafka', properties.bootstrap.server = '127.0.0.1:9092', topic = 'test_topic') format debezium encode;
+                                                                                                                                                           ^
 - input: create user tmp createdb nocreatedb
-  error_msg: 'sql parser error: conflicting or redundant options'
+  error_msg: |-
+    sql parser error: conflicting or redundant options
+    LINE 1: create user tmp createdb nocreatedb
+                                               ^
 - input: create user tmp createdb createdb
-  error_msg: 'sql parser error: conflicting or redundant options'
+  error_msg: |-
+    sql parser error: conflicting or redundant options
+    LINE 1: create user tmp createdb createdb
+                                             ^
 - input: create user tmp with password '123' password null
-  error_msg: 'sql parser error: conflicting or redundant options'
+  error_msg: |-
+    sql parser error: conflicting or redundant options
+    LINE 1: create user tmp with password '123' password null
+                                                             ^
 - input: create user tmp with encrypted password '' password null
-  error_msg: 'sql parser error: conflicting or redundant options'
+  error_msg: |-
+    sql parser error: conflicting or redundant options
+    LINE 1: create user tmp with encrypted password '' password null
+                                                                    ^
 - input: create user tmp with encrypted password null
   error_msg: |-
-    sql parser error: Expected literal string, found: null at line:1, column:45
-    Near " tmp with encrypted password null"
+    sql parser error: expected literal string, found: null
+    LINE 1: create user tmp with encrypted password null
+                                                    ^
+- input: CREATE SECRET secret1 WITH (backend = 'meta') AS 'demo-secret'
+  formatted_sql: CREATE SECRET secret1 WITH (backend = 'meta') AS 'demo-secret'
+- input: CREATE SECRET IF NOT EXISTS secret2 WITH (backend = 'meta') AS 'demo-secret
+  error_msg: |-
+    sql parser error: Unterminated string literal at line 1, column 76
+    LINE 1: CREATE SECRET IF NOT EXISTS secret2 WITH (backend = 'meta') AS 'demo-secret
+                                                                                       ^
diff --git a/src/sqlparser/tests/testdata/drop.yaml b/src/sqlparser/tests/testdata/drop.yaml
index 0fc49060328fb..6b8a70d0bf9be 100644
--- a/src/sqlparser/tests/testdata/drop.yaml
+++ b/src/sqlparser/tests/testdata/drop.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: DROP SOURCE src
   formatted_sql: DROP SOURCE src
   formatted_ast: 'Drop(DropStatement { object_type: Source, if_exists: false, object_name: ObjectName([Ident { value: "src", quote_style: None }]), drop_mode: None })'
@@ -16,3 +16,7 @@
   formatted_sql: DROP USER user
 - input: DROP USER IF EXISTS user
   formatted_sql: DROP USER IF EXISTS user
+- input: DROP SECRET secret
+  formatted_sql: DROP SECRET secret
+- input: DROP SECRET IF EXISTS secret
+  formatted_sql: DROP SECRET IF EXISTS secret
diff --git a/src/sqlparser/tests/testdata/drop_index.yaml b/src/sqlparser/tests/testdata/drop_index.yaml
index 130a6b36673dd..7242786d04ec0 100644
--- a/src/sqlparser/tests/testdata/drop_index.yaml
+++ b/src/sqlparser/tests/testdata/drop_index.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: DROP INDEX idx_a
   formatted_sql: DROP INDEX idx_a
   formatted_ast: 'Drop(DropStatement { object_type: Index, if_exists: false, object_name: ObjectName([Ident { value: "idx_a", quote_style: None }]), drop_mode: None })'
diff --git a/src/sqlparser/tests/testdata/extract.yaml b/src/sqlparser/tests/testdata/extract.yaml
new file mode 100644
index 0000000000000..ba38c3e25f261
--- /dev/null
+++ b/src/sqlparser/tests/testdata/extract.yaml
@@ -0,0 +1,6 @@
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
+- input: SELECT EXTRACT(0 FROM d)
+  error_msg: |-
+    sql parser error: expected date/time field
+    LINE 1: SELECT EXTRACT(0 FROM d)
+                           ^
diff --git a/src/sqlparser/tests/testdata/insert.yaml b/src/sqlparser/tests/testdata/insert.yaml
index 4d1bb445d0027..1b4c286c456bb 100644
--- a/src/sqlparser/tests/testdata/insert.yaml
+++ b/src/sqlparser/tests/testdata/insert.yaml
@@ -1,7 +1,8 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: INSERT public.customer (id, name, active) VALUES (1, 2, 3)
   error_msg: |-
-    sql parser error: Expected INTO, found: public at line:1, column:14
-    Near "INSERT"
+    sql parser error: expected INTO, found: public
+    LINE 1: INSERT public.customer (id, name, active) VALUES (1, 2, 3)
+                   ^
 - input: INSERT INTO t VALUES(1,3), (2,4) RETURNING *, a, a as aaa
   formatted_sql: INSERT INTO t VALUES (1, 3), (2, 4) RETURNING (*, a, a AS aaa)
diff --git a/src/sqlparser/tests/testdata/lambda.yaml b/src/sqlparser/tests/testdata/lambda.yaml
index 9336d9f8a4201..b61286d56dc93 100644
--- a/src/sqlparser/tests/testdata/lambda.yaml
+++ b/src/sqlparser/tests/testdata/lambda.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: select array_transform(array[1,2,3], |x| x * 2)
   formatted_sql: SELECT array_transform(ARRAY[1, 2, 3], |x| x * 2)
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { name: ObjectName([Ident { value: "array_transform", quote_style: None }]), args: [Unnamed(Expr(Array(Array { elem: [Value(Number("1")), Value(Number("2")), Value(Number("3"))], named: true }))), Unnamed(Expr(LambdaFunction { args: [Ident { value: "x", quote_style: None }], body: BinaryOp { left: Identifier(Ident { value: "x", quote_style: None }), op: Multiply, right: Value(Number("2")) } }))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
diff --git a/src/sqlparser/tests/testdata/overlay.yaml b/src/sqlparser/tests/testdata/overlay.yaml
new file mode 100644
index 0000000000000..07e51c46abb60
--- /dev/null
+++ b/src/sqlparser/tests/testdata/overlay.yaml
@@ -0,0 +1,13 @@
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
+- input: SELECT OVERLAY('abc' PLACING 'xyz' FROM 1)
+  formatted_sql: SELECT OVERLAY('abc' PLACING 'xyz' FROM 1)
+- input: SELECT OVERLAY('abc' PLACING 'xyz' FROM 1 FOR 2)
+  formatted_sql: SELECT OVERLAY('abc' PLACING 'xyz' FROM 1 FOR 2)
+- input: SELECT OVERLAY('abc', 'xyz')
+  error_msg: "sql parser error: \nLINE 1: SELECT OVERLAY('abc', 'xyz')\n                            ^"
+- input: SELECT OVERLAY('abc' PLACING 'xyz')
+  error_msg: "sql parser error: \nLINE 1: SELECT OVERLAY('abc' PLACING 'xyz')\n                                          ^"
+- input: SELECT OVERLAY('abc' PLACING 'xyz' FOR 2)
+  error_msg: "sql parser error: \nLINE 1: SELECT OVERLAY('abc' PLACING 'xyz' FOR 2)\n                                           ^"
+- input: SELECT OVERLAY('abc' PLACING 'xyz' FOR 2 FROM 1)
+  error_msg: "sql parser error: \nLINE 1: SELECT OVERLAY('abc' PLACING 'xyz' FOR 2 FROM 1)\n                                           ^"
diff --git a/src/sqlparser/tests/testdata/precedence.yaml b/src/sqlparser/tests/testdata/precedence.yaml
index 72673d1e93299..6aa1f6e42dd47 100644
--- a/src/sqlparser/tests/testdata/precedence.yaml
+++ b/src/sqlparser/tests/testdata/precedence.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: select 2 + 2 ^ 2
   formatted_sql: SELECT 2 + 2 ^ 2
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(BinaryOp { left: Value(Number("2")), op: Plus, right: BinaryOp { left: Value(Number("2")), op: BitwiseXor, right: Value(Number("2")) } })], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
diff --git a/src/sqlparser/tests/testdata/privilege.yaml b/src/sqlparser/tests/testdata/privilege.yaml
index 44d1b40d0bcc3..d165ecac6bd8d 100644
--- a/src/sqlparser/tests/testdata/privilege.yaml
+++ b/src/sqlparser/tests/testdata/privilege.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: GRANT ALL ON DATABASE database TO user1 WITH GRANT OPTION GRANTED BY user
   formatted_sql: GRANT ALL ON DATABASE database TO user1 WITH GRANT OPTION GRANTED BY user
   formatted_ast: 'Grant { privileges: All { with_privileges_keyword: false }, objects: Databases([ObjectName([Ident { value: "database", quote_style: None }])]), grantees: [Ident { value: "user1", quote_style: None }], with_grant_option: true, granted_by: Some(Ident { value: "user", quote_style: None }) }'
diff --git a/src/sqlparser/tests/testdata/qualified_operator.yaml b/src/sqlparser/tests/testdata/qualified_operator.yaml
index 854e6b5302e7f..59f3efdabbdbb 100644
--- a/src/sqlparser/tests/testdata/qualified_operator.yaml
+++ b/src/sqlparser/tests/testdata/qualified_operator.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: select 2 * 3 operator(pg_catalog.+) 4;
   formatted_sql: SELECT 2 * 3 OPERATOR(pg_catalog.+) 4
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(BinaryOp { left: BinaryOp { left: Value(Number("2")), op: Multiply, right: Value(Number("3")) }, op: PGQualified(QualifiedOperator { schema: Some(Ident { value: "pg_catalog", quote_style: None }), name: "+" }), right: Value(Number("4")) })], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
diff --git a/src/sqlparser/tests/testdata/select.yaml b/src/sqlparser/tests/testdata/select.yaml
index b8bcfe18e87e4..ad97f6568787b 100644
--- a/src/sqlparser/tests/testdata/select.yaml
+++ b/src/sqlparser/tests/testdata/select.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: SELECT sqrt(id) FROM foo
   formatted_sql: SELECT sqrt(id) FROM foo
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { name: ObjectName([Ident { value: "sqrt", quote_style: None }]), args: [Unnamed(Expr(Identifier(Ident { value: "id", quote_style: None })))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: None }))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "foo", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
@@ -48,37 +48,57 @@
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [Wildcard(Some([Identifier(Ident { value: "v1", quote_style: None })])), QualifiedWildcard(ObjectName([Ident { value: "bar", quote_style: None }]), Some([CompoundIdentifier([Ident { value: "foo", quote_style: None }, Ident { value: "v2", quote_style: None }])]))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "foo", quote_style: None }]), alias: None, as_of: None }, joins: [] }, TableWithJoins { relation: Table { name: ObjectName([Ident { value: "bar", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: SELECT v3 EXCEPT (v1, v2) FROM foo
   error_msg: |-
-    sql parser error: Expected SELECT, VALUES, or a subquery in the query body, found: v1 at line:1, column:21
-    Near "SELECT v3 EXCEPT ("
+    sql parser error: expected SELECT, VALUES, or a subquery in the query body, found: v1
+    LINE 1: SELECT v3 EXCEPT (v1, v2) FROM foo
+                              ^
 - input: SELECT * EXCEPT (V1, ) FROM foo
   error_msg: |-
-    sql parser error: Expected an expression:, found: ) at line:1, column:23
-    Near " * EXCEPT (V1, )"
+    sql parser error: expected an expression, found: )
+    LINE 1: SELECT * EXCEPT (V1, ) FROM foo
+                                 ^
 - input: SELECT * EXCEPT (v1 FROM foo
   error_msg: |-
-    sql parser error: Expected ( should be followed by ) after column names, found: FROM at line:1, column:25
-    Near "SELECT * EXCEPT (v1"
+    sql parser error: expected ( should be followed by ) after column names, found: FROM
+    LINE 1: SELECT * EXCEPT (v1 FROM foo
+                                ^
 - input: SELECT * FROM t LIMIT 1 FETCH FIRST ROWS ONLY
-  error_msg: 'sql parser error: Cannot specify both LIMIT and FETCH'
+  error_msg: |-
+    sql parser error: Cannot specify both LIMIT and FETCH
+    LINE 1: SELECT * FROM t LIMIT 1 FETCH FIRST ROWS ONLY
+                                          ^
 - input: SELECT * FROM t FETCH FIRST ROWS WITH TIES
-  error_msg: 'sql parser error: WITH TIES cannot be specified without ORDER BY clause'
+  error_msg: |-
+    sql parser error: WITH TIES cannot be specified without ORDER BY clause
+    LINE 1: SELECT * FROM t FETCH FIRST ROWS WITH TIES
+                                                      ^
 - input: select * from (select 1 from 1);
   error_msg: |-
-    sql parser error: Expected identifier, found: 1 at line:1, column:31
-    Near "from (select 1 from 1"
+    sql parser error: expected identifier, found: 1
+    LINE 1: select * from (select 1 from 1);
+                                         ^
 - input: select * from (select * from tumble(t, x, interval '10' minutes))
   error_msg: |-
-    sql parser error: Expected ), found: minutes at line:1, column:62
-    Near "(t, x, interval '10'"
+    sql parser error: expected ), found: minutes
+    LINE 1: select * from (select * from tumble(t, x, interval '10' minutes))
+                                                                    ^
 - input: SELECT 1, FROM t
-  error_msg: 'sql parser error: syntax error at or near FROM at line:1, column:15'
+  error_msg: |-
+    sql parser error: syntax error at or near FROM at line 1, column 11
+    LINE 1: SELECT 1, FROM t
+                           ^
 - input: SELECT 1, WHERE true
-  error_msg: 'sql parser error: syntax error at or near WHERE at line:1, column:16'
+  error_msg: |-
+    sql parser error: syntax error at or near WHERE at line 1, column 11
+    LINE 1: SELECT 1, WHERE true
+                            ^
 - input: SELECT timestamp with time zone '2022-10-01 12:00:00Z' AT TIME ZONE 'US/Pacific'
   formatted_sql: SELECT TIMESTAMP WITH TIME ZONE '2022-10-01 12:00:00Z' AT TIME ZONE 'US/Pacific'
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(AtTimeZone { timestamp: TypedString { data_type: Timestamp(true), value: "2022-10-01 12:00:00Z" }, time_zone: "US/Pacific" })], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: SELECT 0c6
-  error_msg: 'sql parser error: trailing junk after numeric literal at Line: 1, Column 8'
+  error_msg: |-
+    sql parser error: trailing junk after numeric literal at line 1, column 9
+    LINE 1: SELECT 0c6
+                    ^
 - input: SELECT 1e6
   formatted_sql: SELECT 1e6
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Value(Number("1e6")))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
@@ -111,22 +131,34 @@
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Value(Number("-0o755")))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: SELECT 0o129
   error_msg: |-
-    sql parser error: Expected end of statement, found: 9 at line:1, column:13
-    Near "SELECT 0o12"
+    sql parser error: expected end of statement, found: 9
+    LINE 1: SELECT 0o129
+                       ^
 - input: SELECT 0o3.5
   error_msg: |-
-    sql parser error: Expected end of statement, found: .5 at line:1, column:13
-    Near "SELECT 0o3"
+    sql parser error: expected end of statement, found: .5
+    LINE 1: SELECT 0o3.5
+                      ^
 - input: SELECT 0x
-  error_msg: 'sql parser error: incomplete integer literal at Line: 1, Column 8'
+  error_msg: |-
+    sql parser error: incomplete integer literal at line 1, column 10
+    LINE 1: SELECT 0x
+                     ^
 - input: SELECT 1::float(0)
-  error_msg: 'sql parser error: precision for type float must be at least 1 bit'
+  error_msg: |-
+    sql parser error: invalid data_type: precision must be in range 1..54
+    LINE 1: SELECT 1::float(0)
+                            ^
 - input: SELECT 1::float(54)
-  error_msg: 'sql parser error: precision for type float must be less than 54 bits'
+  error_msg: |-
+    sql parser error: invalid data_type: precision must be in range 1..54
+    LINE 1: SELECT 1::float(54)
+                            ^
 - input: SELECT 1::int(2)
   error_msg: |-
-    sql parser error: Expected end of statement, found: ( at line:1, column:14
-    Near "SELECT 1::int"
+    sql parser error: expected end of statement, found: (
+    LINE 1: SELECT 1::int(2)
+                         ^
 - input: select id1, a1, id2, a2 from stream as S join version FOR SYSTEM_TIME AS OF PROCTIME() AS V on id1= id2
   formatted_sql: SELECT id1, a1, id2, a2 FROM stream AS S JOIN version FOR SYSTEM_TIME AS OF PROCTIME() AS V ON id1 = id2
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Identifier(Ident { value: "id1", quote_style: None })), UnnamedExpr(Identifier(Ident { value: "a1", quote_style: None })), UnnamedExpr(Identifier(Ident { value: "id2", quote_style: None })), UnnamedExpr(Identifier(Ident { value: "a2", quote_style: None }))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "stream", quote_style: None }]), alias: Some(TableAlias { name: Ident { value: "S", quote_style: None }, columns: [] }), as_of: None }, joins: [Join { relation: Table { name: ObjectName([Ident { value: "version", quote_style: None }]), alias: Some(TableAlias { name: Ident { value: "V", quote_style: None }, columns: [] }), as_of: Some(ProcessTime) }, join_operator: Inner(On(BinaryOp { left: Identifier(Ident { value: "id1", quote_style: None }), op: Eq, right: Identifier(Ident { value: "id2", quote_style: None }) })) }] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
@@ -134,7 +166,10 @@
   formatted_sql: SELECT percentile_cont(0.3) FROM unnest(ARRAY[1, 2, 4, 5, 10]) AS x
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Function(Function { name: ObjectName([Ident { value: "percentile_cont", quote_style: None }]), args: [Unnamed(Expr(Value(Number("0.3"))))], variadic: false, over: None, distinct: false, order_by: [], filter: None, within_group: Some(OrderByExpr { expr: Identifier(Ident { value: "x", quote_style: None }), asc: Some(false), nulls_first: None }) }))], from: [TableWithJoins { relation: TableFunction { name: ObjectName([Ident { value: "unnest", quote_style: None }]), alias: Some(TableAlias { name: Ident { value: "x", quote_style: None }, columns: [] }), args: [Unnamed(Expr(Array(Array { elem: [Value(Number("1")), Value(Number("2")), Value(Number("4")), Value(Number("5")), Value(Number("10"))], named: true })))], with_ordinality: false }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: select percentile_cont(0.3) within group (order by x, y desc) from t
-  error_msg: 'sql parser error: only one arg in order by is expected here'
+  error_msg: |-
+    sql parser error: expected ), found: ,
+    LINE 1: select percentile_cont(0.3) within group (order by x, y desc) from t
+                                                                ^
 - input: select 'apple' ~~ 'app%'
   formatted_sql: SELECT 'apple' ~~ 'app%'
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(BinaryOp { left: Value(SingleQuotedString("apple")), op: PGLikeMatch, right: Value(SingleQuotedString("app%")) })], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
@@ -171,3 +206,8 @@
 - input: select 'a' not like ANY(array['a', null]);
   formatted_sql: SELECT 'a' NOT LIKE SOME(ARRAY['a', NULL])
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Like { negated: true, expr: Value(SingleQuotedString("a")), pattern: SomeOp(Array(Array { elem: [Value(SingleQuotedString("a")), Value(Null)], named: true })), escape_char: None })], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
+- input: selet 1;
+  error_msg: |-
+    sql parser error: expected statement, found: selet
+    LINE 1: selet 1;
+            ^
diff --git a/src/sqlparser/tests/testdata/set.yaml b/src/sqlparser/tests/testdata/set.yaml
index 947bbea7056c9..1bf2c35e165b7 100644
--- a/src/sqlparser/tests/testdata/set.yaml
+++ b/src/sqlparser/tests/testdata/set.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: SET TIME ZONE LOCAL
   formatted_sql: SET TIME ZONE LOCAL
 - input: SET TIME ZONE DEFAULT
@@ -17,9 +17,11 @@
   formatted_sql: SET search_path = 'default', 'my_path'
 - input: set search_path to default, 'my_path';
   error_msg: |-
-    sql parser error: Expected end of statement, found: , at line:1, column:28
-    Near "set search_path to default"
+    sql parser error: expected end of statement, found: ,
+    LINE 1: set search_path to default, 'my_path';
+                                      ^
 - input: set search_path to 'my_path', default;
   error_msg: |-
-    sql parser error: Expected parameter list value, found: default at line:1, column:36
-    Near "set search_path to 'my_path', default"
+    sql parser error: expected parameter list value, found: default
+    LINE 1: set search_path to 'my_path', default;
+                                          ^
diff --git a/src/sqlparser/tests/testdata/show.yaml b/src/sqlparser/tests/testdata/show.yaml
index c01a67184779d..a3df33bff2cbc 100644
--- a/src/sqlparser/tests/testdata/show.yaml
+++ b/src/sqlparser/tests/testdata/show.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: SHOW DATABASES
   formatted_sql: SHOW DATABASES
   formatted_ast: 'ShowObjects { object: Database, filter: None }'
@@ -53,3 +53,9 @@
 - input: SHOW INDEXES FROM t
   formatted_sql: SHOW INDEXES FROM t
   formatted_ast: 'ShowObjects { object: Indexes { table: ObjectName([Ident { value: "t", quote_style: None }]) }, filter: None }'
+- input: SHOW SECRETS
+  formatted_sql: SHOW SECRETS
+  formatted_ast: 'ShowObjects { object: Secret { schema: None }, filter: None }'
+- input: SHOW SECRETS FROM t
+  formatted_sql: SHOW SECRETS FROM t
+  formatted_ast: 'ShowObjects { object: Secret { schema: Some(Ident { value: "t", quote_style: None }) }, filter: None }'
diff --git a/src/sqlparser/tests/testdata/struct.yaml b/src/sqlparser/tests/testdata/struct.yaml
index 4898714fc5858..ee3cb8ff7abf9 100644
--- a/src/sqlparser/tests/testdata/struct.yaml
+++ b/src/sqlparser/tests/testdata/struct.yaml
@@ -1,5 +1,14 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: SELECT (1 * 2, 1.0)::foo;
   formatted_sql: SELECT CAST(ROW(1 * 2, 1.0) AS foo)
 - input: SELECT ROW(1 * 2, 1.0)::foo;
   formatted_sql: SELECT CAST(ROW(1 * 2, 1.0) AS foo)
+- input: SELECT NULL::STRUCT<v1 INT, v2 INT>
+  formatted_sql: SELECT CAST(NULL AS STRUCT<v1 INT, v2 INT>)
+- input: create table st (v1 int, v2 struct<v1 int, v2 struct<v1 varchar, v2 int>>, v3 struct<v1 varchar, v2 struct<v1 int, v2 varchar>>)
+  formatted_sql: CREATE TABLE st (v1 INT, v2 STRUCT<v1 INT, v2 STRUCT<v1 CHARACTER VARYING, v2 INT>>, v3 STRUCT<v1 CHARACTER VARYING, v2 STRUCT<v1 INT, v2 CHARACTER VARYING>>)
+- input: SELECT NULL::STRUCT<v1 INT, v2 INT>>
+  error_msg: |-
+    sql parser error: invalid data_type: unconsumed `>>`
+    LINE 1: SELECT NULL::STRUCT<v1 INT, v2 INT>>
+                                                ^
diff --git a/src/sqlparser/tests/testdata/subquery.yaml b/src/sqlparser/tests/testdata/subquery.yaml
index a79acbd7a317a..71fba982586e0 100644
--- a/src/sqlparser/tests/testdata/subquery.yaml
+++ b/src/sqlparser/tests/testdata/subquery.yaml
@@ -1,4 +1,4 @@
-# This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
+# This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: select a1 from a where exists (select 1 from b where a1 = b1);
   formatted_sql: SELECT a1 FROM a WHERE EXISTS (SELECT 1 FROM b WHERE a1 = b1)
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Identifier(Ident { value: "a1", quote_style: None }))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "a", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: Some(Exists(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Value(Number("1")))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "b", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: Some(BinaryOp { left: Identifier(Ident { value: "a1", quote_style: None }), op: Eq, right: Identifier(Ident { value: "b1", quote_style: None }) }), group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })), group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
@@ -7,12 +7,14 @@
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Identifier(Ident { value: "a1", quote_style: None }))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "a", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: Some(InSubquery { expr: Identifier(Ident { value: "a1", quote_style: None }), subquery: Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Identifier(Ident { value: "b1", quote_style: None }))], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "b", quote_style: None }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None }, negated: true }), group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
 - input: select a1 from a where a1 < ALL (select b1 from b);
   error_msg: |-
-    sql parser error: Expected ), found: b1 at line:1, column:43
-    Near "where a1 < ALL (select"
+    sql parser error: expected ), found: b1
+    LINE 1: select a1 from a where a1 < ALL (select b1 from b);
+                                                    ^
 - input: select a1 from a where a1 <> SOME (select b1 from b);
   error_msg: |-
-    sql parser error: Expected ), found: b1 at line:1, column:44
-    Near "where a1 <> SOME (select"
+    sql parser error: expected ), found: b1
+    LINE 1: select a1 from a where a1 <> SOME (select b1 from b);
+                                                      ^
 - input: select 1 + (select 2);
   formatted_sql: SELECT 1 + (SELECT 2)
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(BinaryOp { left: Value(Number("1")), op: Plus, right: Subquery(Query { with: None, body: Select(Select { distinct: All, projection: [UnnamedExpr(Value(Number("2")))], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None }) })], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml
index 76004ef7b3f75..b3664706dab6a 100644
--- a/src/storage/Cargo.toml
+++ b/src/storage/Cargo.toml
@@ -14,7 +14,7 @@ ignored = ["workspace-hack"]
 normal = ["workspace-hack"]
 
 [dependencies]
-ahash = "0.8"
+ahash = "0.8.7"
 anyhow = "1"
 arc-swap = "1"
 async-trait = "0.1"
@@ -35,6 +35,7 @@ itertools = { workspace = true }
 libc = "0.2"
 lz4 = "1.24.0"
 memcomparable = "0.2"
+metrics-prometheus = "0.7"
 more-asserts = "0.3"
 num-integer = "0.1"
 parking_lot = { workspace = true }
@@ -52,6 +53,8 @@ risingwave_object_store = { workspace = true }
 risingwave_pb = { workspace = true }
 risingwave_rpc_client = { workspace = true }
 scopeguard = "1"
+serde = { version = "1", features = ["derive"] }
+serde_bytes = "0.11"
 sled = "0.34.7"
 spin = "0.9"
 sync-point = { path = "../utils/sync-point" }
@@ -78,7 +81,7 @@ zstd = { version = "0.13", default-features = false }
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = { version = "0.16", default-features = false }
 libc = "0.2"
-nix = { version = "0.28", features = ["fs", "mman"] }
+nix = { version = "0.29", features = ["fs", "mman"] }
 
 [target.'cfg(target_os = "macos")'.dependencies]
 darwin-libproc = { git = "https://github.com/risingwavelabs/darwin-libproc.git", rev = "a502be24bd0971463f5bcbfe035a248d8ba503b7" }
@@ -89,6 +92,7 @@ mach2 = "0.4"
 workspace-hack = { path = "../workspace-hack" }
 
 [dev-dependencies]
+bincode = "1"
 criterion = { workspace = true, features = ["async_futures", "async_tokio"] }
 expect-test = "1"
 moka = { version = "0.12", features = ["future"] }
diff --git a/src/storage/backup/integration_tests/Makefile.toml b/src/storage/backup/integration_tests/Makefile.toml
index ad5b2810c9f17..3e177cda5d433 100644
--- a/src/storage/backup/integration_tests/Makefile.toml
+++ b/src/storage/backup/integration_tests/Makefile.toml
@@ -13,5 +13,6 @@ BACKUP_TEST_MCLI_CONFIG="${PREFIX_CONFIG}/mcli" \
 BACKUP_TEST_RW_ALL_IN_ONE="${BUILD_BIN}/risingwave" \
 RW_HUMMOCK_URL="hummock+minio://hummockadmin:hummockadmin@127.0.0.1:9301/hummock001" \
 RW_META_ADDR="http://127.0.0.1:5690" \
+RW_SQLITE_DB="${PREFIX_DATA}/sqlite/metadata.db" \
 bash "${test_root}/run_all.sh"
 """
diff --git a/src/storage/backup/integration_tests/common.sh b/src/storage/backup/integration_tests/common.sh
index 1c1af196691f0..d28ab1b9a9827 100644
--- a/src/storage/backup/integration_tests/common.sh
+++ b/src/storage/backup/integration_tests/common.sh
@@ -13,13 +13,52 @@ function clean_all_data {
   cargo make --allow-private clean-data 1>/dev/null 2>&1
 }
 
+function get_meta_store_type() {
+  meta_store_type=${META_STORE_TYPE:-etcd}
+  if [ "${meta_store_type}" = "sql" ]
+  then
+    if ! command -v sqlite3 &> /dev/null;
+    then
+        echo "SQLite3 is not installed."
+        exit 1
+    fi
+  fi
+  echo "${meta_store_type}"
+}
+
+echo "meta store: $(get_meta_store_type)"
+
+function clean_meta_store() {
+  meta_store_type=$(get_meta_store_type)
+  if [ "$(get_meta_store_type)" = "sql" ]; then
+    clean_sqlite_data
+  else
+    clean_etcd_data
+  fi
+}
+
+function clean_sqlite_data() {
+  tables=$(sqlite3 "${RW_SQLITE_DB}" "select name from sqlite_master where type='table';")
+  while IFS= read table
+  do
+    if [ -z "${table}" ]; then
+      break
+    fi
+    sqlite3 "${RW_SQLITE_DB}" "delete from [${table}]"
+  done <<< "${tables}"
+}
+
 function clean_etcd_data() {
   cargo make --allow-private clean-etcd-data 1>/dev/null 2>&1
 }
 
 function start_cluster() {
   stop_cluster
-  cargo make d ci-meta-backup-test 1>/dev/null 2>&1
+  if [ "$(get_meta_store_type)" = "sql" ]; then
+    cargo make d ci-meta-backup-test-sql 1>/dev/null 2>&1
+  else
+    cargo make d ci-meta-backup-test-etcd 1>/dev/null 2>&1
+  fi
   sleep 5
 }
 
@@ -34,8 +73,20 @@ function manual_compaction() {
   ${BACKUP_TEST_RW_ALL_IN_ONE} risectl hummock trigger-manual-compaction "$@" 1>/dev/null 2>&1
 }
 
+function start_meta_store_minio() {
+    if [ "$(get_meta_store_type)" = "sql" ]; then
+      start_sql_minio
+    else
+      start_etcd_minio
+    fi
+}
+
+function start_sql_minio() {
+  cargo make d ci-meta-backup-test-restore-sql 1>/dev/null 2>&1
+}
+
 function start_etcd_minio() {
-  cargo make d ci-meta-backup-test-restore 1>/dev/null 2>&1
+  cargo make d ci-meta-backup-test-restore-etcd 1>/dev/null 2>&1
 }
 
 function create_mvs() {
@@ -66,17 +117,19 @@ function delete_snapshot() {
 function restore() {
   local job_id
   job_id=$1
+  meta_store_type=$(get_meta_store_type)
   echo "try to restore snapshot ${job_id}"
   stop_cluster
-  clean_etcd_data
-  start_etcd_minio
+  clean_meta_store
+  start_meta_store_minio
   ${BACKUP_TEST_RW_ALL_IN_ONE} \
   risectl \
   meta \
   restore-meta \
-  --meta-store-type etcd \
+  --meta-store-type "${meta_store_type}" \
   --meta-snapshot-id "${job_id}" \
   --etcd-endpoints 127.0.0.1:2388 \
+  --sql-endpoint "sqlite://${RW_SQLITE_DB}?mode=rwc" \
   --backup-storage-url minio://hummockadmin:hummockadmin@127.0.0.1:9301/hummock001 \
   --hummock-storage-url minio://hummockadmin:hummockadmin@127.0.0.1:9301/hummock001 \
   1>/dev/null 2>&1
@@ -103,13 +156,15 @@ function execute_sql_and_expect() {
 }
 
 function get_max_committed_epoch() {
-  mce=$(${BACKUP_TEST_RW_ALL_IN_ONE} risectl hummock list-version --verbose 2>&1 | grep max_committed_epoch | sed -n 's/^.*max_committed_epoch: \(.*\),/\1/p')
-  echo "${mce}"
+  mce=$(${BACKUP_TEST_RW_ALL_IN_ONE} risectl hummock list-version --verbose 2>&1 | grep committed_epoch | sed -n 's/^.*committed_epoch: \(.*\),/\1/p')
+  # always take the smallest one
+  echo "${mce}"|sort -n |head -n 1
 }
 
 function get_safe_epoch() {
   safe_epoch=$(${BACKUP_TEST_RW_ALL_IN_ONE} risectl hummock list-version --verbose 2>&1 | grep safe_epoch | sed -n 's/^.*safe_epoch: \(.*\),/\1/p')
-  echo "${safe_epoch}"
+  # always take the largest one
+  echo "${safe_epoch}"|sort -n -r |head -n 1
 }
 
 function get_total_sst_count() {
@@ -118,17 +173,13 @@ function get_total_sst_count() {
 }
 
 function get_max_committed_epoch_in_backup() {
-  local id
-  id=$1
-  sed_str="s/.*{\"id\":${id},\"hummock_version_id\":.*,\"ssts\":\[.*\],\"max_committed_epoch\":\([[:digit:]]*\),\"safe_epoch\":.*}.*/\1/p"
+  sed_str="s/.*\"state_table_info\":{\"[[:digit:]]*\":{\"committedEpoch\":\"\([[:digit:]]*\)\",\"safeEpoch\":\"\([[:digit:]]*\)\".*/\1/p"
   ${BACKUP_TEST_MCLI} -C "${BACKUP_TEST_MCLI_CONFIG}" \
   cat "hummock-minio/hummock001/backup/manifest.json" | sed -n "${sed_str}"
 }
 
 function get_safe_epoch_in_backup() {
-  local id
-  id=$1
-  sed_str="s/.*{\"id\":${id},\"hummock_version_id\":.*,\"ssts\":\[.*\],\"max_committed_epoch\":.*,\"safe_epoch\":\([[:digit:]]*\).*}.*/\1/p"
+  sed_str="s/.*\"state_table_info\":{\"[[:digit:]]*\":{\"committedEpoch\":\"\([[:digit:]]*\)\",\"safeEpoch\":\"\([[:digit:]]*\)\".*/\2/p"
   ${BACKUP_TEST_MCLI} -C "${BACKUP_TEST_MCLI_CONFIG}" \
   cat "hummock-minio/hummock001/backup/manifest.json" | sed -n "${sed_str}"
 }
diff --git a/src/storage/backup/integration_tests/test_query_backup.sh b/src/storage/backup/integration_tests/test_query_backup.sh
index b08216267f27d..dbba68c7e8564 100644
--- a/src/storage/backup/integration_tests/test_query_backup.sh
+++ b/src/storage/backup/integration_tests/test_query_backup.sh
@@ -24,8 +24,8 @@ select * from t1;
 
 job_id=$(backup)
 echo "${job_id}"
-backup_mce=$(get_max_committed_epoch_in_backup "${job_id}")
-backup_safe_epoch=$(get_safe_epoch_in_backup "${job_id}")
+backup_mce=$(get_max_committed_epoch_in_backup)
+backup_safe_epoch=$(get_safe_epoch_in_backup)
 echo "backup MCE: ${backup_mce}"
 echo "backup safe_epoch: ${backup_safe_epoch}"
 
@@ -55,10 +55,10 @@ do
   sleep 5
   min_pinned_snapshot=$(get_min_pinned_snapshot)
 done
-# safe epoch equals to 0 because no compaction has been done
+# safe epoch equals to backup_safe_epoch because no compaction has been done
 safe_epoch=$(get_safe_epoch)
 echo "safe epoch after unpin: ${safe_epoch}"
-[ "${safe_epoch}" -eq 0 ]
+[ "${safe_epoch}" -eq "${backup_safe_epoch}" ]
 # trigger a compaction to increase safe_epoch
 manual_compaction -c 3 -l 0
 # wait until compaction is done
@@ -68,6 +68,7 @@ do
   sleep 5
 done
 echo "safe epoch after compaction: ${safe_epoch}"
+[ "${safe_epoch}" -gt "${backup_safe_epoch}" ]
 
 echo "QUERY_EPOCH=safe_epoch. It should fail because it's not covered by any backup"
 execute_sql_and_expect \
@@ -83,7 +84,6 @@ select * from t1;" \
 
 echo "QUERY_EPOCH=backup_safe_epoch + 1<<16 + 1, it's < safe_epoch but covered by backup"
 epoch=$((backup_safe_epoch + (1<<16) + 1))
-[ ${epoch} -eq 65537 ]
 execute_sql_and_expect \
 "SET QUERY_EPOCH TO ${epoch};
 select * from t1;" \
diff --git a/src/storage/backup/src/error.rs b/src/storage/backup/src/error.rs
index 8bac7c4b1c5d0..2a65584749b9b 100644
--- a/src/storage/backup/src/error.rs
+++ b/src/storage/backup/src/error.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use bincode::Error;
 use risingwave_common::error::BoxedError;
 use thiserror::Error;
 
@@ -61,10 +60,3 @@ pub enum BackupError {
         anyhow::Error,
     ),
 }
-
-impl From<bincode::Error> for BackupError {
-    fn from(value: Error) -> Self {
-        // TODO: match error
-        BackupError::Other(value.into())
-    }
-}
diff --git a/src/storage/backup/src/lib.rs b/src/storage/backup/src/lib.rs
index a1acfde20e400..6b569277c54dd 100644
--- a/src/storage/backup/src/lib.rs
+++ b/src/storage/backup/src/lib.rs
@@ -32,13 +32,16 @@ pub mod meta_snapshot_v1;
 pub mod meta_snapshot_v2;
 pub mod storage;
 
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::hash::Hasher;
 
 use itertools::Itertools;
+use risingwave_common::catalog::TableId;
+use risingwave_common::RW_VERSION;
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::{HummockSstableObjectId, HummockVersionId};
 use risingwave_pb::backup_service::{PbMetaSnapshotManifest, PbMetaSnapshotMetadata};
+use risingwave_pb::hummock::PbStateTableInfo;
 use serde::{Deserialize, Serialize};
 
 use crate::error::{BackupError, BackupResult};
@@ -57,6 +60,9 @@ pub struct MetaSnapshotMetadata {
     #[serde(default)]
     pub format_version: u32,
     pub remarks: Option<String>,
+    #[serde(with = "table_id_key_map")]
+    pub state_table_info: HashMap<TableId, PbStateTableInfo>,
+    pub rw_version: Option<String>,
 }
 
 impl MetaSnapshotMetadata {
@@ -71,9 +77,11 @@ impl MetaSnapshotMetadata {
             hummock_version_id: v.id,
             ssts: v.get_object_ids(),
             max_committed_epoch: v.max_committed_epoch,
-            safe_epoch: v.safe_epoch,
+            safe_epoch: v.visible_table_safe_epoch(),
             format_version,
             remarks,
+            state_table_info: v.state_table_info.info().clone(),
+            rw_version: Some(RW_VERSION.to_owned()),
         }
     }
 }
@@ -112,6 +120,12 @@ impl From<&MetaSnapshotMetadata> for PbMetaSnapshotMetadata {
             safe_epoch: m.safe_epoch,
             format_version: Some(m.format_version),
             remarks: m.remarks.clone(),
+            state_table_info: m
+                .state_table_info
+                .iter()
+                .map(|(t, i)| (t.table_id, i.clone()))
+                .collect(),
+            rw_version: m.rw_version.clone(),
         }
     }
 }
@@ -124,3 +138,40 @@ impl From<&MetaSnapshotManifest> for PbMetaSnapshotManifest {
         }
     }
 }
+
+mod table_id_key_map {
+    use std::collections::HashMap;
+    use std::str::FromStr;
+
+    use risingwave_common::catalog::TableId;
+    use risingwave_pb::hummock::PbStateTableInfo;
+    use serde::{Deserialize, Deserializer, Serialize, Serializer};
+
+    pub fn serialize<S>(
+        map: &HashMap<TableId, PbStateTableInfo>,
+        serializer: S,
+    ) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let map_as_str: HashMap<String, &PbStateTableInfo> =
+            map.iter().map(|(k, v)| (k.to_string(), v)).collect();
+        map_as_str.serialize(serializer)
+    }
+
+    pub fn deserialize<'de, D>(
+        deserializer: D,
+    ) -> Result<HashMap<TableId, PbStateTableInfo>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let map_as_str: HashMap<String, PbStateTableInfo> = HashMap::deserialize(deserializer)?;
+        map_as_str
+            .into_iter()
+            .map(|(k, v)| {
+                let key = u32::from_str(&k).map_err(serde::de::Error::custom)?;
+                Ok((TableId::new(key), v))
+            })
+            .collect()
+    }
+}
diff --git a/src/storage/backup/src/meta_snapshot_v2.rs b/src/storage/backup/src/meta_snapshot_v2.rs
index 20f620f8b6545..bec07a80cf19d 100644
--- a/src/storage/backup/src/meta_snapshot_v2.rs
+++ b/src/storage/backup/src/meta_snapshot_v2.rs
@@ -16,63 +16,137 @@ use std::fmt::{Display, Formatter};
 
 use bytes::{Buf, BufMut};
 use risingwave_hummock_sdk::version::HummockVersion;
-use risingwave_meta_model_v2 as model_v2;
 use serde::{Deserialize, Serialize};
 
 use crate::meta_snapshot::{MetaSnapshot, Metadata};
-use crate::BackupResult;
-
+use crate::{BackupError, BackupResult};
 pub type MetaSnapshotV2 = MetaSnapshot<MetadataV2>;
 
-#[derive(Default)]
-pub struct MetadataV2 {
-    pub cluster_id: String,
-    pub hummock_version: HummockVersion,
-    pub version_stats: model_v2::hummock_version_stats::Model,
-    pub compaction_configs: Vec<model_v2::compaction_config::Model>,
-    // TODO other metadata
+impl From<serde_json::Error> for BackupError {
+    fn from(value: serde_json::Error) -> Self {
+        BackupError::Other(value.into())
+    }
+}
+
+/// Add new item in the end. Do not change the order.
+#[macro_export]
+macro_rules! for_all_metadata_models_v2 {
+    ($macro:ident) => {
+        $macro! {
+            {seaql_migrations, risingwave_meta_model_v2::serde_seaql_migration},
+            {version_stats, risingwave_meta_model_v2::hummock_version_stats},
+            {compaction_configs, risingwave_meta_model_v2::compaction_config},
+            {actors, risingwave_meta_model_v2::actor},
+            {clusters, risingwave_meta_model_v2::cluster},
+            {actor_dispatchers, risingwave_meta_model_v2::actor_dispatcher},
+            {catalog_versions, risingwave_meta_model_v2::catalog_version},
+            {connections, risingwave_meta_model_v2::connection},
+            {databases, risingwave_meta_model_v2::database},
+            {fragments, risingwave_meta_model_v2::fragment},
+            {functions, risingwave_meta_model_v2::function},
+            {indexes, risingwave_meta_model_v2::index},
+            {objects, risingwave_meta_model_v2::object},
+            {object_dependencies, risingwave_meta_model_v2::object_dependency},
+            {schemas, risingwave_meta_model_v2::schema},
+            {sinks, risingwave_meta_model_v2::sink},
+            {sources, risingwave_meta_model_v2::source},
+            {streaming_jobs, risingwave_meta_model_v2::streaming_job},
+            {subscriptions, risingwave_meta_model_v2::subscription},
+            {system_parameters, risingwave_meta_model_v2::system_parameter},
+            {tables, risingwave_meta_model_v2::table},
+            {users, risingwave_meta_model_v2::user},
+            {user_privileges, risingwave_meta_model_v2::user_privilege},
+            {views, risingwave_meta_model_v2::view},
+            {workers, risingwave_meta_model_v2::worker},
+            {worker_properties, risingwave_meta_model_v2::worker_property},
+            {hummock_sequences, risingwave_meta_model_v2::hummock_sequence},
+            {session_parameters, risingwave_meta_model_v2::session_parameter},
+            {secrets, risingwave_meta_model_v2::secret}
+        }
+    };
+}
+
+macro_rules! define_metadata_v2 {
+    ($({ $name:ident, $mod_path:ident::$mod_name:ident }),*) => {
+        #[derive(Default)]
+        pub struct MetadataV2 {
+            pub hummock_version: HummockVersion,
+            $(
+                pub $name: Vec<$mod_path::$mod_name::Model>,
+            )*
+        }
+    };
+}
+
+for_all_metadata_models_v2!(define_metadata_v2);
+
+macro_rules! define_encode_metadata {
+    ($( {$name:ident, $mod_path:ident::$mod_name:ident} ),*) => {
+        fn encode_metadata(
+          metadata: &MetadataV2,
+          buf: &mut Vec<u8>,
+        ) -> BackupResult<()> {
+            let mut _idx = 0;
+            $(
+                if _idx == 1 {
+                    put_1(buf, &metadata.hummock_version.to_protobuf())?;
+                }
+                put_n(buf, &metadata.$name)?;
+                _idx += 1;
+            )*
+            Ok(())
+        }
+    };
 }
 
+for_all_metadata_models_v2!(define_encode_metadata);
+
+macro_rules! define_decode_metadata {
+    ($( {$name:ident, $mod_path:ident::$mod_name:ident} ),*) => {
+        fn decode_metadata(
+          metadata: &mut MetadataV2,
+          mut buf: &[u8],
+        ) -> BackupResult<()> {
+            let mut _idx = 0;
+            $(
+                if _idx == 1 {
+                    metadata.hummock_version = HummockVersion::from_persisted_protobuf(&get_1(&mut buf)?);
+                }
+                metadata.$name = get_n(&mut buf)?;
+                _idx += 1;
+            )*
+            Ok(())
+        }
+    };
+}
+
+for_all_metadata_models_v2!(define_decode_metadata);
+
 impl Display for MetadataV2 {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        writeln!(f, "cluster_id:")?;
-        writeln!(f, "{:#?}", self.cluster_id)?;
-        writeln!(f, "hummock_version:")?;
-        writeln!(f, "{:#?}", self.hummock_version)?;
-        writeln!(f, "version_stats:")?;
-        writeln!(f, "{:#?}", self.version_stats)?;
-        writeln!(f, "compaction_configs:")?;
-        writeln!(f, "{:#?}", self.compaction_configs)?;
-        // TODO: other metadata
+        writeln!(f, "clusters: {:#?}", self.clusters)?;
+        writeln!(
+            f,
+            "Hummock version: id {}, max_committed_epoch: {}",
+            self.hummock_version.id, self.hummock_version.max_committed_epoch
+        )?;
+        // optionally dump other metadata
         Ok(())
     }
 }
 
 impl Metadata for MetadataV2 {
     fn encode_to(&self, buf: &mut Vec<u8>) -> BackupResult<()> {
-        put_with_len_prefix(buf, &self.cluster_id)?;
-        put_with_len_prefix(buf, &self.hummock_version.to_protobuf())?;
-        put_with_len_prefix(buf, &self.version_stats)?;
-        put_with_len_prefix(buf, &self.compaction_configs)?;
-        // TODO: other metadata
-        Ok(())
+        encode_metadata(self, buf)
     }
 
-    fn decode(mut buf: &[u8]) -> BackupResult<Self>
+    fn decode(buf: &[u8]) -> BackupResult<Self>
     where
         Self: Sized,
     {
-        let cluster_id = get_with_len_prefix(&mut buf)?;
-        let pb_hummock_version = get_with_len_prefix(&mut buf)?;
-        let version_stats = get_with_len_prefix(&mut buf)?;
-        let compaction_configs = get_with_len_prefix(&mut buf)?;
-        // TODO: other metadata
-        Ok(Self {
-            cluster_id,
-            hummock_version: HummockVersion::from_persisted_protobuf(&pb_hummock_version),
-            version_stats,
-            compaction_configs,
-        })
+        let mut metadata = Self::default();
+        decode_metadata(&mut metadata, buf)?;
+        Ok(metadata)
     }
 
     fn hummock_version_ref(&self) -> &HummockVersion {
@@ -84,8 +158,39 @@ impl Metadata for MetadataV2 {
     }
 }
 
-fn put_with_len_prefix<T: Serialize>(buf: &mut Vec<u8>, data: &T) -> Result<(), bincode::Error> {
-    let b = bincode::serialize(data)?;
+fn put_n<T: Serialize>(buf: &mut Vec<u8>, data: &[T]) -> Result<(), serde_json::Error> {
+    buf.put_u32_le(
+        data.len()
+            .try_into()
+            .unwrap_or_else(|_| panic!("cannot convert {} into u32", data.len())),
+    );
+    for d in data {
+        put_with_len_prefix(buf, d)?;
+    }
+    Ok(())
+}
+
+fn put_1<T: Serialize>(buf: &mut Vec<u8>, data: &T) -> Result<(), serde_json::Error> {
+    put_n(buf, &[data])
+}
+
+fn get_n<'a, T: Deserialize<'a>>(buf: &mut &'a [u8]) -> Result<Vec<T>, serde_json::Error> {
+    let n = buf.get_u32_le() as usize;
+    let mut elements = Vec::with_capacity(n);
+    for _ in 0..n {
+        elements.push(get_with_len_prefix(buf)?);
+    }
+    Ok(elements)
+}
+
+fn get_1<'a, T: Deserialize<'a>>(buf: &mut &'a [u8]) -> Result<T, serde_json::Error> {
+    let elements = get_n(buf)?;
+    assert_eq!(elements.len(), 1);
+    Ok(elements.into_iter().next().unwrap())
+}
+
+fn put_with_len_prefix<T: Serialize>(buf: &mut Vec<u8>, data: &T) -> Result<(), serde_json::Error> {
+    let b = serde_json::to_vec(data)?;
     buf.put_u32_le(
         b.len()
             .try_into()
@@ -95,9 +200,9 @@ fn put_with_len_prefix<T: Serialize>(buf: &mut Vec<u8>, data: &T) -> Result<(),
     Ok(())
 }
 
-fn get_with_len_prefix<'a, T: Deserialize<'a>>(buf: &mut &'a [u8]) -> Result<T, bincode::Error> {
+fn get_with_len_prefix<'a, T: Deserialize<'a>>(buf: &mut &'a [u8]) -> Result<T, serde_json::Error> {
     let len = buf.get_u32_le() as usize;
-    let d = bincode::deserialize(&buf[..len])?;
+    let d = serde_json::from_slice(&buf[..len])?;
     buf.advance(len);
     Ok(d)
 }
diff --git a/src/storage/backup/src/storage.rs b/src/storage/backup/src/storage.rs
index 54d2eaa741c94..59a304080e65c 100644
--- a/src/storage/backup/src/storage.rs
+++ b/src/storage/backup/src/storage.rs
@@ -192,7 +192,7 @@ pub async fn unused() -> ObjectStoreMetaSnapshotStorage {
         Arc::new(ObjectStoreImpl::InMem(MonitoredObjectStore::new(
             InMemObjectStore::new(),
             Arc::new(ObjectStoreMetrics::unused()),
-            ObjectStoreConfig::default(),
+            Arc::new(ObjectStoreConfig::default()),
         ))),
     )
     .await
diff --git a/src/storage/benches/bench_block_cache.rs b/src/storage/benches/bench_block_cache.rs
index 497d532732daf..f00e6a7b0e2be 100644
--- a/src/storage/benches/bench_block_cache.rs
+++ b/src/storage/benches/bench_block_cache.rs
@@ -21,14 +21,15 @@ use std::time::{Duration, Instant};
 use async_trait::async_trait;
 use bytes::{BufMut, Bytes, BytesMut};
 use criterion::{criterion_group, criterion_main, Criterion};
-use foyer::memory as foyer;
 use moka::future::Cache;
 use rand::rngs::SmallRng;
 use rand::{RngCore, SeedableRng};
 use risingwave_common::cache::CachePriority;
 use risingwave_storage::hummock::{HummockError, HummockResult, LruCache};
+use serde::{Deserialize, Serialize};
 use tokio::runtime::{Builder, Runtime};
 
+#[derive(Debug, Serialize, Deserialize)]
 pub struct Block {
     sst: u64,
     offset: u64,
@@ -123,16 +124,13 @@ pub struct FoyerCache {
 
 impl FoyerCache {
     pub fn lru(capacity: usize, fake_io_latency: Duration) -> Self {
-        let inner = foyer::Cache::lru(foyer::LruCacheConfig {
-            capacity,
-            shards: 8,
-            eviction_config: foyer::LruConfig {
-                high_priority_pool_ratio: 0.0,
-            },
-            object_pool_capacity: 8 * 1024,
-            hash_builder: ahash::RandomState::default(),
-            event_listener: foyer::DefaultCacheEventListener::default(),
-        });
+        let inner = foyer::CacheBuilder::new(capacity)
+            .with_shards(8)
+            .with_eviction_config(foyer::LruConfig {
+                high_priority_pool_ratio: 0.8,
+            })
+            .with_object_pool_capacity(8 * 1024)
+            .build();
         Self {
             inner,
             fake_io_latency,
@@ -140,19 +138,16 @@ impl FoyerCache {
     }
 
     pub fn lfu(capacity: usize, fake_io_latency: Duration) -> Self {
-        let inner = foyer::Cache::lfu(foyer::LfuCacheConfig {
-            capacity,
-            shards: 8,
-            eviction_config: foyer::LfuConfig {
+        let inner = foyer::CacheBuilder::new(capacity)
+            .with_shards(8)
+            .with_eviction_config(foyer::LfuConfig {
                 window_capacity_ratio: 0.1,
                 protected_capacity_ratio: 0.8,
                 cmsketch_eps: 0.001,
                 cmsketch_confidence: 0.9,
-            },
-            object_pool_capacity: 8 * 1024,
-            hash_builder: ahash::RandomState::default(),
-            event_listener: foyer::DefaultCacheEventListener::default(),
-        });
+            })
+            .with_object_pool_capacity(8 * 1024)
+            .build();
         Self {
             inner,
             fake_io_latency,
@@ -165,12 +160,12 @@ impl CacheBase for FoyerCache {
     async fn try_get_with(&self, sst_object_id: u64, block_idx: u64) -> HummockResult<Arc<Block>> {
         let entry = self
             .inner
-            .entry((sst_object_id, block_idx), || {
+            .fetch((sst_object_id, block_idx), || {
                 let latency = self.fake_io_latency;
                 async move {
                     get_fake_block(sst_object_id, block_idx, latency)
                         .await
-                        .map(|block| (Arc::new(block), 1, foyer::CacheContext::Default))
+                        .map(|block| (Arc::new(block), foyer::CacheContext::Default))
                 }
             })
             .await?;
@@ -178,6 +173,72 @@ impl CacheBase for FoyerCache {
     }
 }
 
+pub struct FoyerHybridCache {
+    inner: foyer::HybridCache<(u64, u64), Arc<Block>>,
+    fake_io_latency: Duration,
+}
+
+impl FoyerHybridCache {
+    pub async fn lru(capacity: usize, fake_io_latency: Duration) -> Self {
+        let inner = foyer::HybridCacheBuilder::new()
+            .memory(capacity)
+            .with_shards(8)
+            .with_eviction_config(foyer::LruConfig {
+                high_priority_pool_ratio: 0.8,
+            })
+            .with_object_pool_capacity(8 * 1024)
+            .storage()
+            .build()
+            .await
+            .unwrap();
+        Self {
+            inner,
+            fake_io_latency,
+        }
+    }
+
+    pub async fn lfu(capacity: usize, fake_io_latency: Duration) -> Self {
+        let inner = foyer::HybridCacheBuilder::new()
+            .memory(capacity)
+            .with_shards(8)
+            .with_eviction_config(foyer::LfuConfig {
+                window_capacity_ratio: 0.1,
+                protected_capacity_ratio: 0.8,
+                cmsketch_eps: 0.001,
+                cmsketch_confidence: 0.9,
+            })
+            .with_object_pool_capacity(8 * 1024)
+            .storage()
+            .build()
+            .await
+            .unwrap();
+        Self {
+            inner,
+            fake_io_latency,
+        }
+    }
+}
+
+#[async_trait]
+impl CacheBase for FoyerHybridCache {
+    async fn try_get_with(&self, sst_object_id: u64, block_idx: u64) -> HummockResult<Arc<Block>> {
+        let entry = self
+            .inner
+            .fetch((sst_object_id, block_idx), || {
+                let latency = self.fake_io_latency;
+                async move {
+                    get_fake_block(sst_object_id, block_idx, latency)
+                        .await
+                        .map(|block| (Arc::new(block), foyer::CacheContext::Default))
+                        .map_err(anyhow::Error::from)
+                }
+            })
+            .await
+            .map_err(HummockError::foyer_error)?;
+        Ok(entry.value().clone())
+    }
+}
+
 static IO_COUNT: AtomicUsize = AtomicUsize::new(0);
 
 async fn get_fake_block(sst: u64, offset: u64, io_latency: Duration) -> HummockResult<Block> {
@@ -260,6 +321,20 @@ fn bench_block_cache(c: &mut Criterion) {
     bench_cache(block_cache, c, 10000);
     let block_cache = Arc::new(FoyerCache::lfu(2048, Duration::from_millis(0)));
     bench_cache(block_cache, c, 10000);
+    let block_cache = Arc::new(
+        tokio::runtime::Builder::new_current_thread()
+            .build()
+            .unwrap()
+            .block_on(FoyerHybridCache::lru(2048, Duration::from_millis(0))),
+    );
+    bench_cache(block_cache, c, 10000);
+    let block_cache = Arc::new(
+        tokio::runtime::Builder::new_current_thread()
+            .build()
+            .unwrap()
+            .block_on(FoyerHybridCache::lfu(2048, Duration::from_millis(0))),
+    );
+    bench_cache(block_cache, c, 10000);
 
     let block_cache = Arc::new(MokaCache::new(2048, Duration::from_millis(1)));
     bench_cache(block_cache, c, 1000);
@@ -269,6 +344,20 @@ fn bench_block_cache(c: &mut Criterion) {
     bench_cache(block_cache, c, 1000);
     let block_cache = Arc::new(FoyerCache::lfu(2048, Duration::from_millis(1)));
     bench_cache(block_cache, c, 1000);
+    let block_cache = Arc::new(
+        tokio::runtime::Builder::new_current_thread()
+            .build()
+            .unwrap()
+            .block_on(FoyerHybridCache::lru(2048, Duration::from_millis(1))),
+    );
+    bench_cache(block_cache, c, 1000);
+    let block_cache = Arc::new(
+        tokio::runtime::Builder::new_current_thread()
+            .build()
+            .unwrap()
+            .block_on(FoyerHybridCache::lfu(2048, Duration::from_millis(1))),
+    );
+    bench_cache(block_cache, c, 1000);
 
     let block_cache = Arc::new(MokaCache::new(256, Duration::from_millis(10)));
     bench_cache(block_cache, c, 200);
@@ -278,6 +367,20 @@ fn bench_block_cache(c: &mut Criterion) {
     bench_cache(block_cache, c, 200);
     let block_cache = Arc::new(FoyerCache::lfu(256, Duration::from_millis(10)));
     bench_cache(block_cache, c, 200);
+    let block_cache = Arc::new(
+        tokio::runtime::Builder::new_current_thread()
+            .build()
+            .unwrap()
+            .block_on(FoyerHybridCache::lru(256, Duration::from_millis(10))),
+    );
+    bench_cache(block_cache, c, 200);
+    let block_cache = Arc::new(
+        tokio::runtime::Builder::new_current_thread()
+            .build()
+            .unwrap()
+            .block_on(FoyerHybridCache::lfu(256, Duration::from_millis(10))),
+    );
+    bench_cache(block_cache, c, 200);
 }
 
 criterion_group!(benches, bench_block_cache);
diff --git a/src/storage/benches/bench_compactor.rs b/src/storage/benches/bench_compactor.rs
index 02a0de8c7e238..1ccb3a4eccaa4 100644
--- a/src/storage/benches/bench_compactor.rs
+++ b/src/storage/benches/bench_compactor.rs
@@ -17,9 +17,9 @@ use std::sync::Arc;
 
 use criterion::async_executor::FuturesExecutor;
 use criterion::{criterion_group, criterion_main, Criterion};
-use foyer::memory::CacheContext;
+use foyer::{CacheContext, HybridCacheBuilder};
 use risingwave_common::catalog::{ColumnDesc, ColumnId, TableId};
-use risingwave_common::config::{EvictionConfig, MetricLevel, ObjectStoreConfig};
+use risingwave_common::config::{MetricLevel, ObjectStoreConfig};
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::row::OwnedRow;
 use risingwave_common::types::DataType;
@@ -44,35 +44,45 @@ use risingwave_storage::hummock::sstable::SstableIteratorReadOptions;
 use risingwave_storage::hummock::sstable_store::SstableStoreRef;
 use risingwave_storage::hummock::value::HummockValue;
 use risingwave_storage::hummock::{
-    CachePolicy, FileCache, SstableBuilder, SstableBuilderOptions, SstableIterator, SstableStore,
+    CachePolicy, SstableBuilder, SstableBuilderOptions, SstableIterator, SstableStore,
     SstableStoreConfig, SstableWriterOptions, Xor16FilterBuilder,
 };
 use risingwave_storage::monitor::{
     global_hummock_state_store_metrics, CompactorMetrics, StoreLocalStatistic,
 };
 
-pub fn mock_sstable_store() -> SstableStoreRef {
+pub async fn mock_sstable_store() -> SstableStoreRef {
     let store = InMemObjectStore::new().monitored(
         Arc::new(ObjectStoreMetrics::unused()),
-        ObjectStoreConfig::default(),
+        Arc::new(ObjectStoreConfig::default()),
     );
     let store = Arc::new(ObjectStoreImpl::InMem(store));
     let path = "test".to_string();
+    let meta_cache = HybridCacheBuilder::new()
+        .memory(64 << 20)
+        .with_shards(2)
+        .storage()
+        .build()
+        .await
+        .unwrap();
+    let block_cache = HybridCacheBuilder::new()
+        .memory(128 << 20)
+        .with_shards(2)
+        .storage()
+        .build()
+        .await
+        .unwrap();
     Arc::new(SstableStore::new(SstableStoreConfig {
         store,
         path,
-        block_cache_capacity: 64 << 20,
-        meta_cache_capacity: 128 << 20,
-        meta_cache_shard_num: 2,
-        block_cache_shard_num: 2,
-        block_cache_eviction: EvictionConfig::for_test(),
-        meta_cache_eviction: EvictionConfig::for_test(),
+
         prefetch_buffer_capacity: 64 << 20,
         max_prefetch_block_number: 16,
-        data_file_cache: FileCache::none(),
-        meta_file_cache: FileCache::none(),
         recent_filter: None,
         state_store_metrics: Arc::new(global_hummock_state_store_metrics(MetricLevel::Disabled)),
+
+        meta_cache,
+        block_cache,
     }))
 }
 
@@ -213,11 +223,11 @@ async fn scan_all_table(info: &SstableInfo, sstable_store: SstableStoreRef) {
 
 fn bench_table_build(c: &mut Criterion) {
     c.bench_function("bench_table_build", |b| {
-        let sstable_store = mock_sstable_store();
         let runtime = tokio::runtime::Builder::new_current_thread()
             .enable_time()
             .build()
             .unwrap();
+        let sstable_store = runtime.block_on(mock_sstable_store());
         b.to_async(&runtime).iter(|| async {
             build_table(sstable_store.clone(), 0, 0..(MAX_KEY_COUNT as u64), 1).await;
         });
@@ -225,11 +235,11 @@ fn bench_table_build(c: &mut Criterion) {
 }
 
 fn bench_table_scan(c: &mut Criterion) {
-    let sstable_store = mock_sstable_store();
     let runtime = tokio::runtime::Builder::new_current_thread()
         .enable_time()
         .build()
         .unwrap();
+    let sstable_store = runtime.block_on(mock_sstable_store());
     let info = runtime.block_on(async {
         build_table(sstable_store.clone(), 0, 0..(MAX_KEY_COUNT as u64), 1).await
     });
@@ -289,7 +299,7 @@ fn bench_merge_iterator_compactor(c: &mut Criterion) {
         .enable_time()
         .build()
         .unwrap();
-    let sstable_store = mock_sstable_store();
+    let sstable_store = runtime.block_on(mock_sstable_store());
     let test_key_size = 256 * 1024;
     let info1 = runtime
         .block_on(async { build_table(sstable_store.clone(), 1, 0..test_key_size / 2, 1).await });
@@ -365,7 +375,7 @@ fn bench_drop_column_compaction_impl(c: &mut Criterion, column_num: usize) {
         .enable_time()
         .build()
         .unwrap();
-    let sstable_store = mock_sstable_store();
+    let sstable_store = runtime.block_on(mock_sstable_store());
     let test_key_size = 256 * 1024;
     let info1 = runtime.block_on(async {
         build_table_2(
diff --git a/src/storage/benches/bench_fs_operation.rs b/src/storage/benches/bench_fs_operation.rs
index 0983883fdaa44..8d0b20987686c 100644
--- a/src/storage/benches/bench_fs_operation.rs
+++ b/src/storage/benches/bench_fs_operation.rs
@@ -217,6 +217,7 @@ fn gen_std_files(path: &Path) -> impl IntoIterator<Item = std::fs::File> {
             .read(true)
             .write(true)
             .create(true)
+            .truncate(true)
             .open(file_path)
             .unwrap();
         ret
diff --git a/src/storage/benches/bench_imm_compact.rs b/src/storage/benches/bench_imm_compact.rs
index dbc5eb8c2b591..5b52ef7fcee1b 100644
--- a/src/storage/benches/bench_imm_compact.rs
+++ b/src/storage/benches/bench_imm_compact.rs
@@ -18,8 +18,9 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
 use risingwave_common::catalog::TableId;
 use risingwave_hummock_sdk::key::TableKey;
 use risingwave_storage::hummock::compactor::merge_imms_in_memory;
-use risingwave_storage::hummock::shared_buffer::shared_buffer_batch::SharedBufferBatch;
-use risingwave_storage::hummock::value::HummockValue;
+use risingwave_storage::hummock::shared_buffer::shared_buffer_batch::{
+    SharedBufferBatch, SharedBufferValue,
+};
 
 fn gen_interleave_shared_buffer_batch(
     batch_size: usize,
@@ -34,7 +35,7 @@ fn gen_interleave_shared_buffer_batch(
                 TableKey(Bytes::copy_from_slice(
                     format!("test_key_{:08}", j * batch_count + i).as_bytes(),
                 )),
-                HummockValue::put(Bytes::copy_from_slice("value".as_bytes())),
+                SharedBufferValue::Insert(Bytes::copy_from_slice("value".as_bytes())),
             ));
         }
         let batch = SharedBufferBatch::for_test(batch_data, epoch, Default::default());
@@ -51,7 +52,7 @@ fn criterion_benchmark(c: &mut Criterion) {
         &batches,
         |b, batches| {
             b.to_async(FuturesExecutor).iter(|| async {
-                let imm = merge_imms_in_memory(TableId::default(), 0, batches.clone(), None).await;
+                let imm = merge_imms_in_memory(TableId::default(), batches.clone(), None).await;
                 assert_eq!(imm.key_count(), 10000 * 100);
                 assert_eq!(imm.value_count(), 10000 * 100);
             })
@@ -71,7 +72,7 @@ fn criterion_benchmark(c: &mut Criterion) {
         &later_batches,
         |b, batches| {
             b.to_async(FuturesExecutor).iter(|| async {
-                let imm = merge_imms_in_memory(TableId::default(), 0, batches.clone(), None).await;
+                let imm = merge_imms_in_memory(TableId::default(), batches.clone(), None).await;
                 assert_eq!(imm.key_count(), 2000 * 100);
                 assert_eq!(imm.value_count(), 2000 * 100 * 5);
             })
diff --git a/src/storage/benches/bench_merge_iter.rs b/src/storage/benches/bench_merge_iter.rs
index c7b42f1894265..91fc5bc2f3058 100644
--- a/src/storage/benches/bench_merge_iter.rs
+++ b/src/storage/benches/bench_merge_iter.rs
@@ -23,9 +23,8 @@ use risingwave_storage::hummock::iterator::{
     Forward, HummockIterator, HummockIteratorUnion, MergeIterator, SkipWatermarkIterator,
 };
 use risingwave_storage::hummock::shared_buffer::shared_buffer_batch::{
-    SharedBufferBatch, SharedBufferBatchIterator,
+    SharedBufferBatch, SharedBufferBatchIterator, SharedBufferValue,
 };
-use risingwave_storage::hummock::value::HummockValue;
 
 fn gen_interleave_shared_buffer_batch_iter(
     batch_size: usize,
@@ -39,7 +38,7 @@ fn gen_interleave_shared_buffer_batch_iter(
                 TableKey(Bytes::copy_from_slice(
                     format!("test_key_{:08}", j * batch_count + i).as_bytes(),
                 )),
-                HummockValue::put(Bytes::copy_from_slice("value".as_bytes())),
+                SharedBufferValue::Insert(Bytes::copy_from_slice("value".as_bytes())),
             ));
         }
         let batch = SharedBufferBatch::for_test(batch_data, 2333, Default::default());
@@ -69,7 +68,7 @@ fn gen_interleave_shared_buffer_batch_enum_iter(
                 TableKey(Bytes::copy_from_slice(
                     format!("test_key_{:08}", j * batch_count + i).as_bytes(),
                 )),
-                HummockValue::put(Bytes::copy_from_slice("value".as_bytes())),
+                SharedBufferValue::Insert(Bytes::copy_from_slice("value".as_bytes())),
             ));
         }
         let batch = SharedBufferBatch::for_test(batch_data, 2333, Default::default());
diff --git a/src/storage/benches/bench_multi_builder.rs b/src/storage/benches/bench_multi_builder.rs
index ef07fd107986f..b67be3467c48a 100644
--- a/src/storage/benches/bench_multi_builder.rs
+++ b/src/storage/benches/bench_multi_builder.rs
@@ -20,18 +20,23 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use criterion::{criterion_group, criterion_main, Criterion};
-use futures::future::try_join_all;
-use itertools::Itertools;
+use foyer::HybridCacheBuilder;
+use rand::random;
 use risingwave_common::catalog::TableId;
-use risingwave_common::config::{EvictionConfig, MetricLevel, ObjectStoreConfig};
+use risingwave_common::config::{MetricLevel, ObjectStoreConfig};
 use risingwave_hummock_sdk::key::{FullKey, UserKey};
-use risingwave_object_store::object::{ObjectStore, ObjectStoreImpl, S3ObjectStore};
+use risingwave_object_store::object::{
+    InMemObjectStore, ObjectStore, ObjectStoreImpl, S3ObjectStore,
+};
+use risingwave_pb::hummock::SstableInfo;
+use risingwave_storage::hummock::iterator::{ConcatIterator, ConcatIteratorInner, HummockIterator};
 use risingwave_storage::hummock::multi_builder::{CapacitySplitTableBuilder, TableBuilderFactory};
 use risingwave_storage::hummock::value::HummockValue;
 use risingwave_storage::hummock::{
-    BatchSstableWriterFactory, CachePolicy, FileCache, HummockResult, MemoryLimiter,
-    SstableBuilder, SstableBuilderOptions, SstableStore, SstableStoreConfig, SstableWriterFactory,
-    SstableWriterOptions, StreamingSstableWriterFactory, Xor16FilterBuilder,
+    BackwardSstableIterator, BatchSstableWriterFactory, CachePolicy, HummockResult, MemoryLimiter,
+    SstableBuilder, SstableBuilderOptions, SstableIteratorReadOptions, SstableStore,
+    SstableStoreConfig, SstableWriterFactory, SstableWriterOptions, StreamingSstableWriterFactory,
+    Xor16FilterBuilder,
 };
 use risingwave_storage::monitor::{global_hummock_state_store_metrics, ObjectStoreMetrics};
 
@@ -100,7 +105,7 @@ fn test_user_key_of(idx: u64) -> UserKey<Vec<u8>> {
 
 async fn build_tables<F: SstableWriterFactory>(
     mut builder: CapacitySplitTableBuilder<LocalTableBuilderFactory<F>>,
-) {
+) -> Vec<SstableInfo> {
     for i in RANGE {
         builder
             .add_full_key_for_test(
@@ -111,19 +116,48 @@ async fn build_tables<F: SstableWriterFactory>(
             .await
             .unwrap();
     }
-    let split_table_outputs = builder.finish().await.unwrap();
-    let join_handles = split_table_outputs
+
+    builder
+        .finish()
+        .await
+        .unwrap()
         .into_iter()
-        .map(|o| o.upload_join_handle)
-        .collect_vec();
-    try_join_all(join_handles).await.unwrap();
+        .map(|info| info.sst_info)
+        .collect()
+}
+
+async fn generate_sstable_store(object_store: Arc<ObjectStoreImpl>) -> Arc<SstableStore> {
+    let meta_cache = HybridCacheBuilder::new()
+        .memory(64 << 20)
+        .with_shards(2)
+        .storage()
+        .build()
+        .await
+        .unwrap();
+    let block_cache = HybridCacheBuilder::new()
+        .memory(128 << 20)
+        .with_shards(2)
+        .storage()
+        .build()
+        .await
+        .unwrap();
+    Arc::new(SstableStore::new(SstableStoreConfig {
+        store: object_store,
+        path: "test".to_string(),
+        prefetch_buffer_capacity: 64 << 20,
+        max_prefetch_block_number: 16,
+        recent_filter: None,
+        state_store_metrics: Arc::new(global_hummock_state_store_metrics(MetricLevel::Disabled)),
+        meta_cache,
+        block_cache,
+    }))
 }
 
 fn bench_builder(
     c: &mut Criterion,
-    bucket: &str,
     capacity_mb: usize,
     enable_streaming_upload: bool,
+    local: bool,
 ) {
     let runtime = tokio::runtime::Builder::new_current_thread()
         .enable_all()
@@ -131,32 +165,31 @@ fn bench_builder(
         .unwrap();
 
     let metrics = Arc::new(ObjectStoreMetrics::unused());
+    let default_config = Arc::new(ObjectStoreConfig::default());
+
     let object_store = runtime.block_on(async {
-        S3ObjectStore::new_with_config(
-            bucket.to_string(),
-            metrics.clone(),
-            ObjectStoreConfig::default(),
-        )
-        .await
-        .monitored(metrics, ObjectStoreConfig::default())
+        if local {
+            S3ObjectStore::new_minio_engine(
+                "minio://hummockadmin:hummockadmin@127.0.0.1:9301/hummock001",
+                metrics.clone(),
+                default_config.clone(),
+            )
+            .await
+            .monitored(metrics, default_config)
+        } else {
+            S3ObjectStore::new_with_config(
+                env::var("S3_BUCKET").unwrap(),
+                metrics.clone(),
+                default_config.clone(),
+            )
+            .await
+            .monitored(metrics, default_config)
+        }
     });
+
     let object_store = Arc::new(ObjectStoreImpl::S3(object_store));
-    let sstable_store = Arc::new(SstableStore::new(SstableStoreConfig {
-        store: object_store,
-        path: "test".to_string(),
-        block_cache_capacity: 64 << 20,
-        meta_cache_capacity: 128 << 20,
-        meta_cache_shard_num: 2,
-        block_cache_shard_num: 2,
-        block_cache_eviction: EvictionConfig::for_test(),
-        meta_cache_eviction: EvictionConfig::for_test(),
-        prefetch_buffer_capacity: 64 << 20,
-        max_prefetch_block_number: 16,
-        data_file_cache: FileCache::none(),
-        meta_file_cache: FileCache::none(),
-        recent_filter: None,
-        state_store_metrics: Arc::new(global_hummock_state_store_metrics(MetricLevel::Disabled)),
-    }));
+
+    let sstable_store = runtime.block_on(async { generate_sstable_store(object_store).await });
 
     let mut group = c.benchmark_group("bench_multi_builder");
     group
@@ -193,12 +226,133 @@ fn bench_builder(
 // SST size: 4, 32, 64, 128, 256MiB
 fn bench_multi_builder(c: &mut Criterion) {
     let sst_capacities = vec![4, 32, 64, 128, 256];
-    let bucket = env::var("S3_BUCKET").unwrap();
+    let is_local_test = env::var("LOCAL_TEST").is_ok();
+
     for capacity in sst_capacities {
-        bench_builder(c, &bucket, capacity, false);
-        bench_builder(c, &bucket, capacity, true);
+        bench_builder(c, capacity, false, is_local_test);
+        bench_builder(c, capacity, true, is_local_test);
     }
 }
 
-criterion_group!(benches, bench_multi_builder);
+fn bench_table_scan(c: &mut Criterion) {
+    let capacity_mb: usize = 32;
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+
+    let store = InMemObjectStore::new().monitored(
+        Arc::new(ObjectStoreMetrics::unused()),
+        Arc::new(ObjectStoreConfig::default()),
+    );
+    let object_store = Arc::new(ObjectStoreImpl::InMem(store));
+    let sstable_store = runtime.block_on(async { generate_sstable_store(object_store).await });
+
+    let ssts = runtime.block_on(async {
+        build_tables(CapacitySplitTableBuilder::for_test(
+            LocalTableBuilderFactory::new(
+                1,
+                BatchSstableWriterFactory::new(sstable_store.clone()),
+                get_builder_options(capacity_mb),
+            ),
+        ))
+        .await
+    });
+    println!("sst count: {}", ssts.len());
+    let mut group = c.benchmark_group("bench_multi_builder");
+    group
+        .sample_size(SAMPLE_COUNT)
+        .measurement_time(ESTIMATED_MEASUREMENT_TIME);
+    let read_options = Arc::new(SstableIteratorReadOptions::default());
+    group.bench_function("bench_table_scan", |b| {
+        let sstable_ssts = ssts.clone();
+        b.to_async(&runtime).iter(|| {
+            let sstable_ssts = sstable_ssts.clone();
+            let sstable_store = sstable_store.clone();
+            let read_options = read_options.clone();
+            async move {
+                let mut iter = ConcatIterator::new(
+                    sstable_ssts.clone(),
+                    sstable_store.clone(),
+                    read_options.clone(),
+                );
+                iter.rewind().await.unwrap();
+                let mut count = 0;
+                while iter.is_valid() {
+                    count += 1;
+                    iter.next().await.unwrap();
+                }
+                assert_eq!(count, RANGE.end - RANGE.start);
+            }
+        });
+    });
+    group.bench_function("bench_table_reverse_scan", |b| {
+        let mut sstable_ssts = ssts.clone();
+        sstable_ssts.reverse();
+        b.to_async(&runtime).iter(|| {
+            let sstable_ssts = sstable_ssts.clone();
+            let sstable_store = sstable_store.clone();
+            let read_options = read_options.clone();
+            async move {
+                let mut iter = ConcatIteratorInner::<BackwardSstableIterator>::new(
+                    sstable_ssts.clone(),
+                    sstable_store.clone(),
+                    read_options.clone(),
+                );
+                iter.rewind().await.unwrap();
+                let mut count = 0;
+                while iter.is_valid() {
+                    count += 1;
+                    iter.next().await.unwrap();
+                }
+                assert_eq!(count, RANGE.end - RANGE.start);
+            }
+        });
+    });
+    group.bench_function("bench_point_scan", |b| {
+        let sstable_ssts = ssts.clone();
+        b.to_async(&runtime).iter(|| {
+            let sstable_ssts = sstable_ssts.clone();
+            let sstable_store = sstable_store.clone();
+            let read_options = read_options.clone();
+            let idx = random::<u64>() % (RANGE.end - RANGE.start);
+            let key = FullKey::from_user_key(test_user_key_of(idx), 1);
+            async move {
+                let mut iter = ConcatIterator::new(
+                    sstable_ssts.clone(),
+                    sstable_store.clone(),
+                    read_options.clone(),
+                );
+                iter.seek(key.to_ref()).await.unwrap();
+                if iter.is_valid() {
+                    iter.next().await.unwrap();
+                }
+            }
+        });
+    });
+    group.bench_function("bench_point_reverse_scan", |b| {
+        let mut sstable_ssts = ssts.clone();
+        sstable_ssts.reverse();
+        b.to_async(&runtime).iter(|| {
+            let sstable_ssts = sstable_ssts.clone();
+            let sstable_store = sstable_store.clone();
+            let read_options = read_options.clone();
+            let idx = random::<u64>() % (RANGE.end - RANGE.start);
+            let key = FullKey::from_user_key(test_user_key_of(idx), 1);
+            async move {
+                let mut iter = ConcatIteratorInner::<BackwardSstableIterator>::new(
+                    sstable_ssts.clone(),
+                    sstable_store.clone(),
+                    read_options.clone(),
+                );
+                iter.seek(key.to_ref()).await.unwrap();
+                if iter.is_valid() {
+                    iter.next().await.unwrap();
+                }
+            }
+        });
+    });
+}
+
+criterion_group!(benches, bench_multi_builder, bench_table_scan);
 criterion_main!(benches);
diff --git a/src/storage/benches/bench_table_watermarks.rs b/src/storage/benches/bench_table_watermarks.rs
index 11ec3c4bdcb54..f1fdf66c2f895 100644
--- a/src/storage/benches/bench_table_watermarks.rs
+++ b/src/storage/benches/bench_table_watermarks.rs
@@ -15,7 +15,7 @@
 #![feature(lazy_cell)]
 
 use std::collections::hash_map::Entry;
-use std::collections::{HashMap, VecDeque};
+use std::collections::{HashMap, HashSet, VecDeque};
 use std::sync::{Arc, LazyLock};
 
 use bytes::Bytes;
@@ -25,11 +25,13 @@ use risingwave_common::buffer::{Bitmap, BitmapBuilder};
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::util::epoch::test_epoch;
+use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_sdk::table_watermark::{
     TableWatermarks, TableWatermarksIndex, VnodeWatermark, WatermarkDirection,
 };
-use risingwave_hummock_sdk::version::HummockVersion;
+use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionStateTableInfo};
 use risingwave_hummock_sdk::HummockEpoch;
+use risingwave_pb::hummock::StateTableInfoDelta;
 use risingwave_storage::hummock::local_version::pinned_version::PinnedVersion;
 use spin::Mutex;
 use tokio::sync::mpsc::unbounded_channel;
@@ -115,17 +117,31 @@ fn gen_version(
         new_epoch_idx,
         vnode_part_count,
     ));
-    // let table_watermarks =
-    //     gen_committed_table_watermarks(old_epoch_idx, new_epoch_idx, vnode_part_count);
-    HummockVersion {
-        id: new_epoch_idx as _,
-        max_committed_epoch: test_epoch(new_epoch_idx as _),
-        safe_epoch: test_epoch(old_epoch_idx as _),
-        table_watermarks: (0..table_count)
-            .map(|table_id| (TableId::new(table_id as _), table_watermarks.clone()))
+    let mut version = HummockVersion::default();
+    let committed_epoch = test_epoch(new_epoch_idx as _);
+    version.id = new_epoch_idx as _;
+    version.max_committed_epoch = committed_epoch;
+    version.table_watermarks = (0..table_count)
+        .map(|table_id| (TableId::new(table_id as _), table_watermarks.clone()))
+        .collect();
+    let mut state_table_info = HummockVersionStateTableInfo::empty();
+    state_table_info.apply_delta(
+        &(0..table_count)
+            .map(|table_id| {
+                (
+                    TableId::new(table_id as _),
+                    StateTableInfoDelta {
+                        committed_epoch,
+                        safe_epoch: test_epoch(old_epoch_idx as _),
+                        compaction_group_id: StaticCompactionGroupId::StateDefault as _,
+                    },
+                )
+            })
             .collect(),
-        ..Default::default()
-    }
+        &HashSet::new(),
+    );
+    version.state_table_info = state_table_info;
+    version
 }
 
 fn bench_table_watermarks(c: &mut Criterion) {
diff --git a/src/storage/compactor/src/lib.rs b/src/storage/compactor/src/lib.rs
index 3ceb9f8954e3b..d9fbe5189f43f 100644
--- a/src/storage/compactor/src/lib.rs
+++ b/src/storage/compactor/src/lib.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![feature(lint_reasons)]
+
 mod compactor_observer;
 mod rpc;
 pub mod server;
diff --git a/src/storage/compactor/src/server.rs b/src/storage/compactor/src/server.rs
index 95ee16ca3862c..e11491fe0360b 100644
--- a/src/storage/compactor/src/server.rs
+++ b/src/storage/compactor/src/server.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::net::SocketAddr;
-use std::sync::atomic::AtomicU32;
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -122,17 +121,22 @@ pub async fn prepare_start_parameters(
             .expect("object store must be hummock for compactor server"),
         object_metrics,
         "Hummock",
-        config.storage.object_store.clone(),
+        Arc::new(config.storage.object_store.clone()),
     )
     .await;
 
     let object_store = Arc::new(object_store);
-    let sstable_store = Arc::new(SstableStore::for_compactor(
-        object_store,
-        storage_opts.data_directory.to_string(),
-        1 << 20, // set 1MB memory to avoid panic.
-        meta_cache_capacity_bytes,
-    ));
+    let sstable_store = Arc::new(
+        SstableStore::for_compactor(
+            object_store,
+            storage_opts.data_directory.to_string(),
+            0,
+            meta_cache_capacity_bytes,
+        )
+        .await
+        // FIXME(MrCroxx): Handle this error.
+        .unwrap(),
+    );
 
     let memory_limiter = Arc::new(MemoryLimiter::new(compactor_memory_limit_bytes));
     let storage_memory_config = extract_storage_memory_config(&config);
@@ -242,10 +246,6 @@ pub async fn compactor_serve(
     let compaction_executor = Arc::new(CompactionExecutor::new(
         opts.compaction_worker_threads_number,
     ));
-    let max_task_parallelism = Arc::new(AtomicU32::new(
-        (compaction_executor.worker_num() as f32 * storage_opts.compactor_max_task_multiplier)
-            .ceil() as u32,
-    ));
 
     let compactor_context = CompactorContext {
         storage_opts,
@@ -257,8 +257,6 @@ pub async fn compactor_serve(
 
         task_progress_manager: Default::default(),
         await_tree_reg: await_tree_reg.clone(),
-        running_task_parallelism: Arc::new(AtomicU32::new(0)),
-        max_task_parallelism,
     };
     let mut sub_tasks = vec![
         MetaClient::start_heartbeat_loop(
@@ -378,10 +376,6 @@ pub async fn shared_compactor_serve(
     let compaction_executor = Arc::new(CompactionExecutor::new(
         opts.compaction_worker_threads_number,
     ));
-    let max_task_parallelism = Arc::new(AtomicU32::new(
-        (compaction_executor.worker_num() as f32 * storage_opts.compactor_max_task_multiplier)
-            .ceil() as u32,
-    ));
     let compactor_context = CompactorContext {
         storage_opts,
         sstable_store,
@@ -391,8 +385,6 @@ pub async fn shared_compactor_serve(
         memory_limiter,
         task_progress_manager: Default::default(),
         await_tree_reg,
-        running_task_parallelism: Arc::new(AtomicU32::new(0)),
-        max_task_parallelism,
     };
     let join_handle = tokio::spawn(async move {
         tonic::transport::Server::builder()
diff --git a/src/storage/compactor/src/telemetry.rs b/src/storage/compactor/src/telemetry.rs
index 2bf7aaaa3f508..b01fdd1e258f3 100644
--- a/src/storage/compactor/src/telemetry.rs
+++ b/src/storage/compactor/src/telemetry.rs
@@ -33,6 +33,7 @@ impl CompactorTelemetryCreator {
 
 #[async_trait::async_trait]
 impl TelemetryReportCreator for CompactorTelemetryCreator {
+    #[allow(refining_impl_trait)]
     async fn create_report(
         &self,
         tracking_id: String,
diff --git a/src/storage/hummock_sdk/Cargo.toml b/src/storage/hummock_sdk/Cargo.toml
index 76f7913127f58..fa080b90b7469 100644
--- a/src/storage/hummock_sdk/Cargo.toml
+++ b/src/storage/hummock_sdk/Cargo.toml
@@ -23,6 +23,8 @@ prost = { workspace = true }
 risingwave_common = { workspace = true }
 risingwave_common_estimate_size = { workspace = true }
 risingwave_pb = { workspace = true }
+serde = { version = "1", features = ["derive"] }
+serde_bytes = "0.11"
 tracing = "0.1"
 
 [target.'cfg(not(madsim))'.dependencies]
diff --git a/src/storage/hummock_sdk/src/change_log.rs b/src/storage/hummock_sdk/src/change_log.rs
index cec5ef58034c6..49224c92091f5 100644
--- a/src/storage/hummock_sdk/src/change_log.rs
+++ b/src/storage/hummock_sdk/src/change_log.rs
@@ -12,14 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use risingwave_pb::hummock::{PbEpochNewChangeLog, PbTableChangeLog, SstableInfo};
+use std::collections::HashMap;
 
-#[derive(Debug, Clone, PartialEq)]
-pub struct EpochNewChangeLog {
-    pub new_value: Vec<SstableInfo>,
-    pub old_value: Vec<SstableInfo>,
-    pub epochs: Vec<u64>,
-}
+use risingwave_common::catalog::TableId;
+use risingwave_pb::hummock::hummock_version_delta::ChangeLogDelta;
+use risingwave_pb::hummock::{EpochNewChangeLog, SstableInfo, TableChangeLog as PbTableChangeLog};
+use tracing::warn;
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct TableChangeLog(pub Vec<EpochNewChangeLog>);
@@ -34,37 +32,82 @@ impl TableChangeLog {
         });
         &self.0[start..end]
     }
+
+    pub fn get_epochs(&self, min_epoch: u64, max_count: usize) -> Vec<u64> {
+        self.filter_epoch((min_epoch, u64::MAX))
+            .iter()
+            .flat_map(|epoch_change_log| epoch_change_log.epochs.iter().cloned())
+            .filter(|a| a >= &min_epoch)
+            .clone()
+            .take(max_count)
+            .collect()
+    }
+
+    pub fn truncate(&mut self, truncate_epoch: u64) {
+        // TODO: may optimize by using VecDeque to maintain the log
+        self.0
+            .retain(|change_log| *change_log.epochs.last().expect("non-empty") > truncate_epoch);
+        if let Some(first_log) = self.0.first_mut() {
+            first_log.epochs.retain(|epoch| *epoch > truncate_epoch);
+        }
+    }
 }
 
 impl TableChangeLog {
     pub fn to_protobuf(&self) -> PbTableChangeLog {
         PbTableChangeLog {
-            change_logs: self
-                .0
-                .iter()
-                .map(|epoch_new_log| PbEpochNewChangeLog {
-                    epochs: epoch_new_log.epochs.clone(),
-                    new_value: epoch_new_log.new_value.clone(),
-                    old_value: epoch_new_log.old_value.clone(),
-                })
-                .collect(),
+            change_logs: self.0.clone(),
         }
     }
 
     pub fn from_protobuf(val: &PbTableChangeLog) -> Self {
-        Self(
-            val.change_logs
-                .iter()
-                .map(|epoch_new_log| EpochNewChangeLog {
-                    epochs: epoch_new_log.epochs.clone(),
-                    new_value: epoch_new_log.new_value.clone(),
-                    old_value: epoch_new_log.old_value.clone(),
-                })
-                .collect(),
-        )
+        Self(val.change_logs.clone())
     }
 }
 
+pub fn build_table_change_log_delta<'a>(
+    old_value_ssts: impl Iterator<Item = SstableInfo>,
+    new_value_ssts: impl Iterator<Item = &'a SstableInfo>,
+    epochs: &Vec<u64>,
+    log_store_table_ids: impl Iterator<Item = (u32, u64)>,
+) -> HashMap<TableId, ChangeLogDelta> {
+    let mut table_change_log: HashMap<_, _> = log_store_table_ids
+        .map(|(table_id, truncate_epoch)| {
+            (
+                TableId::new(table_id),
+                ChangeLogDelta {
+                    truncate_epoch,
+                    new_log: Some(EpochNewChangeLog {
+                        new_value: vec![],
+                        old_value: vec![],
+                        epochs: epochs.clone(),
+                    }),
+                },
+            )
+        })
+        .collect();
+    for sst in old_value_ssts {
+        for table_id in &sst.table_ids {
+            match table_change_log.get_mut(&TableId::new(*table_id)) {
+                Some(log) => {
+                    log.new_log.as_mut().unwrap().old_value.push(sst.clone());
+                }
+                None => {
+                    warn!(table_id, ?sst, "old value sst contains non-log-store table");
+                }
+            }
+        }
+    }
+    for sst in new_value_ssts {
+        for table_id in &sst.table_ids {
+            if let Some(log) = table_change_log.get_mut(&TableId::new(*table_id)) {
+                log.new_log.as_mut().unwrap().new_value.push(sst.clone());
+            }
+        }
+    }
+    table_change_log
+}
+
 #[cfg(test)]
 mod tests {
     use itertools::Itertools;
@@ -110,4 +153,69 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_truncate() {
+        let mut table_change_log = TableChangeLog(vec![
+            EpochNewChangeLog {
+                new_value: vec![],
+                old_value: vec![],
+                epochs: vec![1],
+            },
+            EpochNewChangeLog {
+                new_value: vec![],
+                old_value: vec![],
+                epochs: vec![2],
+            },
+            EpochNewChangeLog {
+                new_value: vec![],
+                old_value: vec![],
+                epochs: vec![3, 4],
+            },
+            EpochNewChangeLog {
+                new_value: vec![],
+                old_value: vec![],
+                epochs: vec![5],
+            },
+        ]);
+
+        table_change_log.truncate(1);
+        assert_eq!(
+            table_change_log,
+            TableChangeLog(vec![
+                EpochNewChangeLog {
+                    new_value: vec![],
+                    old_value: vec![],
+                    epochs: vec![2],
+                },
+                EpochNewChangeLog {
+                    new_value: vec![],
+                    old_value: vec![],
+                    epochs: vec![3, 4],
+                },
+                EpochNewChangeLog {
+                    new_value: vec![],
+                    old_value: vec![],
+                    epochs: vec![5],
+                },
+            ])
+        );
+
+        table_change_log.truncate(3);
+        assert_eq!(
+            table_change_log,
+            TableChangeLog(vec![
+                EpochNewChangeLog {
+                    new_value: vec![],
+                    old_value: vec![],
+                    epochs: vec![4],
+                },
+                EpochNewChangeLog {
+                    new_value: vec![],
+                    old_value: vec![],
+                    epochs: vec![5],
+                },
+            ])
+        )
+    }
 }
diff --git a/src/storage/hummock_sdk/src/compact.rs b/src/storage/hummock_sdk/src/compact.rs
index f251e89b8091a..d6824ab646c43 100644
--- a/src/storage/hummock_sdk/src/compact.rs
+++ b/src/storage/hummock_sdk/src/compact.rs
@@ -185,7 +185,6 @@ pub fn estimate_memory_for_compact_task(
     block_size: u64,
     recv_buffer_size: u64,
     sst_capacity: u64,
-    support_streaming_upload: bool,
 ) -> u64 {
     let mut result = 0;
     // When building the SstableStreamIterator, sstable_syncable will fetch the SstableMeta and seek
@@ -223,12 +222,10 @@ pub fn estimate_memory_for_compact_task(
     // output
     // builder will maintain SstableInfo + block_builder(block) + writer (block to vec)
     let estimated_meta_size = sst_capacity * task_max_sst_meta_ratio / 100;
-    if support_streaming_upload {
-        result += estimated_meta_size + 2 * block_size
-    } else {
-        result += estimated_meta_size + sst_capacity; // Use sst_capacity to avoid BatchUploader
-                                                      // memory bursts.
-    }
+
+    // FIXME: sst_capacity is the upper bound of the memory usage of the streaming sstable uploader
+    // A more reasonable memory limit method needs to be adopted, this is just a temporary fix.
+    result += estimated_meta_size + sst_capacity;
 
     result
 }
diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs
index ded56ede6fe65..102e364b8d00b 100644
--- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs
+++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs
@@ -31,6 +31,7 @@ use risingwave_pb::hummock::{
 use tracing::warn;
 
 use super::StateTableId;
+use crate::change_log::TableChangeLog;
 use crate::compaction_group::StaticCompactionGroupId;
 use crate::key_range::KeyRangeCommon;
 use crate::prost_key_range::KeyRangeExt;
@@ -126,7 +127,7 @@ pub struct SstDeltaInfo {
     pub delete_sst_object_ids: Vec<HummockSstableObjectId>,
 }
 
-pub type BranchedSstInfo = HashMap<CompactionGroupId, /* SST Id */ HummockSstableId>;
+pub type BranchedSstInfo = HashMap<CompactionGroupId, Vec<HummockSstableId>>;
 
 impl HummockVersion {
     pub fn get_compaction_group_levels(&self, compaction_group_id: CompactionGroupId) -> &Levels {
@@ -165,6 +166,15 @@ impl HummockVersion {
                     .iter()
                     .map(|table_info| table_info.get_object_id())
             })
+            .chain(self.table_change_log.values().flat_map(|change_log| {
+                change_log.0.iter().flat_map(|epoch_change_log| {
+                    epoch_change_log
+                        .old_value
+                        .iter()
+                        .map(|sst| sst.object_id)
+                        .chain(epoch_change_log.new_value.iter().map(|sst| sst.object_id))
+                })
+            }))
             .collect()
     }
 
@@ -236,25 +246,21 @@ impl HummockVersion {
                 if !existing_table_ids.contains(&u32_table_id) {
                     None
                 } else {
-                    extract_single_table_watermark(table_watermarks, self.safe_epoch)
-                        .map(|table_watermarks| (table_id.table_id, table_watermarks))
+                    extract_single_table_watermark(
+                        table_watermarks,
+                        self.state_table_info
+                            .info()
+                            .get(table_id)
+                            .expect("table should exist")
+                            .safe_epoch,
+                    )
+                    .map(|table_watermarks| (table_id.table_id, table_watermarks))
                 }
             })
             .collect()
     }
 }
 
-pub type SstSplitInfo = (
-    // Object id.
-    HummockSstableObjectId,
-    // SST id.
-    HummockSstableId,
-    // Old SST id in parent group.
-    HummockSstableId,
-    // New SST id in parent group.
-    HummockSstableId,
-);
-
 impl HummockVersion {
     pub fn count_new_ssts_in_group_split(
         &self,
@@ -290,13 +296,12 @@ impl HummockVersion {
         member_table_ids: HashSet<StateTableId>,
         new_sst_start_id: u64,
         allow_trivial_split: bool,
-    ) -> Vec<SstSplitInfo> {
+    ) {
         let mut new_sst_id = new_sst_start_id;
-        let mut split_id_vers = vec![];
         if parent_group_id == StaticCompactionGroupId::NewCompactionGroup as CompactionGroupId
             || !self.levels.contains_key(&parent_group_id)
         {
-            return split_id_vers;
+            return;
         }
         let [parent_levels, cur_levels] = self
             .levels
@@ -328,7 +333,6 @@ impl HummockVersion {
                     &member_table_ids,
                     allow_trivial_split,
                     sub_level,
-                    &mut split_id_vers,
                     &mut new_sst_id,
                 );
                 sub_level
@@ -364,7 +368,6 @@ impl HummockVersion {
                 &member_table_ids,
                 allow_trivial_split,
                 level,
-                &mut split_id_vers,
                 &mut new_sst_id,
             );
             cur_levels.levels[idx].total_file_size += insert_table_infos
@@ -392,7 +395,6 @@ impl HummockVersion {
                     level.uncompressed_file_size -= sst_info.uncompressed_file_size;
                 });
         }
-        split_id_vers
     }
 
     pub fn build_sst_delta_infos(&self, version_delta: &HummockVersionDelta) -> Vec<SstDeltaInfo> {
@@ -476,11 +478,15 @@ impl HummockVersion {
         infos
     }
 
-    pub fn apply_version_delta(
-        &mut self,
-        version_delta: &HummockVersionDelta,
-    ) -> Vec<SstSplitInfo> {
-        let mut sst_split_info = vec![];
+    pub fn apply_version_delta(&mut self, version_delta: &HummockVersionDelta) {
+        assert_eq!(self.id, version_delta.prev_id);
+
+        let changed_table_info = self.state_table_info.apply_delta(
+            &version_delta.state_table_info_delta,
+            &version_delta.removed_table_ids,
+        );
+
+        // apply to `levels`, which is different compaction groups
         for (compaction_group_id, group_deltas) in &version_delta.group_deltas {
             let summary = summarize_group_deltas(group_deltas);
             if let Some(group_construct) = &summary.group_construct {
@@ -490,34 +496,60 @@ impl HummockVersion {
                 );
                 let parent_group_id = group_construct.parent_group_id;
                 new_levels.parent_group_id = parent_group_id;
+                #[expect(deprecated)]
+                // for backward-compatibility of previous hummock version delta
                 new_levels
                     .member_table_ids
                     .clone_from(&group_construct.table_ids);
                 self.levels.insert(*compaction_group_id, new_levels);
-                sst_split_info.extend(self.init_with_parent_group(
+                let member_table_ids =
+                    if group_construct.version >= CompatibilityVersion::NoMemberTableIds as _ {
+                        self.state_table_info
+                            .compaction_group_member_table_ids(*compaction_group_id)
+                            .iter()
+                            .map(|table_id| table_id.table_id)
+                            .collect()
+                    } else {
+                        #[expect(deprecated)]
+                        // for backward-compatibility of previous hummock version delta
+                        HashSet::from_iter(group_construct.table_ids.clone())
+                    };
+
+                self.init_with_parent_group(
                     parent_group_id,
                     *compaction_group_id,
-                    HashSet::from_iter(group_construct.table_ids.clone()),
+                    member_table_ids,
                     group_construct.get_new_sst_start_id(),
-                    group_construct.version() == CompatibilityVersion::VersionUnspecified,
-                ));
+                    group_construct.version < CompatibilityVersion::NoTrivialSplit as _,
+                );
             } else if let Some(group_change) = &summary.group_table_change {
-                sst_split_info.extend(self.init_with_parent_group(
+                // TODO: may deprecate this branch? This enum variant is not created anywhere
+                assert!(
+                    group_change.version <= CompatibilityVersion::NoTrivialSplit as _,
+                    "DeltaType::GroupTableChange is not used anymore after CompatibilityVersion::NoMemberTableIds is added"
+                );
+                #[expect(deprecated)]
+                // for backward-compatibility of previous hummock version delta
+                self.init_with_parent_group(
                     group_change.origin_group_id,
                     group_change.target_group_id,
                     HashSet::from_iter(group_change.table_ids.clone()),
                     group_change.new_sst_start_id,
                     group_change.version() == CompatibilityVersion::VersionUnspecified,
-                ));
+                );
 
                 let levels = self
                     .levels
                     .get_mut(&group_change.origin_group_id)
                     .expect("compaction group should exist");
+                #[expect(deprecated)]
+                // for backward-compatibility of previous hummock version delta
                 let mut moving_tables = levels
                     .member_table_ids
                     .extract_if(|t| group_change.table_ids.contains(t))
                     .collect_vec();
+                #[expect(deprecated)]
+                // for backward-compatibility of previous hummock version delta
                 self.levels
                     .get_mut(compaction_group_id)
                     .expect("compaction group should exist")
@@ -530,6 +562,7 @@ impl HummockVersion {
                 .get_mut(compaction_group_id)
                 .expect("compaction group should exist");
 
+            #[expect(deprecated)] // for backward-compatibility of previous hummock version delta
             for group_meta_delta in &summary.group_meta_changes {
                 levels
                     .member_table_ids
@@ -577,7 +610,11 @@ impl HummockVersion {
                 }
             } else {
                 // `max_committed_epoch` is not changed. The delta is caused by compaction.
-                levels.apply_compact_ssts(summary);
+                levels.apply_compact_ssts(
+                    summary,
+                    self.state_table_info
+                        .compaction_group_member_table_ids(*compaction_group_id),
+                );
             }
             if has_destroy {
                 self.levels.remove(compaction_group_id);
@@ -585,48 +622,106 @@ impl HummockVersion {
         }
         self.id = version_delta.id;
         self.max_committed_epoch = version_delta.max_committed_epoch;
-        for table_id in &version_delta.removed_table_ids {
-            let _ = self.table_watermarks.remove(table_id);
-        }
+        self.set_safe_epoch(version_delta.safe_epoch);
+
+        // apply to table watermark
 
-        let mut modified_table_watermarks: HashMap<TableId, TableWatermarks> = HashMap::new();
+        // Store the table watermarks that needs to be updated. None means to remove the table watermark of the table id
+        let mut modified_table_watermarks: HashMap<TableId, Option<TableWatermarks>> =
+            HashMap::new();
 
+        // apply to table watermark
         for (table_id, table_watermarks) in &version_delta.new_table_watermarks {
             if let Some(current_table_watermarks) = self.table_watermarks.get(table_id) {
-                let mut current_table_watermarks = (**current_table_watermarks).clone();
-                current_table_watermarks.apply_new_table_watermarks(table_watermarks);
-                modified_table_watermarks.insert(*table_id, current_table_watermarks);
+                if version_delta.removed_table_ids.contains(table_id) {
+                    modified_table_watermarks.insert(*table_id, None);
+                } else {
+                    let mut current_table_watermarks = (**current_table_watermarks).clone();
+                    current_table_watermarks.apply_new_table_watermarks(table_watermarks);
+                    modified_table_watermarks.insert(*table_id, Some(current_table_watermarks));
+                }
             } else {
-                modified_table_watermarks.insert(*table_id, table_watermarks.clone());
+                modified_table_watermarks.insert(*table_id, Some(table_watermarks.clone()));
             }
         }
-        if version_delta.safe_epoch != self.safe_epoch {
-            assert!(version_delta.safe_epoch > self.safe_epoch);
-            for (table_id, table_watermarks) in &self.table_watermarks {
-                let table_watermarks = modified_table_watermarks
-                    .entry(*table_id)
-                    .or_insert_with(|| (**table_watermarks).clone());
+        for (table_id, table_watermarks) in &self.table_watermarks {
+            if let Some(table_delta) = version_delta.state_table_info_delta.get(table_id)
+                && let Some(Some(prev_table)) = changed_table_info.get(table_id)
+                && table_delta.safe_epoch > prev_table.safe_epoch
+            {
+                // safe epoch has progressed, need further clear.
+            } else {
+                // safe epoch not progressed. No need to truncate
+                continue;
+            }
+            let table_watermarks = modified_table_watermarks
+                .entry(*table_id)
+                .or_insert_with(|| Some((**table_watermarks).clone()));
+            if let Some(table_watermarks) = table_watermarks {
                 table_watermarks.clear_stale_epoch_watermark(version_delta.safe_epoch);
             }
-            self.safe_epoch = version_delta.safe_epoch;
         }
-
+        // apply the staging table watermark to hummock version
         for (table_id, table_watermarks) in modified_table_watermarks {
-            self.table_watermarks
-                .insert(table_id, Arc::new(table_watermarks));
+            if let Some(table_watermarks) = table_watermarks {
+                self.table_watermarks
+                    .insert(table_id, Arc::new(table_watermarks));
+            } else {
+                self.table_watermarks.remove(&table_id);
+            }
         }
 
-        sst_split_info
-    }
+        // apply to table change log
+        for (table_id, change_log_delta) in &version_delta.change_log_delta {
+            let new_change_log = change_log_delta.new_log.as_ref().unwrap();
+            match self.table_change_log.entry(*table_id) {
+                Entry::Occupied(entry) => {
+                    let change_log = entry.into_mut();
+                    if let Some(prev_log) = change_log.0.last() {
+                        assert!(
+                            prev_log.epochs.last().expect("non-empty")
+                                < new_change_log.epochs.first().expect("non-empty")
+                        );
+                    }
+                    change_log.0.push(new_change_log.clone());
+                }
+                Entry::Vacant(entry) => {
+                    entry.insert(TableChangeLog(vec![new_change_log.clone()]));
+                }
+            };
+        }
+
+        // If a table has no new change log entry (even an empty one), it means we have stopped maintained
+        // the change log for the table, and then we will remove the table change log.
+        // The table change log will also be removed when the table id is removed.
+        self.table_change_log.retain(|table_id, _| {
+            if version_delta.removed_table_ids.contains(table_id) {
+                return false;
+            }
+            if let Some(table_info_delta) = version_delta.state_table_info_delta.get(table_id)
+                && let Some(Some(prev_table_info)) = changed_table_info.get(table_id) && table_info_delta.committed_epoch > prev_table_info.committed_epoch {
+                // the table exists previously, and its committed epoch has progressed.
+            } else {
+                // otherwise, the table change log should be kept anyway
+                return true;
+            }
+            let contains = version_delta.change_log_delta.contains_key(table_id);
+            if !contains {
+                warn!(
+                        ?table_id,
+                        max_committed_epoch = version_delta.max_committed_epoch,
+                        "table change log dropped due to no further change log at newly committed epoch",
+                    );
+            }
+            contains
+        });
 
-    pub fn build_compaction_group_info(&self) -> HashMap<TableId, CompactionGroupId> {
-        let mut ret = HashMap::new();
-        for (compaction_group_id, levels) in &self.levels {
-            for table_id in &levels.member_table_ids {
-                ret.insert(TableId::new(*table_id), *compaction_group_id);
+        // truncate the remaining table change log
+        for (table_id, change_log_delta) in &version_delta.change_log_delta {
+            if let Some(change_log) = self.table_change_log.get_mut(table_id) {
+                change_log.truncate(change_log_delta.truncate_epoch);
             }
         }
-        ret
     }
 
     pub fn build_branched_sst_info(&self) -> BTreeMap<HummockSstableObjectId, BranchedSstInfo> {
@@ -642,10 +737,10 @@ impl HummockVersion {
                     }
                     let object_id = table_info.get_object_id();
                     let entry: &mut BranchedSstInfo = ret.entry(object_id).or_default();
-                    if let Some(exist_sst_id) = entry.get(compaction_group_id) {
-                        panic!("we do not allow more than one sst with the same object id in one grou. object-id: {}, duplicated sst id: {:?} and {}", object_id, exist_sst_id, table_info.sst_id);
-                    }
-                    entry.insert(*compaction_group_id, table_info.sst_id);
+                    entry
+                        .entry(*compaction_group_id)
+                        .or_default()
+                        .push(table_info.sst_id)
                 }
             }
         }
@@ -683,7 +778,11 @@ impl Levels {
             .sum()
     }
 
-    pub fn apply_compact_ssts(&mut self, summary: GroupDeltasSummary) {
+    pub fn apply_compact_ssts(
+        &mut self,
+        summary: GroupDeltasSummary,
+        member_table_ids: &BTreeSet<TableId>,
+    ) {
         let GroupDeltasSummary {
             delete_sst_levels,
             delete_sst_ids_set,
@@ -736,9 +835,11 @@ impl Levels {
                     insert_sub_level_id, delete_sst_ids_set, l0.sub_levels.iter().map(|level| level.sub_level_id).collect_vec()
                 );
                 if l0.sub_levels[index].table_infos.is_empty()
-                    && self.member_table_ids.len() == 1
+                    && member_table_ids.len() == 1
                     && insert_table_infos.iter().all(|sst| {
-                        sst.table_ids.len() == 1 && sst.table_ids[0] == self.member_table_ids[0]
+                        sst.table_ids.len() == 1
+                            && sst.table_ids[0]
+                                == member_table_ids.iter().next().expect("non-empty").table_id
                     })
                 {
                     // Only change vnode_partition_count for group which has only one state-table.
@@ -756,7 +857,7 @@ impl Levels {
                     self.levels[idx].vnode_partition_count = new_vnode_partition_count;
                 } else if self.levels[idx].vnode_partition_count != 0
                     && new_vnode_partition_count == 0
-                    && self.member_table_ids.len() > 1
+                    && member_table_ids.len() > 1
                 {
                     self.levels[idx].vnode_partition_count = 0;
                 }
@@ -827,6 +928,7 @@ pub fn build_initial_compaction_group_levels(
             vnode_partition_count: 0,
         });
     }
+    #[expect(deprecated)] // for backward-compatibility of previous hummock version delta
     Levels {
         levels,
         l0: Some(OverlappingLevel {
@@ -844,7 +946,6 @@ fn split_sst_info_for_level(
     member_table_ids: &HashSet<u32>,
     allow_trivial_split: bool,
     level: &mut Level,
-    split_id_vers: &mut Vec<SstSplitInfo>,
     new_sst_id: &mut u64,
 ) -> Vec<SstableInfo> {
     // Remove SST from sub level may result in empty sub level. It will be purged
@@ -863,7 +964,6 @@ fn split_sst_info_for_level(
             let mut branch_table_info = sst_info.clone();
             branch_table_info.sst_id = *new_sst_id;
             *new_sst_id += 1;
-            let parent_old_sst_id = sst_info.get_sst_id();
             if is_trivial {
                 // This is a compatibility design. we only clear the table-ids for files which would
                 // be removed in later code. In the version-delta generated by new
@@ -874,30 +974,12 @@ fn split_sst_info_for_level(
                 sst_info.sst_id = *new_sst_id;
                 *new_sst_id += 1;
             }
-            split_id_vers.push((
-                branch_table_info.object_id,
-                branch_table_info.sst_id,
-                parent_old_sst_id,
-                sst_info.sst_id,
-            ));
             insert_table_infos.push(branch_table_info);
         }
     }
     insert_table_infos
 }
 
-pub fn try_get_compaction_group_id_by_table_id(
-    version: &HummockVersion,
-    table_id: StateTableId,
-) -> Option<CompactionGroupId> {
-    for (group_id, levels) in &version.levels {
-        if levels.member_table_ids.contains(&table_id) {
-            return Some(*group_id);
-        }
-    }
-    None
-}
-
 /// Gets all compaction group ids.
 pub fn get_compaction_group_ids(
     version: &HummockVersion,
@@ -908,9 +990,10 @@ pub fn get_compaction_group_ids(
 /// Gets all member table ids.
 pub fn get_member_table_ids(version: &HummockVersion) -> HashSet<StateTableId> {
     version
-        .levels
-        .iter()
-        .flat_map(|(_, levels)| levels.member_table_ids.clone())
+        .state_table_info
+        .info()
+        .keys()
+        .map(|table_id| table_id.table_id)
         .collect()
 }
 
@@ -918,14 +1001,10 @@ pub fn get_table_compaction_group_id_mapping(
     version: &HummockVersion,
 ) -> HashMap<StateTableId, CompactionGroupId> {
     version
-        .levels
+        .state_table_info
+        .info()
         .iter()
-        .flat_map(|(group_id, levels)| {
-            levels
-                .member_table_ids
-                .iter()
-                .map(|table_id| (*table_id, *group_id))
-        })
+        .map(|(table_id, info)| (table_id.table_id, info.compaction_group_id))
         .collect()
 }
 
@@ -1057,19 +1136,6 @@ pub fn insert_new_sub_level(
     l0.sub_levels.insert(insert_pos, level);
 }
 
-pub fn build_version_delta_after_version(version: &HummockVersion) -> HummockVersionDelta {
-    HummockVersionDelta {
-        id: version.id + 1,
-        prev_id: version.id,
-        safe_epoch: version.safe_epoch,
-        trivial_move: false,
-        max_committed_epoch: version.max_committed_epoch,
-        group_deltas: Default::default(),
-        new_table_watermarks: HashMap::new(),
-        removed_table_ids: vec![],
-    }
-}
-
 /// Delete sstables if the table id is in the id set.
 ///
 /// Return `true` if some sst is deleted, and `false` is the deletion is trivial
@@ -1148,14 +1214,14 @@ pub fn validate_version(version: &HummockVersion) -> Vec<String> {
     let mut res = Vec::new();
 
     // Ensure safe_epoch <= max_committed_epoch
-    if version.safe_epoch > version.max_committed_epoch {
+    if version.visible_table_safe_epoch() > version.max_committed_epoch {
         res.push(format!(
             "VERSION: safe_epoch {} > max_committed_epoch {}",
-            version.safe_epoch, version.max_committed_epoch
+            version.visible_table_safe_epoch(),
+            version.max_committed_epoch
         ));
     }
 
-    let mut table_to_group = HashMap::new();
     // Ensure each table maps to only one compaction group
     for (group_id, levels) in &version.levels {
         // Ensure compaction group id matches
@@ -1166,31 +1232,6 @@ pub fn validate_version(version: &HummockVersion) -> Vec<String> {
             ));
         }
 
-        // Ensure table id is sorted
-        if !levels.member_table_ids.is_sorted() {
-            res.push(format!(
-                "GROUP {}: memtable_table_ids is not sorted: {:?}",
-                group_id, levels.member_table_ids
-            ));
-        }
-
-        // Ensure table id is unique
-        for table_id in &levels.member_table_ids {
-            match table_to_group.entry(table_id) {
-                Entry::Occupied(e) => {
-                    res.push(format!(
-                        "GROUP {}: Duplicated table_id {}. First found in group {}",
-                        group_id,
-                        table_id,
-                        e.get()
-                    ));
-                }
-                Entry::Vacant(e) => {
-                    e.insert(group_id);
-                }
-            }
-        }
-
         let validate_level = |group: CompactionGroupId,
                               expected_level_idx: u32,
                               level: &Level,
@@ -1222,13 +1263,7 @@ pub fn validate_version(version: &HummockVersion) -> Vec<String> {
             let mut prev_table_info: Option<&SstableInfo> = None;
             for table_info in &level.table_infos {
                 // Ensure table_ids are sorted and unique
-                if !table_info.table_ids.is_sorted_by(|a, b| {
-                    if a < b {
-                        Some(Ordering::Less)
-                    } else {
-                        Some(Ordering::Greater)
-                    }
-                }) {
+                if !table_info.table_ids.is_sorted_by(|a, b| a < b) {
                     res.push(format!(
                         "{} SST {}: table_ids not sorted",
                         level_identifier, table_info.object_id
@@ -1298,25 +1333,20 @@ mod tests {
 
     #[test]
     fn test_get_sst_object_ids() {
-        let mut version = HummockVersion {
-            id: 0,
-            levels: HashMap::from_iter([(
-                0,
-                Levels {
-                    levels: vec![],
-                    l0: Some(OverlappingLevel {
-                        sub_levels: vec![],
-                        total_file_size: 0,
-                        uncompressed_file_size: 0,
-                    }),
-                    ..Default::default()
-                },
-            )]),
-            max_committed_epoch: 0,
-            safe_epoch: 0,
-            table_watermarks: HashMap::new(),
-            table_change_log: HashMap::new(),
-        };
+        let mut version = HummockVersion::default();
+        version.id = 0;
+        version.levels = HashMap::from_iter([(
+            0,
+            Levels {
+                levels: vec![],
+                l0: Some(OverlappingLevel {
+                    sub_levels: vec![],
+                    total_file_size: 0,
+                    uncompressed_file_size: 0,
+                }),
+                ..Default::default()
+            },
+        )]);
         assert_eq!(version.get_object_ids().len(), 0);
 
         // Add to sub level
@@ -1352,35 +1382,30 @@ mod tests {
 
     #[test]
     fn test_apply_version_delta() {
-        let mut version = HummockVersion {
-            id: 0,
-            levels: HashMap::from_iter([
-                (
+        let mut version = HummockVersion::default();
+        version.id = 0;
+        version.levels = HashMap::from_iter([
+            (
+                0,
+                build_initial_compaction_group_levels(
                     0,
-                    build_initial_compaction_group_levels(
-                        0,
-                        &CompactionConfig {
-                            max_level: 6,
-                            ..Default::default()
-                        },
-                    ),
+                    &CompactionConfig {
+                        max_level: 6,
+                        ..Default::default()
+                    },
                 ),
-                (
+            ),
+            (
+                1,
+                build_initial_compaction_group_levels(
                     1,
-                    build_initial_compaction_group_levels(
-                        1,
-                        &CompactionConfig {
-                            max_level: 6,
-                            ..Default::default()
-                        },
-                    ),
+                    &CompactionConfig {
+                        max_level: 6,
+                        ..Default::default()
+                    },
                 ),
-            ]),
-            max_committed_epoch: 0,
-            safe_epoch: 0,
-            table_watermarks: HashMap::new(),
-            table_change_log: HashMap::new(),
-        };
+            ),
+        ]);
         let version_delta = HummockVersionDelta {
             id: 1,
             group_deltas: HashMap::from_iter([
@@ -1443,28 +1468,23 @@ mod tests {
             }],
             ..Default::default()
         };
-        assert_eq!(
-            version,
-            HummockVersion {
-                id: 1,
-                levels: HashMap::from_iter([
-                    (
+        assert_eq!(version, {
+            let mut version = HummockVersion::default();
+            version.id = 1;
+            version.levels = HashMap::from_iter([
+                (
+                    2,
+                    build_initial_compaction_group_levels(
                         2,
-                        build_initial_compaction_group_levels(
-                            2,
-                            &CompactionConfig {
-                                max_level: 6,
-                                ..Default::default()
-                            }
-                        ),
+                        &CompactionConfig {
+                            max_level: 6,
+                            ..Default::default()
+                        },
                     ),
-                    (1, cg1,),
-                ]),
-                max_committed_epoch: 0,
-                safe_epoch: 0,
-                table_watermarks: HashMap::new(),
-                table_change_log: HashMap::new(),
-            }
-        );
+                ),
+                (1, cg1),
+            ]);
+            version
+        });
     }
 }
diff --git a/src/storage/hummock_sdk/src/key.rs b/src/storage/hummock_sdk/src/key.rs
index 4264dd161d3a1..6a33d1ff1a09b 100644
--- a/src/storage/hummock_sdk/src/key.rs
+++ b/src/storage/hummock_sdk/src/key.rs
@@ -24,6 +24,7 @@ use bytes::{Buf, BufMut, Bytes, BytesMut};
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common_estimate_size::EstimateSize;
+use serde::{Deserialize, Serialize};
 
 use crate::{EpochWithGap, HummockEpoch};
 
@@ -440,8 +441,14 @@ impl CopyFromSlice for Bytes {
 ///
 /// Its name come from the assumption that Hummock is always accessed by a table-like structure
 /// identified by a [`TableId`].
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
-pub struct TableKey<T: AsRef<[u8]>>(pub T);
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Serialize, Deserialize)]
+pub struct TableKey<T: AsRef<[u8]>>(
+    #[serde(bound(
+        serialize = "T: serde::Serialize + serde_bytes::Serialize",
+        deserialize = "T: serde::Deserialize<'de> + serde_bytes::Deserialize<'de>"
+    ))]
+    pub T,
+);
 
 impl<T: AsRef<[u8]>> Debug for TableKey<T> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -476,7 +483,11 @@ impl<T: AsRef<[u8]>> TableKey<T> {
             "too short table key: {:?}",
             self.0.as_ref()
         );
-        let (vnode, inner_key) = self.0.as_ref().split_array_ref::<{ VirtualNode::SIZE }>();
+        let (vnode, inner_key) = self
+            .0
+            .as_ref()
+            .split_first_chunk::<{ VirtualNode::SIZE }>()
+            .unwrap();
         (VirtualNode::from_be_bytes(*vnode), inner_key)
     }
 
@@ -531,11 +542,15 @@ pub fn gen_key_from_str(vnode: VirtualNode, payload: &str) -> TableKey<Bytes> {
 /// will group these two values into one struct for convenient filtering.
 ///
 /// The encoded format is | `table_id` | `table_key` |.
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Serialize, Deserialize)]
 pub struct UserKey<T: AsRef<[u8]>> {
     // When comparing `UserKey`, we first compare `table_id`, then `table_key`. So the order of
     // declaration matters.
     pub table_id: TableId,
+    #[serde(bound(
+        serialize = "T: serde::Serialize + serde_bytes::Serialize",
+        deserialize = "T: serde::Deserialize<'de> + serde_bytes::Deserialize<'de>"
+    ))]
     pub table_key: TableKey<T>,
 }
 
@@ -867,10 +882,14 @@ impl<T: AsRef<[u8]> + Ord + Eq> PartialOrd for FullKey<T> {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
 pub struct PointRange<T: AsRef<[u8]>> {
     // When comparing `PointRange`, we first compare `left_user_key`, then
     // `is_exclude_left_key`. Therefore the order of declaration matters.
+    #[serde(bound(
+        serialize = "T: serde::Serialize + serde_bytes::Serialize",
+        deserialize = "T: serde::Deserialize<'de> + serde_bytes::Deserialize<'de>"
+    ))]
     pub left_user_key: UserKey<T>,
     /// `PointRange` represents the left user key itself if `is_exclude_left_key==false`
     /// while represents the right δ Neighborhood of the left user key if
@@ -1096,8 +1115,6 @@ impl<T: AsRef<[u8]> + Ord + Eq, const SKIP_DEDUP: bool> FullKeyTracker<T, SKIP_D
 
 #[cfg(test)]
 mod tests {
-    use std::cmp::Ordering;
-
     use risingwave_common::util::epoch::test_epoch;
 
     use super::*;
diff --git a/src/storage/hummock_sdk/src/lib.rs b/src/storage/hummock_sdk/src/lib.rs
index 1fbd627e13f02..97e1a334dcf98 100644
--- a/src/storage/hummock_sdk/src/lib.rs
+++ b/src/storage/hummock_sdk/src/lib.rs
@@ -17,13 +17,12 @@
 #![feature(hash_extract_if)]
 #![feature(lint_reasons)]
 #![feature(map_many_mut)]
-#![feature(bound_map)]
 #![feature(type_alias_impl_trait)]
 #![feature(impl_trait_in_assoc_type)]
 #![feature(is_sorted)]
 #![feature(let_chains)]
 #![feature(btree_cursors)]
-#![feature(split_array)]
+#![feature(lazy_cell)]
 
 mod key_cmp;
 use std::cmp::Ordering;
@@ -101,6 +100,8 @@ pub struct SyncResult {
     pub uncommitted_ssts: Vec<LocalSstableInfo>,
     /// The collected table watermarks written by state tables.
     pub table_watermarks: HashMap<TableId, TableWatermarks>,
+    /// Sstable that holds the uncommitted old value
+    pub old_value_ssts: Vec<LocalSstableInfo>,
 }
 
 #[derive(Debug, Clone)]
diff --git a/src/storage/hummock_sdk/src/table_stats.rs b/src/storage/hummock_sdk/src/table_stats.rs
index 6f6bb9b9e3d40..4fce0e0f048ae 100644
--- a/src/storage/hummock_sdk/src/table_stats.rs
+++ b/src/storage/hummock_sdk/src/table_stats.rs
@@ -13,8 +13,9 @@
 // limitations under the License.
 
 use std::borrow::Borrow;
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 
+use risingwave_common::catalog::TableId;
 use risingwave_pb::hummock::PbTableStats;
 
 use crate::version::HummockVersion;
@@ -106,11 +107,12 @@ pub fn purge_prost_table_stats(
     table_stats: &mut PbTableStatsMap,
     hummock_version: &HummockVersion,
 ) -> bool {
-    let mut all_tables_in_version: HashSet<u32> = HashSet::default();
     let prev_count = table_stats.len();
-    for group in hummock_version.levels.values() {
-        all_tables_in_version.extend(group.member_table_ids.clone());
-    }
-    table_stats.retain(|k, _| all_tables_in_version.contains(k));
+    table_stats.retain(|table_id, _| {
+        hummock_version
+            .state_table_info
+            .info()
+            .contains_key(&TableId::new(*table_id))
+    });
     prev_count != table_stats.len()
 }
diff --git a/src/storage/hummock_sdk/src/table_watermark.rs b/src/storage/hummock_sdk/src/table_watermark.rs
index 2c5c2539e9ff7..73f7bac358e1b 100644
--- a/src/storage/hummock_sdk/src/table_watermark.rs
+++ b/src/storage/hummock_sdk/src/table_watermark.rs
@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::cmp::Ordering;
 use std::collections::hash_map::Entry;
 use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
+use std::fmt::Display;
 use std::mem::size_of;
 use std::ops::Bound::{Excluded, Included, Unbounded};
 use std::sync::Arc;
@@ -259,11 +259,11 @@ pub enum WatermarkDirection {
     Descending,
 }
 
-impl ToString for WatermarkDirection {
-    fn to_string(&self) -> String {
+impl Display for WatermarkDirection {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            WatermarkDirection::Ascending => "Ascending".to_string(),
-            WatermarkDirection::Descending => "Descending".to_string(),
+            WatermarkDirection::Ascending => write!(f, "Ascending"),
+            WatermarkDirection::Descending => write!(f, "Descending"),
         }
     }
 }
@@ -551,13 +551,8 @@ impl TableWatermarks {
         // epoch watermark are added from later epoch to earlier epoch.
         // reverse to ensure that earlier epochs are at the front
         result_epoch_watermark.reverse();
-        assert!(
-            result_epoch_watermark.is_sorted_by(|(first_epoch, _), (second_epoch, _)| {
-                let ret = first_epoch.cmp(second_epoch);
-                assert_ne!(ret, Ordering::Equal);
-                Some(ret)
-            })
-        );
+        assert!(result_epoch_watermark
+            .is_sorted_by(|(first_epoch, _), (second_epoch, _)| { first_epoch < second_epoch }));
         *self = TableWatermarks {
             watermarks: result_epoch_watermark,
             direction: self.direction,
@@ -1036,11 +1031,8 @@ mod tests {
             watermark3.clone(),
         );
 
-        let mut version = HummockVersion {
-            max_committed_epoch: EPOCH1,
-            safe_epoch: EPOCH1,
-            ..Default::default()
-        };
+        let mut version = HummockVersion::default();
+        version.max_committed_epoch = EPOCH1;
         let test_table_id = TableId::from(233);
         version.table_watermarks.insert(
             test_table_id,
diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs
index 3bf6fa02df463..0315ee9b8f298 100644
--- a/src/storage/hummock_sdk/src/version.rs
+++ b/src/storage/hummock_sdk/src/version.rs
@@ -12,29 +12,205 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{HashMap, HashSet};
-use std::mem::size_of;
-use std::sync::Arc;
+use std::collections::hash_map::Entry;
+use std::collections::{BTreeSet, HashMap, HashSet};
+use std::mem::{replace, size_of};
+use std::ops::Deref;
+use std::sync::{Arc, LazyLock};
 
 use prost::Message;
 use risingwave_common::catalog::TableId;
+use risingwave_common::util::epoch::INVALID_EPOCH;
 use risingwave_pb::hummock::group_delta::DeltaType;
-use risingwave_pb::hummock::hummock_version::PbLevels;
-use risingwave_pb::hummock::hummock_version_delta::GroupDeltas as PbGroupDeltas;
-use risingwave_pb::hummock::{PbHummockVersion, PbHummockVersionDelta};
+use risingwave_pb::hummock::hummock_version::Levels as PbLevels;
+use risingwave_pb::hummock::hummock_version_delta::{ChangeLogDelta, GroupDeltas as PbGroupDeltas};
+use risingwave_pb::hummock::{
+    CompactionConfig, HummockVersion as PbHummockVersion,
+    HummockVersionDelta as PbHummockVersionDelta, SstableInfo, StateTableInfo as PbStateTableInfo,
+    StateTableInfo, StateTableInfoDelta,
+};
+use tracing::warn;
 
 use crate::change_log::TableChangeLog;
+use crate::compaction_group::hummock_version_ext::build_initial_compaction_group_levels;
+use crate::compaction_group::StaticCompactionGroupId;
 use crate::table_watermark::TableWatermarks;
-use crate::{CompactionGroupId, HummockSstableObjectId};
+use crate::{CompactionGroupId, HummockSstableObjectId, HummockVersionId, FIRST_VERSION_ID};
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct HummockVersionStateTableInfo {
+    state_table_info: HashMap<TableId, PbStateTableInfo>,
+
+    // in memory index
+    compaction_group_member_tables: HashMap<CompactionGroupId, BTreeSet<TableId>>,
+}
+
+impl HummockVersionStateTableInfo {
+    pub fn empty() -> Self {
+        Self {
+            state_table_info: HashMap::new(),
+            compaction_group_member_tables: HashMap::new(),
+        }
+    }
+
+    fn build_compaction_group_member_tables(
+        state_table_info: &HashMap<TableId, PbStateTableInfo>,
+    ) -> HashMap<CompactionGroupId, BTreeSet<TableId>> {
+        let mut ret: HashMap<_, BTreeSet<_>> = HashMap::new();
+        for (table_id, info) in state_table_info {
+            assert!(ret
+                .entry(info.compaction_group_id)
+                .or_default()
+                .insert(*table_id));
+        }
+        ret
+    }
+
+    pub fn build_table_compaction_group_id(&self) -> HashMap<TableId, CompactionGroupId> {
+        self.state_table_info
+            .iter()
+            .map(|(table_id, info)| (*table_id, info.compaction_group_id))
+            .collect()
+    }
+
+    pub fn from_protobuf(state_table_info: &HashMap<u32, PbStateTableInfo>) -> Self {
+        let state_table_info = state_table_info
+            .iter()
+            .map(|(table_id, info)| (TableId::new(*table_id), info.clone()))
+            .collect();
+        let compaction_group_member_tables =
+            Self::build_compaction_group_member_tables(&state_table_info);
+        Self {
+            state_table_info,
+            compaction_group_member_tables,
+        }
+    }
+
+    pub fn to_protobuf(&self) -> HashMap<u32, PbStateTableInfo> {
+        self.state_table_info
+            .iter()
+            .map(|(table_id, info)| (table_id.table_id, info.clone()))
+            .collect()
+    }
+
+    pub fn apply_delta(
+        &mut self,
+        delta: &HashMap<TableId, StateTableInfoDelta>,
+        removed_table_id: &HashSet<TableId>,
+    ) -> HashMap<TableId, Option<StateTableInfo>> {
+        let mut changed_table = HashMap::new();
+        fn remove_table_from_compaction_group(
+            compaction_group_member_tables: &mut HashMap<CompactionGroupId, BTreeSet<TableId>>,
+            compaction_group_id: CompactionGroupId,
+            table_id: TableId,
+        ) {
+            let member_tables = compaction_group_member_tables
+                .get_mut(&compaction_group_id)
+                .expect("should exist");
+            assert!(member_tables.remove(&table_id));
+            if member_tables.is_empty() {
+                assert!(compaction_group_member_tables
+                    .remove(&compaction_group_id)
+                    .is_some());
+            }
+        }
+        for table_id in removed_table_id {
+            if let Some(prev_info) = self.state_table_info.remove(table_id) {
+                remove_table_from_compaction_group(
+                    &mut self.compaction_group_member_tables,
+                    prev_info.compaction_group_id,
+                    *table_id,
+                );
+                assert!(changed_table.insert(*table_id, Some(prev_info)).is_none());
+            } else {
+                warn!(
+                    table_id = table_id.table_id,
+                    "table to remove does not exist"
+                );
+            }
+        }
+        for (table_id, delta) in delta {
+            if removed_table_id.contains(table_id) {
+                continue;
+            }
+            let new_info = StateTableInfo {
+                committed_epoch: delta.committed_epoch,
+                safe_epoch: delta.safe_epoch,
+                compaction_group_id: delta.compaction_group_id,
+            };
+            match self.state_table_info.entry(*table_id) {
+                Entry::Occupied(mut entry) => {
+                    let prev_info = entry.get_mut();
+                    assert!(
+                        new_info.safe_epoch >= prev_info.safe_epoch
+                            && new_info.committed_epoch >= prev_info.committed_epoch,
+                        "state table info regress. table id: {}, prev_info: {:?}, new_info: {:?}",
+                        table_id.table_id,
+                        prev_info,
+                        new_info
+                    );
+                    if prev_info.compaction_group_id != new_info.compaction_group_id {
+                        // table moved to another compaction group
+                        remove_table_from_compaction_group(
+                            &mut self.compaction_group_member_tables,
+                            prev_info.compaction_group_id,
+                            *table_id,
+                        );
+                        assert!(self
+                            .compaction_group_member_tables
+                            .entry(new_info.compaction_group_id)
+                            .or_default()
+                            .insert(*table_id));
+                    }
+                    let prev_info = replace(prev_info, new_info);
+                    changed_table.insert(*table_id, Some(prev_info));
+                }
+                Entry::Vacant(entry) => {
+                    assert!(self
+                        .compaction_group_member_tables
+                        .entry(new_info.compaction_group_id)
+                        .or_default()
+                        .insert(*table_id));
+                    entry.insert(new_info);
+                    changed_table.insert(*table_id, None);
+                }
+            }
+        }
+        debug_assert_eq!(
+            self.compaction_group_member_tables,
+            Self::build_compaction_group_member_tables(&self.state_table_info)
+        );
+        changed_table
+    }
+
+    pub fn info(&self) -> &HashMap<TableId, StateTableInfo> {
+        &self.state_table_info
+    }
+
+    pub fn compaction_group_member_table_ids(
+        &self,
+        compaction_group_id: CompactionGroupId,
+    ) -> &BTreeSet<TableId> {
+        static EMPTY_SET: LazyLock<BTreeSet<TableId>> = LazyLock::new(BTreeSet::new);
+        self.compaction_group_member_tables
+            .get(&compaction_group_id)
+            .unwrap_or_else(|| EMPTY_SET.deref())
+    }
+
+    pub fn compaction_group_member_tables(&self) -> &HashMap<CompactionGroupId, BTreeSet<TableId>> {
+        &self.compaction_group_member_tables
+    }
+}
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct HummockVersion {
     pub id: u64,
     pub levels: HashMap<CompactionGroupId, PbLevels>,
     pub max_committed_epoch: u64,
-    pub safe_epoch: u64,
+    safe_epoch: u64,
     pub table_watermarks: HashMap<TableId, Arc<TableWatermarks>>,
     pub table_change_log: HashMap<TableId, TableChangeLog>,
+    pub state_table_info: HummockVersionStateTableInfo,
 }
 
 impl Default for HummockVersion {
@@ -86,6 +262,9 @@ impl HummockVersion {
                     )
                 })
                 .collect(),
+            state_table_info: HummockVersionStateTableInfo::from_protobuf(
+                &pb_version.state_table_info,
+            ),
         }
     }
 
@@ -109,6 +288,7 @@ impl HummockVersion {
                 .iter()
                 .map(|(table_id, change_log)| (table_id.table_id, change_log.to_protobuf()))
                 .collect(),
+            state_table_info: self.state_table_info.to_protobuf(),
         }
     }
 
@@ -126,6 +306,93 @@ impl HummockVersion {
                 .map(|table_watermark| table_watermark.estimated_encode_len())
                 .sum::<usize>()
     }
+
+    pub fn next_version_id(&self) -> HummockVersionId {
+        self.id + 1
+    }
+
+    pub fn need_fill_backward_compatible_state_table_info_delta(&self) -> bool {
+        // for backward-compatibility of previous hummock version delta
+        self.state_table_info.state_table_info.is_empty()
+            && self.levels.values().any(|group| {
+                // state_table_info is not previously filled, but there previously exists some tables
+                #[expect(deprecated)]
+                !group.member_table_ids.is_empty()
+            })
+    }
+
+    pub fn may_fill_backward_compatible_state_table_info_delta(
+        &self,
+        delta: &mut HummockVersionDelta,
+    ) {
+        #[expect(deprecated)]
+        // for backward-compatibility of previous hummock version delta
+        for (cg_id, group) in &self.levels {
+            for table_id in &group.member_table_ids {
+                assert!(
+                    delta
+                        .state_table_info_delta
+                        .insert(
+                            TableId::new(*table_id),
+                            StateTableInfoDelta {
+                                committed_epoch: self.max_committed_epoch,
+                                safe_epoch: self.safe_epoch,
+                                compaction_group_id: *cg_id,
+                            }
+                        )
+                        .is_none(),
+                    "duplicate table id {} in cg {}",
+                    table_id,
+                    cg_id
+                );
+            }
+        }
+    }
+
+    pub(crate) fn set_safe_epoch(&mut self, safe_epoch: u64) {
+        self.safe_epoch = safe_epoch;
+    }
+
+    pub fn visible_table_safe_epoch(&self) -> u64 {
+        self.safe_epoch
+    }
+
+    pub fn create_init_version(default_compaction_config: Arc<CompactionConfig>) -> HummockVersion {
+        let mut init_version = HummockVersion {
+            id: FIRST_VERSION_ID,
+            levels: Default::default(),
+            max_committed_epoch: INVALID_EPOCH,
+            safe_epoch: INVALID_EPOCH,
+            table_watermarks: HashMap::new(),
+            table_change_log: HashMap::new(),
+            state_table_info: HummockVersionStateTableInfo::empty(),
+        };
+        for group_id in [
+            StaticCompactionGroupId::StateDefault as CompactionGroupId,
+            StaticCompactionGroupId::MaterializedView as CompactionGroupId,
+        ] {
+            init_version.levels.insert(
+                group_id,
+                build_initial_compaction_group_levels(group_id, default_compaction_config.as_ref()),
+            );
+        }
+        init_version
+    }
+
+    pub fn version_delta_after(&self) -> HummockVersionDelta {
+        HummockVersionDelta {
+            id: self.next_version_id(),
+            prev_id: self.id,
+            safe_epoch: self.safe_epoch,
+            trivial_move: false,
+            max_committed_epoch: self.max_committed_epoch,
+            group_deltas: Default::default(),
+            new_table_watermarks: HashMap::new(),
+            removed_table_ids: HashSet::new(),
+            change_log_delta: HashMap::new(),
+            state_table_info_delta: Default::default(),
+        }
+    }
 }
 
 #[derive(Debug, PartialEq, Clone)]
@@ -137,7 +404,9 @@ pub struct HummockVersionDelta {
     pub safe_epoch: u64,
     pub trivial_move: bool,
     pub new_table_watermarks: HashMap<TableId, TableWatermarks>,
-    pub removed_table_ids: Vec<TableId>,
+    pub removed_table_ids: HashSet<TableId>,
+    pub change_log_delta: HashMap<TableId, ChangeLogDelta>,
+    pub state_table_info_delta: HashMap<TableId, StateTableInfoDelta>,
 }
 
 impl Default for HummockVersionDelta {
@@ -182,6 +451,24 @@ impl HummockVersionDelta {
                 .iter()
                 .map(|table_id| TableId::new(*table_id))
                 .collect(),
+            change_log_delta: delta
+                .change_log_delta
+                .iter()
+                .map(|(table_id, log_delta)| {
+                    (
+                        TableId::new(*table_id),
+                        ChangeLogDelta {
+                            new_log: log_delta.new_log.clone(),
+                            truncate_epoch: log_delta.truncate_epoch,
+                        },
+                    )
+                })
+                .collect(),
+            state_table_info_delta: delta
+                .state_table_info_delta
+                .iter()
+                .map(|(table_id, delta)| (TableId::new(*table_id), delta.clone()))
+                .collect(),
         }
     }
 
@@ -203,6 +490,16 @@ impl HummockVersionDelta {
                 .iter()
                 .map(|table_id| table_id.table_id)
                 .collect(),
+            change_log_delta: self
+                .change_log_delta
+                .iter()
+                .map(|(table_id, log_delta)| (table_id.table_id, log_delta.clone()))
+                .collect(),
+            state_table_info_delta: self
+                .state_table_info_delta
+                .iter()
+                .map(|(table_id, delta)| (table_id.table_id, delta.clone()))
+                .collect(),
         }
     }
 }
@@ -213,27 +510,31 @@ impl HummockVersionDelta {
     /// Note: the result can be false positive because we only collect the set of sst object ids in the `inserted_table_infos`,
     /// but it is possible that the object is moved or split from other compaction groups or levels.
     pub fn newly_added_object_ids(&self) -> HashSet<HummockSstableObjectId> {
-        let mut ret = HashSet::new();
-        for group_deltas in self.group_deltas.values() {
-            for group_delta in &group_deltas.group_deltas {
-                if let Some(delta_type) = &group_delta.delta_type {
-                    match delta_type {
-                        DeltaType::IntraLevel(level_delta) => {
-                            ret.extend(
-                                level_delta
-                                    .inserted_table_infos
-                                    .iter()
-                                    .map(|sst| sst.object_id),
-                            );
-                        }
-                        DeltaType::GroupConstruct(_)
-                        | DeltaType::GroupDestroy(_)
-                        | DeltaType::GroupMetaChange(_)
-                        | DeltaType::GroupTableChange(_) => {}
-                    }
-                }
-            }
-        }
-        ret
+        self.group_deltas
+            .values()
+            .flat_map(|group_deltas| {
+                group_deltas.group_deltas.iter().flat_map(|group_delta| {
+                    group_delta.delta_type.iter().flat_map(|delta_type| {
+                        static EMPTY_VEC: Vec<SstableInfo> = Vec::new();
+                        let sst_slice = match delta_type {
+                            DeltaType::IntraLevel(level_delta) => &level_delta.inserted_table_infos,
+                            DeltaType::GroupConstruct(_)
+                            | DeltaType::GroupDestroy(_)
+                            | DeltaType::GroupMetaChange(_)
+                            | DeltaType::GroupTableChange(_) => &EMPTY_VEC,
+                        };
+                        sst_slice.iter().map(|sst| sst.object_id)
+                    })
+                })
+            })
+            .chain(self.change_log_delta.values().flat_map(|delta| {
+                let new_log = delta.new_log.as_ref().unwrap();
+                new_log
+                    .new_value
+                    .iter()
+                    .map(|sst| sst.object_id)
+                    .chain(new_log.old_value.iter().map(|sst| sst.object_id))
+            }))
+            .collect()
     }
 }
diff --git a/src/storage/hummock_test/Cargo.toml b/src/storage/hummock_test/Cargo.toml
index 63993240d32a2..8a29c37400bf7 100644
--- a/src/storage/hummock_test/Cargo.toml
+++ b/src/storage/hummock_test/Cargo.toml
@@ -51,7 +51,7 @@ futures = { version = "0.3", default-features = false, features = [
 futures-async-stream = "0.2.9"
 risingwave_hummock_sdk = { workspace = true }
 risingwave_test_runner = { workspace = true }
-serial_test = "3.0"
+serial_test = "3.1"
 sync-point = { path = "../../utils/sync-point" }
 
 [features]
diff --git a/src/storage/hummock_test/benches/bench_hummock_iter.rs b/src/storage/hummock_test/benches/bench_hummock_iter.rs
index 059853433a2fc..9f0cef22e4e43 100644
--- a/src/storage/hummock_test/benches/bench_hummock_iter.rs
+++ b/src/storage/hummock_test/benches/bench_hummock_iter.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 
 use bytes::Bytes;
 use criterion::{criterion_group, criterion_main, Criterion};
-use foyer::memory::CacheContext;
+use foyer::CacheContext;
 use futures::pin_mut;
 use risingwave_common::util::epoch::test_epoch;
 use risingwave_hummock_sdk::key::TableKey;
@@ -32,7 +32,6 @@ use risingwave_storage::hummock::test_utils::default_opts_for_test;
 use risingwave_storage::hummock::{CachePolicy, HummockStorage};
 use risingwave_storage::storage_value::StorageValue;
 use risingwave_storage::store::*;
-use risingwave_storage::StateStore;
 
 fn gen_interleave_shared_buffer_batch_iter(
     batch_size: usize,
@@ -57,7 +56,6 @@ fn gen_interleave_shared_buffer_batch_iter(
 fn criterion_benchmark(c: &mut Criterion) {
     let runtime = tokio::runtime::Runtime::new().unwrap();
     let batches = gen_interleave_shared_buffer_batch_iter(10000, 100);
-    let sstable_store = mock_sstable_store();
     let hummock_options = Arc::new(default_opts_for_test());
     let (env, hummock_manager_ref, _cluster_manager_ref, worker_node) =
         runtime.block_on(setup_compute_env(8080));
@@ -67,6 +65,7 @@ fn criterion_benchmark(c: &mut Criterion) {
     ));
 
     let global_hummock_storage = runtime.block_on(async {
+        let sstable_store = mock_sstable_store().await;
         HummockStorage::for_test(
             hummock_options,
             sstable_store,
diff --git a/src/storage/hummock_test/src/bin/replay/main.rs b/src/storage/hummock_test/src/bin/replay/main.rs
index 693a6448a4ddc..9c1827944bb71 100644
--- a/src/storage/hummock_test/src/bin/replay/main.rs
+++ b/src/storage/hummock_test/src/bin/replay/main.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(bound_map)]
 #![feature(coroutines)]
 #![feature(stmt_expr_attributes)]
 #![feature(proc_macro_hygiene)]
@@ -29,10 +28,10 @@ use std::path::Path;
 use std::sync::Arc;
 
 use clap::Parser;
+use foyer::HybridCacheBuilder;
 use replay_impl::{get_replay_notification_client, GlobalReplayImpl};
 use risingwave_common::config::{
-    extract_storage_memory_config, load_config, EvictionConfig, NoOverride, ObjectStoreConfig,
-    StorageConfig,
+    extract_storage_memory_config, load_config, NoOverride, ObjectStoreConfig, StorageConfig,
 };
 use risingwave_common::system_param::reader::SystemParamsReader;
 use risingwave_hummock_trace::{
@@ -44,7 +43,7 @@ use risingwave_object_store::object::build_remote_object_store;
 use risingwave_storage::filter_key_extractor::{
     FakeRemoteTableAccessor, RpcFilterKeyExtractorManager,
 };
-use risingwave_storage::hummock::{FileCache, HummockStorage, SstableStore, SstableStoreConfig};
+use risingwave_storage::hummock::{HummockStorage, SstableStore, SstableStoreConfig};
 use risingwave_storage::monitor::{CompactorMetrics, HummockStateStoreMetrics, ObjectStoreMetrics};
 use risingwave_storage::opts::StorageOpts;
 use serde::{Deserialize, Serialize};
@@ -107,25 +106,34 @@ async fn create_replay_hummock(r: Record, args: &Args) -> Result<impl GlobalRepl
         &args.object_storage,
         object_store_metrics,
         "Hummock",
-        ObjectStoreConfig::default(),
+        Arc::new(ObjectStoreConfig::default()),
     )
     .await;
 
+    let meta_cache = HybridCacheBuilder::new()
+        .memory(storage_opts.meta_cache_capacity_mb * (1 << 20))
+        .with_shards(storage_opts.meta_cache_shard_num)
+        .storage()
+        .build()
+        .await
+        .unwrap();
+    let block_cache = HybridCacheBuilder::new()
+        .memory(storage_opts.block_cache_capacity_mb * (1 << 20))
+        .with_shards(storage_opts.block_cache_shard_num)
+        .storage()
+        .build()
+        .await
+        .unwrap();
+
     let sstable_store = Arc::new(SstableStore::new(SstableStoreConfig {
         store: Arc::new(object_store),
         path: storage_opts.data_directory.to_string(),
-        block_cache_capacity: storage_opts.block_cache_capacity_mb * (1 << 20),
-        meta_cache_capacity: storage_opts.meta_cache_capacity_mb * (1 << 20),
-        block_cache_shard_num: storage_opts.block_cache_shard_num,
-        meta_cache_shard_num: storage_opts.meta_cache_shard_num,
-        block_cache_eviction: EvictionConfig::for_test(),
-        meta_cache_eviction: EvictionConfig::for_test(),
         prefetch_buffer_capacity: storage_opts.prefetch_buffer_capacity_mb * (1 << 20),
         max_prefetch_block_number: storage_opts.max_prefetch_block_number,
-        data_file_cache: FileCache::none(),
-        meta_file_cache: FileCache::none(),
         recent_filter: None,
         state_store_metrics: state_store_metrics.clone(),
+        meta_cache,
+        block_cache,
     }));
 
     let (hummock_meta_client, notification_client, notifier) = {
diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs
index 7b8da8c92f077..a10b8963a6d7b 100644
--- a/src/storage/hummock_test/src/compactor_tests.rs
+++ b/src/storage/hummock_test/src/compactor_tests.rs
@@ -17,11 +17,10 @@ pub(crate) mod tests {
 
     use std::collections::{BTreeMap, BTreeSet, VecDeque};
     use std::ops::Bound;
-    use std::sync::atomic::AtomicU32;
     use std::sync::Arc;
 
     use bytes::{BufMut, Bytes, BytesMut};
-    use foyer::memory::CacheContext;
+    use foyer::CacheContext;
     use itertools::Itertools;
     use rand::{Rng, RngCore, SeedableRng};
     use risingwave_common::buffer::BitmapBuilder;
@@ -104,7 +103,7 @@ pub(crate) mod tests {
             write_conflict_detection_enabled: true,
             ..Default::default()
         });
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
 
         let hummock = GlobalHummockStorage::for_test(
             options,
@@ -138,7 +137,7 @@ pub(crate) mod tests {
             write_conflict_detection_enabled: true,
             ..Default::default()
         });
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
 
         GlobalHummockStorage::for_test(
             options,
@@ -200,12 +199,6 @@ pub(crate) mod tests {
         storage_opts: Arc<StorageOpts>,
         sstable_store: SstableStoreRef,
     ) -> CompactorContext {
-        let compaction_executor = Arc::new(CompactionExecutor::new(Some(1)));
-        let max_task_parallelism = Arc::new(AtomicU32::new(
-            (compaction_executor.worker_num() as f32 * storage_opts.compactor_max_task_multiplier)
-                .ceil() as u32,
-        ));
-
         CompactorContext {
             storage_opts,
             sstable_store,
@@ -215,8 +208,6 @@ pub(crate) mod tests {
             memory_limiter: MemoryLimiter::unlimit(),
             task_progress_manager: Default::default(),
             await_tree_reg: None,
-            running_task_parallelism: Arc::new(AtomicU32::new(0)),
-            max_task_parallelism,
         }
     }
 
diff --git a/src/storage/hummock_test/src/failpoint_tests.rs b/src/storage/hummock_test/src/failpoint_tests.rs
index d2ab797cb634a..240c07cd82c4b 100644
--- a/src/storage/hummock_test/src/failpoint_tests.rs
+++ b/src/storage/hummock_test/src/failpoint_tests.rs
@@ -16,7 +16,7 @@ use std::ops::Bound;
 use std::sync::Arc;
 
 use bytes::{BufMut, Bytes};
-use foyer::memory::CacheContext;
+use foyer::CacheContext;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_hummock_sdk::key::TABLE_PREFIX_LEN;
@@ -43,7 +43,7 @@ use crate::test_utils::{gen_key_from_str, TestIngestBatch};
 async fn test_failpoints_state_store_read_upload() {
     let mem_upload_err = "mem_upload_err";
     let mem_read_err = "mem_read_err";
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     let hummock_options = Arc::new(default_opts_for_test());
     let (env, hummock_manager_ref, _cluster_manager_ref, worker_node) =
         setup_compute_env(8080).await;
@@ -147,8 +147,8 @@ async fn test_failpoints_state_store_read_upload() {
         .await
         .unwrap();
     // clear block cache
-    sstable_store.clear_block_cache();
-    sstable_store.clear_meta_cache();
+    sstable_store.clear_block_cache().await.unwrap();
+    sstable_store.clear_meta_cache().await.unwrap();
     fail::cfg(mem_read_err, "return").unwrap();
 
     let anchor_prefix_hint = {
diff --git a/src/storage/hummock_test/src/hummock_read_version_tests.rs b/src/storage/hummock_test/src/hummock_read_version_tests.rs
index 0da472629751d..2329970b485f5 100644
--- a/src/storage/hummock_test/src/hummock_read_version_tests.rs
+++ b/src/storage/hummock_test/src/hummock_read_version_tests.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
 use std::ops::Bound;
 use std::sync::Arc;
 
@@ -26,6 +27,7 @@ use risingwave_hummock_sdk::key::{key_with_epoch, map_table_key_range};
 use risingwave_hummock_sdk::LocalSstableInfo;
 use risingwave_meta::hummock::test_utils::setup_compute_env;
 use risingwave_pb::hummock::{KeyRange, SstableInfo};
+use risingwave_storage::hummock::event_handler::TEST_LOCAL_INSTANCE_ID;
 use risingwave_storage::hummock::iterator::test_utils::{
     iterator_test_table_key_of, iterator_test_user_key_of,
 };
@@ -48,13 +50,17 @@ async fn test_read_version_basic() {
     let mut epoch = test_epoch(1);
     let table_id = 0;
     let vnodes = Arc::new(Bitmap::ones(VirtualNode::COUNT));
-    let mut read_version = HummockReadVersion::new(TableId::from(table_id), pinned_version, vnodes);
+    let mut read_version = HummockReadVersion::new(
+        TableId::from(table_id),
+        TEST_LOCAL_INSTANCE_ID,
+        pinned_version,
+        vnodes,
+    );
 
     {
         // single imm
-        let kv_pairs = gen_dummy_batch(1);
-        let sorted_items = SharedBufferBatch::build_shared_buffer_item_batches(kv_pairs);
-        let size = SharedBufferBatch::measure_batch_size(&sorted_items);
+        let sorted_items = gen_dummy_batch(1);
+        let size = SharedBufferBatch::measure_batch_size(&sorted_items, None).0;
         let imm = SharedBufferBatch::build_shared_buffer_batch_for_test(
             epoch,
             0,
@@ -87,9 +93,8 @@ async fn test_read_version_basic() {
         // several epoch
         for i in 0..5 {
             epoch.inc_epoch();
-            let kv_pairs = gen_dummy_batch(i + 2);
-            let sorted_items = SharedBufferBatch::build_shared_buffer_item_batches(kv_pairs);
-            let size = SharedBufferBatch::measure_batch_size(&sorted_items);
+            let sorted_items = gen_dummy_batch(i + 2);
+            let size = SharedBufferBatch::measure_batch_size(&sorted_items, None).0;
             let imm = SharedBufferBatch::build_shared_buffer_batch_for_test(
                 epoch,
                 0,
@@ -178,8 +183,9 @@ async fn test_read_version_basic() {
                     ..Default::default()
                 }),
             ],
+            vec![],
             epoch_id_vec_for_clear,
-            batch_id_vec_for_clear,
+            HashMap::from_iter([(TEST_LOCAL_INSTANCE_ID, batch_id_vec_for_clear)]),
             1,
         ));
 
@@ -268,6 +274,7 @@ async fn test_read_filter_basic() {
     let vnodes = Arc::new(Bitmap::ones(VirtualNode::COUNT));
     let read_version = Arc::new(RwLock::new(HummockReadVersion::new(
         TableId::from(table_id),
+        TEST_LOCAL_INSTANCE_ID,
         pinned_version,
         vnodes.clone(),
     )));
@@ -275,9 +282,8 @@ async fn test_read_filter_basic() {
 
     {
         // single imm
-        let kv_pairs = gen_dummy_batch(epoch);
-        let sorted_items = SharedBufferBatch::build_shared_buffer_item_batches(kv_pairs);
-        let size = SharedBufferBatch::measure_batch_size(&sorted_items);
+        let sorted_items = gen_dummy_batch(epoch);
+        let size = SharedBufferBatch::measure_batch_size(&sorted_items, None).0;
         let imm = SharedBufferBatch::build_shared_buffer_batch_for_test(
             epoch,
             0,
diff --git a/src/storage/hummock_test/src/hummock_storage_tests.rs b/src/storage/hummock_test/src/hummock_storage_tests.rs
index f7711b7fcdf13..5c6e1607de926 100644
--- a/src/storage/hummock_test/src/hummock_storage_tests.rs
+++ b/src/storage/hummock_test/src/hummock_storage_tests.rs
@@ -16,7 +16,7 @@ use std::ops::Range;
 use std::sync::Arc;
 
 use bytes::{BufMut, Bytes};
-use foyer::memory::CacheContext;
+use foyer::CacheContext;
 use futures::TryStreamExt;
 use itertools::Itertools;
 use risingwave_common::buffer::BitmapBuilder;
@@ -37,7 +37,6 @@ use risingwave_storage::hummock::store::version::read_filter_for_version;
 use risingwave_storage::hummock::{CachePolicy, HummockStorage, LocalHummockStorage};
 use risingwave_storage::storage_value::StorageValue;
 use risingwave_storage::store::*;
-use risingwave_storage::StateStore;
 
 use crate::local_state_store_test_utils::LocalStateStoreTestExt;
 use crate::test_utils::{gen_key_from_str, prepare_hummock_test_env, TestIngestBatch};
@@ -531,6 +530,9 @@ async fn test_state_store_sync() {
         .await
         .unwrap();
 
+    let epoch3 = epoch2.next_epoch();
+    hummock_storage.seal_current_epoch(epoch3, SealCurrentEpochOptions::for_test());
+
     let res = test_env.storage.seal_and_sync_epoch(epoch1).await.unwrap();
     test_env
         .meta_client
@@ -636,12 +638,36 @@ async fn test_state_store_sync() {
             .into_stream(to_owned_item);
         futures::pin_mut!(iter);
 
+        let rev_iter = test_env
+            .storage
+            .rev_iter(
+                (
+                    Unbounded,
+                    Included(gen_key_from_str(VirtualNode::ZERO, "eeee")),
+                ),
+                epoch1,
+                ReadOptions {
+                    table_id: TEST_TABLE_ID,
+                    cache_policy: CachePolicy::Fill(CacheContext::Default),
+                    ..Default::default()
+                },
+            )
+            .await
+            .unwrap()
+            .into_stream(to_owned_item);
+        futures::pin_mut!(rev_iter);
+        let mut rev_results = vec![];
+        while let Some(result) = rev_iter.try_next().await.unwrap() {
+            rev_results.push(result);
+        }
         let kv_map_batch_1 = [
             (gen_key_from_str(VirtualNode::ZERO, "aaaa"), "1111", epoch1),
             (gen_key_from_str(VirtualNode::ZERO, "bbbb"), "2222", epoch1),
         ];
         for (k, v, e) in kv_map_batch_1 {
             let result = iter.try_next().await.unwrap();
+            let rev_result = rev_results.pop();
+            assert_eq!(result, rev_result);
             assert_eq!(
                 result,
                 Some((
@@ -658,6 +684,8 @@ async fn test_state_store_sync() {
 
         for (k, v, e) in kv_map_batch_2 {
             let result = iter.try_next().await.unwrap();
+            let rev_result = rev_results.pop();
+            assert_eq!(result, rev_result);
             assert_eq!(
                 result,
                 Some((
@@ -691,12 +719,37 @@ async fn test_state_store_sync() {
 
         futures::pin_mut!(iter);
 
+        let rev_iter = test_env
+            .storage
+            .rev_iter(
+                (
+                    Unbounded,
+                    Included(gen_key_from_str(VirtualNode::ZERO, "eeee")),
+                ),
+                epoch2,
+                ReadOptions {
+                    table_id: TEST_TABLE_ID,
+                    cache_policy: CachePolicy::Fill(CacheContext::Default),
+                    ..Default::default()
+                },
+            )
+            .await
+            .unwrap()
+            .into_stream(to_owned_item);
+        futures::pin_mut!(rev_iter);
+        let mut rev_results = vec![];
+        while let Some(result) = rev_iter.try_next().await.unwrap() {
+            rev_results.push(result);
+        }
+
         let kv_map_batch_1 = [("aaaa", "1111", epoch1), ("bbbb", "2222", epoch1)];
 
         let kv_map_batch_2 = [("cccc", "3333", epoch1), ("dddd", "4444", epoch1)];
         let kv_map_batch_3 = [("eeee", "6666", epoch2)];
         for (k, v, e) in kv_map_batch_1 {
             let result = iter.try_next().await.unwrap();
+            let rev_result = rev_results.pop();
+            assert_eq!(result, rev_result);
             assert_eq!(
                 result,
                 Some((
@@ -708,6 +761,8 @@ async fn test_state_store_sync() {
 
         for (k, v, e) in kv_map_batch_2 {
             let result = iter.try_next().await.unwrap();
+            let rev_result = rev_results.pop();
+            assert_eq!(result, rev_result);
             assert_eq!(
                 result,
                 Some((
@@ -777,14 +832,15 @@ async fn test_delete_get() {
         .await
         .unwrap();
 
+    let epoch2 = epoch1.next_epoch();
+    hummock_storage.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
     let res = test_env.storage.seal_and_sync_epoch(epoch1).await.unwrap();
     test_env
         .meta_client
         .commit_epoch(epoch1, res)
         .await
         .unwrap();
-    let epoch2 = epoch1.next_epoch();
-    hummock_storage.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
+
     let batch2 = vec![(
         gen_key_from_str(VirtualNode::ZERO, "bb"),
         StorageValue::new_delete(),
@@ -799,6 +855,7 @@ async fn test_delete_get() {
         )
         .await
         .unwrap();
+    hummock_storage.seal_current_epoch(u64::MAX, SealCurrentEpochOptions::for_test());
     let res = test_env.storage.seal_and_sync_epoch(epoch2).await.unwrap();
     test_env
         .meta_client
@@ -953,6 +1010,8 @@ async fn test_multiple_epoch_sync() {
     };
     test_get().await;
 
+    let epoch4 = epoch3.next_epoch();
+    hummock_storage.seal_current_epoch(epoch4, SealCurrentEpochOptions::for_test());
     test_env.storage.seal_epoch(epoch1, false);
     let sync_result2 = test_env.storage.seal_and_sync_epoch(epoch2).await.unwrap();
     let sync_result3 = test_env.storage.seal_and_sync_epoch(epoch3).await.unwrap();
@@ -1027,6 +1086,9 @@ async fn test_iter_with_min_epoch() {
         .await
         .unwrap();
 
+    let epoch3 = (33 * 1000) << 16;
+    hummock_storage.seal_current_epoch(epoch3, SealCurrentEpochOptions::for_test());
+
     {
         // test before sync
         {
@@ -1277,6 +1339,9 @@ async fn test_hummock_version_reader() {
             .await
             .unwrap();
 
+        let epoch4 = (34 * 1000) << 16;
+        hummock_storage.seal_current_epoch(epoch4, SealCurrentEpochOptions::for_test());
+
         {
             // test before sync
             {
@@ -1687,6 +1752,8 @@ async fn test_get_with_min_epoch() {
         .await
         .unwrap();
 
+    hummock_storage.seal_current_epoch(u64::MAX, SealCurrentEpochOptions::for_test());
+
     {
         // test before sync
         let k = gen_key(0);
diff --git a/src/storage/hummock_test/src/lib.rs b/src/storage/hummock_test/src/lib.rs
index 890277678b756..a89ead17944f5 100644
--- a/src/storage/hummock_test/src/lib.rs
+++ b/src/storage/hummock_test/src/lib.rs
@@ -14,9 +14,7 @@
 #![feature(proc_macro_hygiene, stmt_expr_attributes)]
 #![feature(custom_test_frameworks)]
 #![test_runner(risingwave_test_runner::test_runner::run_failpont_tests)]
-#![feature(bound_map)]
 #![feature(type_alias_impl_trait)]
-#![feature(associated_type_bounds)]
 
 #[cfg(test)]
 mod compactor_tests;
diff --git a/src/storage/hummock_test/src/snapshot_tests.rs b/src/storage/hummock_test/src/snapshot_tests.rs
index 981ed3687bd40..b85457f309f73 100644
--- a/src/storage/hummock_test/src/snapshot_tests.rs
+++ b/src/storage/hummock_test/src/snapshot_tests.rs
@@ -15,7 +15,7 @@ use std::ops::Bound;
 use std::sync::Arc;
 
 use bytes::Bytes;
-use foyer::memory::CacheContext;
+use foyer::CacheContext;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::util::epoch::{test_epoch, EpochExt};
 use risingwave_hummock_sdk::key::prefixed_range_with_vnode;
diff --git a/src/storage/hummock_test/src/state_store_tests.rs b/src/storage/hummock_test/src/state_store_tests.rs
index 40dd23d78d987..fe129deb52245 100644
--- a/src/storage/hummock_test/src/state_store_tests.rs
+++ b/src/storage/hummock_test/src/state_store_tests.rs
@@ -13,34 +13,39 @@
 // limitations under the License.
 
 use std::ops::Bound;
+use std::ops::Bound::{Excluded, Included, Unbounded};
 use std::sync::Arc;
 
 use bytes::Bytes;
 use expect_test::expect;
-use foyer::memory::CacheContext;
+use foyer::CacheContext;
 use futures::{pin_mut, StreamExt};
+use itertools::Itertools;
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::{TableId, TableOption};
+use risingwave_common::hash::table_distribution::TableDistribution;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::util::epoch::{test_epoch, EpochExt};
-use risingwave_hummock_sdk::key::prefixed_range_with_vnode;
+use risingwave_hummock_sdk::key::{prefixed_range_with_vnode, TableKeyRange};
 use risingwave_hummock_sdk::{
     HummockReadEpoch, HummockSstableObjectId, LocalSstableInfo, SyncResult,
 };
 use risingwave_meta::hummock::test_utils::setup_compute_env;
-use risingwave_meta::hummock::MockHummockMetaClient;
 use risingwave_rpc_client::HummockMetaClient;
+use risingwave_storage::hummock::iterator::change_log::test_utils::{
+    apply_test_log_data, gen_test_data,
+};
 use risingwave_storage::hummock::iterator::test_utils::mock_sstable_store;
 use risingwave_storage::hummock::test_utils::{count_stream, default_opts_for_test};
 use risingwave_storage::hummock::{CachePolicy, HummockStorage};
+use risingwave_storage::memory::MemoryStateStore;
 use risingwave_storage::storage_value::StorageValue;
 use risingwave_storage::store::*;
+use risingwave_storage::store_impl::verify::VerifyStateStore;
 
 use crate::get_notification_client_for_test;
 use crate::local_state_store_test_utils::LocalStateStoreTestExt;
-use crate::test_utils::{
-    gen_key_from_str, with_hummock_storage_v2, HummockStateStoreTestTrait, TestIngestBatch,
-};
+use crate::test_utils::{gen_key_from_str, with_hummock_storage_v2, TestIngestBatch};
 
 #[tokio::test]
 async fn test_empty_read_v2() {
@@ -80,13 +85,6 @@ async fn test_empty_read_v2() {
 #[tokio::test]
 async fn test_basic_v2() {
     let (hummock_storage, meta_client) = with_hummock_storage_v2(Default::default()).await;
-    test_basic_inner(hummock_storage, meta_client).await;
-}
-
-async fn test_basic_inner(
-    hummock_storage: impl HummockStateStoreTestTrait,
-    meta_client: Arc<MockHummockMetaClient>,
-) {
     let anchor = gen_key_from_str(VirtualNode::ZERO, "aa");
 
     // First batch inserts the anchor and others.
@@ -409,14 +407,8 @@ async fn test_basic_inner(
 
 #[tokio::test]
 async fn test_state_store_sync_v2() {
-    let (hummock_storage, meta_client) = with_hummock_storage_v2(Default::default()).await;
-    test_state_store_sync_inner(hummock_storage, meta_client).await;
-}
+    let (hummock_storage, _meta_client) = with_hummock_storage_v2(Default::default()).await;
 
-async fn test_state_store_sync_inner(
-    hummock_storage: impl HummockStateStoreTestTrait,
-    _meta_client: Arc<MockHummockMetaClient>,
-) {
     let mut epoch = hummock_storage
         .get_pinned_version()
         .max_committed_epoch()
@@ -534,7 +526,7 @@ async fn test_state_store_sync_inner(
 /// Fix this when we finished epoch management.
 #[ignore]
 async fn test_reload_storage() {
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     let hummock_options = Arc::new(default_opts_for_test());
     let (env, hummock_manager_ref, _cluster_manager_ref, worker_node) =
         setup_compute_env(8080).await;
@@ -1027,13 +1019,7 @@ async fn test_reload_storage() {
 #[tokio::test]
 async fn test_delete_get_v2() {
     let (hummock_storage, meta_client) = with_hummock_storage_v2(Default::default()).await;
-    test_delete_get_inner(hummock_storage, meta_client).await;
-}
 
-async fn test_delete_get_inner(
-    hummock_storage: impl HummockStateStoreTestTrait,
-    meta_client: Arc<MockHummockMetaClient>,
-) {
     let initial_epoch = hummock_storage.get_pinned_version().max_committed_epoch();
     let epoch1 = initial_epoch.next_epoch();
     let batch1 = vec![
@@ -1060,11 +1046,12 @@ async fn test_delete_get_inner(
         )
         .await
         .unwrap();
-    let res = hummock_storage.seal_and_sync_epoch(epoch1).await.unwrap();
-    meta_client.commit_epoch(epoch1, res).await.unwrap();
     let epoch2 = epoch1.next_epoch();
 
     local.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test());
+    let res = hummock_storage.seal_and_sync_epoch(epoch1).await.unwrap();
+    meta_client.commit_epoch(epoch1, res).await.unwrap();
+
     let batch2 = vec![(
         gen_key_from_str(VirtualNode::ZERO, "bb"),
         StorageValue::new_delete(),
@@ -1103,13 +1090,7 @@ async fn test_delete_get_inner(
 #[tokio::test]
 async fn test_multiple_epoch_sync_v2() {
     let (hummock_storage, meta_client) = with_hummock_storage_v2(Default::default()).await;
-    test_multiple_epoch_sync_inner(hummock_storage, meta_client).await;
-}
 
-async fn test_multiple_epoch_sync_inner(
-    hummock_storage: impl HummockStateStoreTestTrait,
-    meta_client: Arc<MockHummockMetaClient>,
-) {
     let initial_epoch = hummock_storage.get_pinned_version().max_committed_epoch();
     let epoch1 = initial_epoch.next_epoch();
     let batch1 = vec![
@@ -1546,3 +1527,118 @@ async fn test_replicated_local_hummock_storage() {
         expected.assert_debug_eq(&actual);
     }
 }
+
+#[tokio::test]
+async fn test_iter_log() {
+    let table_id = TableId::new(233);
+    let (hummock_storage, meta_client) = with_hummock_storage_v2(table_id).await;
+    let epoch_count = 10;
+    let key_count = 10000;
+
+    let test_log_data = gen_test_data(epoch_count, key_count, 0.05, 0.2);
+    let in_memory_state_store = MemoryStateStore::new();
+
+    let mut in_memory_local = in_memory_state_store
+        .new_local(NewLocalOptions {
+            table_id,
+            op_consistency_level: OpConsistencyLevel::ConsistentOldValue {
+                check_old_value: CHECK_BYTES_EQUAL.clone(),
+                is_log_store: true,
+            },
+            table_option: Default::default(),
+            is_replicated: false,
+            vnodes: TableDistribution::all_vnodes(),
+        })
+        .await;
+
+    apply_test_log_data(test_log_data.clone(), &mut in_memory_local, 0.0).await;
+
+    let mut hummock_local = hummock_storage
+        .new_local(NewLocalOptions {
+            table_id,
+            op_consistency_level: OpConsistencyLevel::ConsistentOldValue {
+                check_old_value: CHECK_BYTES_EQUAL.clone(),
+                is_log_store: true,
+            },
+            table_option: Default::default(),
+            is_replicated: false,
+            vnodes: TableDistribution::all_vnodes(),
+        })
+        .await;
+    // flush for about 10 times per epoch
+    apply_test_log_data(test_log_data.clone(), &mut hummock_local, 0.001).await;
+
+    for (epoch, _) in &test_log_data {
+        let res = hummock_storage.seal_and_sync_epoch(*epoch).await.unwrap();
+        if *epoch != test_log_data[0].0 {
+            assert!(!res.old_value_ssts.is_empty());
+        }
+        assert!(!res.uncommitted_ssts.is_empty());
+        meta_client.commit_epoch(*epoch, res).await.unwrap();
+    }
+
+    hummock_storage
+        .try_wait_epoch_for_test(test_log_data.last().unwrap().0)
+        .await;
+
+    let verify_state_store = VerifyStateStore {
+        actual: hummock_storage,
+        expected: Some(in_memory_state_store),
+        _phantom: Default::default(),
+    };
+
+    let verify_iter_log = |key_range: TableKeyRange| {
+        let state_store = &verify_state_store;
+        let test_log_data = &test_log_data;
+        async move {
+            for start_epoch_idx in 0..epoch_count {
+                for end_epoch_idx in start_epoch_idx..epoch_count {
+                    let min_epoch = test_log_data[start_epoch_idx].0;
+                    let max_epoch = test_log_data[end_epoch_idx].0;
+                    let mut iter = state_store
+                        .iter_log(
+                            (min_epoch, max_epoch),
+                            key_range.clone(),
+                            ReadLogOptions { table_id },
+                        )
+                        .await
+                        .unwrap();
+                    while iter.try_next().await.unwrap().is_some() {}
+                }
+            }
+        }
+    };
+
+    verify_iter_log((Unbounded, Unbounded)).await;
+
+    if cfg!(madsim) {
+        // avoid timeout in madsim
+        return;
+    }
+
+    let keys = test_log_data
+        .iter()
+        .flat_map(|(_, logs)| logs.iter().map(|(key, _)| key.clone()))
+        .sorted()
+        .dedup()
+        .collect_vec();
+    let test_key_count = 5;
+    let step = keys.len() / test_key_count;
+
+    for i in 0..test_key_count {
+        let start_key_idx = i * step;
+        let start_key = keys[start_key_idx].clone();
+        let start_bound = Included(start_key);
+        for j in i..test_key_count {
+            let end_key_idx = j * step;
+            let end_key = keys[end_key_idx].clone();
+            let end_bound = if j % 2 == 0 {
+                Included(end_key)
+            } else {
+                Excluded(end_key)
+            };
+            verify_iter_log((start_bound.clone(), end_bound)).await;
+        }
+        verify_iter_log((start_bound, Unbounded)).await;
+    }
+}
diff --git a/src/storage/hummock_test/src/sync_point_tests.rs b/src/storage/hummock_test/src/sync_point_tests.rs
index 655852451d93b..3074bba6dd75f 100644
--- a/src/storage/hummock_test/src/sync_point_tests.rs
+++ b/src/storage/hummock_test/src/sync_point_tests.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use bytes::Bytes;
-use foyer::memory::CacheContext;
+use foyer::CacheContext;
 use risingwave_common::catalog::hummock::CompactionFilterFlag;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
diff --git a/src/storage/hummock_test/src/test_utils.rs b/src/storage/hummock_test/src/test_utils.rs
index 82b01bccaa12f..7c6be2eb041ca 100644
--- a/src/storage/hummock_test/src/test_utils.rs
+++ b/src/storage/hummock_test/src/test_utils.rs
@@ -81,7 +81,7 @@ pub async fn prepare_first_valid_version(
     };
 
     (
-        PinnedVersion::new(hummock_version, unbounded_channel().0),
+        PinnedVersion::new(*hummock_version, unbounded_channel().0),
         tx,
         rx,
     )
@@ -117,6 +117,7 @@ impl<S: LocalStateStore> TestIngestBatch for S {
 #[cfg(test)]
 #[async_trait::async_trait]
 pub(crate) trait HummockStateStoreTestTrait: StateStore {
+    #[allow(dead_code)]
     fn get_pinned_version(&self) -> PinnedVersion;
     async fn seal_and_sync_epoch(&self, epoch: u64) -> StorageResult<SyncResult> {
         self.seal_epoch(epoch, true);
@@ -134,7 +135,7 @@ impl HummockStateStoreTestTrait for HummockStorage {
 pub async fn with_hummock_storage_v2(
     table_id: TableId,
 ) -> (HummockStorage, Arc<MockHummockMetaClient>) {
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     let hummock_options = Arc::new(default_opts_for_test());
     let (env, hummock_manager_ref, _cluster_manager_ref, worker_node) =
         setup_compute_env(8080).await;
@@ -262,7 +263,7 @@ impl HummockTestEnv {
 }
 
 pub async fn prepare_hummock_test_env() -> HummockTestEnv {
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     let hummock_options = Arc::new(default_opts_for_test());
     let (env, hummock_manager_ref, _cluster_manager_ref, worker_node) =
         setup_compute_env(8080).await;
diff --git a/src/storage/hummock_test/src/vacuum_tests.rs b/src/storage/hummock_test/src/vacuum_tests.rs
index d35c6fe48c3e7..5c6dbdc556d5a 100644
--- a/src/storage/hummock_test/src/vacuum_tests.rs
+++ b/src/storage/hummock_test/src/vacuum_tests.rs
@@ -31,7 +31,7 @@ use risingwave_storage::hummock::vacuum::Vacuum;
 
 #[tokio::test]
 async fn test_vacuum() {
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     // Put some SSTs to object store
     let object_ids = (1..10).collect_vec();
     let mut sstables = vec![];
@@ -70,7 +70,7 @@ async fn test_vacuum() {
 async fn test_full_scan() {
     let (_env, hummock_manager_ref, _cluster_manager_ref, worker_node) =
         setup_compute_env(8080).await;
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     let _mock_hummock_meta_client = Arc::new(MockHummockMetaClient::new(
         hummock_manager_ref,
         worker_node.id,
diff --git a/src/storage/hummock_trace/src/lib.rs b/src/storage/hummock_trace/src/lib.rs
index 0de5680908254..64417832206e0 100644
--- a/src/storage/hummock_trace/src/lib.rs
+++ b/src/storage/hummock_trace/src/lib.rs
@@ -14,7 +14,6 @@
 
 #![feature(lazy_cell)]
 #![feature(cursor_remaining)]
-#![feature(bound_map)]
 #![feature(trait_alias)]
 #![feature(coroutines)]
 
diff --git a/src/storage/hummock_trace/src/opts.rs b/src/storage/hummock_trace/src/opts.rs
index a0073d620a5bc..07c87f4688042 100644
--- a/src/storage/hummock_trace/src/opts.rs
+++ b/src/storage/hummock_trace/src/opts.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use bincode::{Decode, Encode};
-use foyer::memory::CacheContext;
+use foyer::CacheContext;
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::cache::CachePriority;
 use risingwave_common::catalog::{TableId, TableOption};
@@ -33,7 +33,6 @@ pub struct TracedPrefetchOptions {
 pub enum TracedCachePolicy {
     Disable,
     Fill(TracedCachePriority),
-    FileFileCache,
     NotFill,
 }
 
@@ -65,7 +64,7 @@ impl From<CacheContext> for TracedCachePriority {
     fn from(value: CacheContext) -> Self {
         match value {
             CacheContext::Default => Self::High,
-            CacheContext::LruPriorityLow => Self::Low,
+            CacheContext::LowPriority => Self::Low,
         }
     }
 }
@@ -74,7 +73,7 @@ impl From<TracedCachePriority> for CacheContext {
     fn from(value: TracedCachePriority) -> Self {
         match value {
             TracedCachePriority::High => Self::Default,
-            TracedCachePriority::Low => Self::LruPriorityLow,
+            TracedCachePriority::Low => Self::LowPriority,
         }
     }
 }
diff --git a/src/storage/hummock_trace/src/replay/worker.rs b/src/storage/hummock_trace/src/replay/worker.rs
index 29e814c3f9ade..2c2cef690b19d 100644
--- a/src/storage/hummock_trace/src/replay/worker.rs
+++ b/src/storage/hummock_trace/src/replay/worker.rs
@@ -530,13 +530,10 @@ mod tests {
 
     use bytes::Bytes;
     use mockall::predicate;
-    use tokio::sync::mpsc::unbounded_channel;
 
     use super::*;
     use crate::replay::{MockGlobalReplayInterface, MockLocalReplayInterface};
-    use crate::{
-        MockReplayIterStream, StorageType, TracedBytes, TracedNewLocalOptions, TracedReadOptions,
-    };
+    use crate::{MockReplayIterStream, TracedBytes, TracedReadOptions};
 
     #[tokio::test]
     async fn test_handle_record() {
diff --git a/src/storage/hummock_trace/src/write.rs b/src/storage/hummock_trace/src/write.rs
index 528cdac5f659d..5cf10b39378f2 100644
--- a/src/storage/hummock_trace/src/write.rs
+++ b/src/storage/hummock_trace/src/write.rs
@@ -103,7 +103,7 @@ impl<W: Write, S: Serializer<W>> Drop for TraceWriterImpl<W, S> {
 mod test {
     use std::io::Cursor;
 
-    use bincode::{config, decode_from_slice, encode_to_vec};
+    use bincode::{decode_from_slice, encode_to_vec};
     use byteorder::{BigEndian, ReadBytesExt};
     use bytes::Bytes;
 
diff --git a/src/storage/src/hummock/backup_reader.rs b/src/storage/src/hummock/backup_reader.rs
index cd0a231420c1e..3fa0748b21737 100644
--- a/src/storage/src/hummock/backup_reader.rs
+++ b/src/storage/src/hummock/backup_reader.rs
@@ -24,7 +24,8 @@ use futures::FutureExt;
 use risingwave_backup::error::BackupError;
 use risingwave_backup::meta_snapshot::{MetaSnapshot, Metadata};
 use risingwave_backup::storage::{MetaSnapshotStorage, ObjectStoreMetaSnapshotStorage};
-use risingwave_backup::{meta_snapshot_v1, MetaSnapshotId};
+use risingwave_backup::{meta_snapshot_v1, meta_snapshot_v2, MetaSnapshotId};
+use risingwave_common::catalog::TableId;
 use risingwave_common::config::ObjectStoreConfig;
 use risingwave_common::system_param::local_manager::SystemParamsReaderRef;
 use risingwave_common::system_param::reader::SystemParamsRead;
@@ -52,7 +53,7 @@ async fn create_snapshot_store(
             &config.0,
             Arc::new(ObjectStoreMetrics::unused()),
             "Meta Backup",
-            object_store_config.clone(),
+            Arc::new(object_store_config.clone()),
         )
         .await,
     );
@@ -182,24 +183,31 @@ impl BackupReader {
     /// Otherwise, reading the version may encounter object store error, due to SST absence.
     pub async fn try_get_hummock_version(
         self: &BackupReaderRef,
+        table_id: TableId,
         epoch: u64,
     ) -> StorageResult<Option<PinnedVersion>> {
         // Use the same store throughout the call.
         let current_store = self.store.load_full();
         // 1. check manifest to locate snapshot, if any.
-        let snapshot_id = current_store
+        let Some(snapshot_metadata) = current_store
             .0
             .manifest()
             .snapshot_metadata
             .iter()
-            .find(|v| epoch >= v.safe_epoch && epoch <= v.max_committed_epoch)
-            .map(|s| s.id);
-        let snapshot_id = match snapshot_id {
-            None => {
-                return Ok(None);
-            }
-            Some(s) => s,
+            .find(|v| {
+                if v.state_table_info.is_empty() {
+                    return epoch >= v.safe_epoch && epoch <= v.max_committed_epoch;
+                }
+                if let Some(m) = v.state_table_info.get(&table_id) {
+                    return epoch >= m.safe_epoch && epoch <= m.committed_epoch;
+                }
+                false
+            })
+            .cloned()
+        else {
+            return Ok(None);
         };
+        let snapshot_id = snapshot_metadata.id;
         // 2. load hummock version of chosen snapshot.
         let future = {
             let mut req_guard = self.inflight_request.lock();
@@ -211,16 +219,28 @@ impl BackupReader {
             } else {
                 let this = self.clone();
                 let f = async move {
-                    // TODO: change to v2
-                    let snapshot: meta_snapshot_v1::MetaSnapshotV1 =
-                        current_store.0.get(snapshot_id).await.map_err(|e| {
-                            format!(
-                                "failed to get meta snapshot {}: {}",
-                                snapshot_id,
-                                e.as_report()
-                            )
-                        })?;
-                    let version_holder = build_version_holder(snapshot);
+                    let to_not_found_error = |e: BackupError| {
+                        format!(
+                            "failed to get meta snapshot {}: {}",
+                            snapshot_id,
+                            e.as_report()
+                        )
+                    };
+                    let version_holder = if snapshot_metadata.format_version < 2 {
+                        let snapshot: meta_snapshot_v1::MetaSnapshotV1 = current_store
+                            .0
+                            .get(snapshot_id)
+                            .await
+                            .map_err(to_not_found_error)?;
+                        build_version_holder(snapshot)
+                    } else {
+                        let snapshot: meta_snapshot_v2::MetaSnapshotV2 = current_store
+                            .0
+                            .get(snapshot_id)
+                            .await
+                            .map_err(to_not_found_error)?;
+                        build_version_holder(snapshot)
+                    };
                     let version_clone = version_holder.0.clone();
                     this.versions.write().insert(snapshot_id, version_holder);
                     Ok(version_clone)
diff --git a/src/storage/src/hummock/block_cache.rs b/src/storage/src/hummock/block_cache.rs
index 88f01bf25e22c..c33c21272df7e 100644
--- a/src/storage/src/hummock/block_cache.rs
+++ b/src/storage/src/hummock/block_cache.rs
@@ -15,24 +15,17 @@
 use std::ops::Deref;
 use std::sync::Arc;
 
-use ahash::RandomState;
 use await_tree::InstrumentAwait;
-use foyer::memory::{
-    Cache, CacheContext, CacheEntry, Entry, EntryState, LfuCacheConfig, LruCacheConfig,
-    S3FifoCacheConfig,
-};
-use futures::Future;
+use foyer::{FetchState, HybridCacheEntry, HybridFetch};
 use risingwave_common::config::EvictionConfig;
-use risingwave_hummock_sdk::HummockSstableObjectId;
 
-use super::{Block, BlockCacheEventListener, HummockResult};
+use super::{Block, HummockResult, SstableBlockIndex};
 use crate::hummock::HummockError;
 
-type CachedBlockEntry =
-    CacheEntry<(HummockSstableObjectId, u64), Box<Block>, BlockCacheEventListener>;
+type HybridCachedBlockEntry = HybridCacheEntry<SstableBlockIndex, Box<Block>>;
 
 enum BlockEntry {
-    Cache(#[allow(dead_code)] CachedBlockEntry),
+    HybridCache(#[allow(dead_code)] HybridCachedBlockEntry),
     Owned(#[allow(dead_code)] Box<Block>),
     RefEntry(#[allow(dead_code)] Arc<Block>),
 }
@@ -59,10 +52,10 @@ impl BlockHolder {
         }
     }
 
-    pub fn from_cached_block(entry: CachedBlockEntry) -> Self {
-        let ptr = entry.deref().as_ref() as *const _;
+    pub fn from_hybrid_cache_entry(entry: HybridCachedBlockEntry) -> Self {
+        let ptr = entry.value().as_ref() as *const _;
         Self {
-            _handle: BlockEntry::Cache(entry),
+            _handle: BlockEntry::HybridCache(entry),
             block: ptr,
         }
     }
@@ -84,17 +77,11 @@ pub struct BlockCacheConfig {
     pub capacity: usize,
     pub shard_num: usize,
     pub eviction: EvictionConfig,
-    pub listener: BlockCacheEventListener,
-}
-
-#[derive(Clone)]
-pub struct BlockCache {
-    inner: Cache<(HummockSstableObjectId, u64), Box<Block>, BlockCacheEventListener>,
 }
 
 pub enum BlockResponse {
     Block(BlockHolder),
-    Entry(Entry<(HummockSstableObjectId, u64), Box<Block>, HummockError, BlockCacheEventListener>),
+    Entry(HybridFetch<SstableBlockIndex, Box<Block>>),
 }
 
 impl BlockResponse {
@@ -104,121 +91,20 @@ impl BlockResponse {
             BlockResponse::Entry(entry) => entry,
         };
         match entry.state() {
-            EntryState::Hit => entry.await.map(BlockHolder::from_cached_block),
-            EntryState::Wait => entry
+            FetchState::Hit => entry
+                .await
+                .map(BlockHolder::from_hybrid_cache_entry)
+                .map_err(HummockError::foyer_error),
+            FetchState::Wait => entry
                 .verbose_instrument_await("wait_pending_fetch_block")
                 .await
-                .map(BlockHolder::from_cached_block),
-            EntryState::Miss => entry
+                .map(BlockHolder::from_hybrid_cache_entry)
+                .map_err(HummockError::foyer_error),
+            FetchState::Miss => entry
                 .verbose_instrument_await("fetch_block")
                 .await
-                .map(BlockHolder::from_cached_block),
+                .map(BlockHolder::from_hybrid_cache_entry)
+                .map_err(HummockError::foyer_error),
         }
     }
 }
-
-impl BlockCache {
-    pub fn new(config: BlockCacheConfig) -> Self {
-        assert!(
-            config.capacity > 0,
-            "Block cache capacity must be positive."
-        );
-
-        let capacity = config.capacity;
-        let shards = config.shard_num;
-        let object_pool_capacity = shards * 1024;
-        let hash_builder = RandomState::default();
-        let event_listener = config.listener;
-
-        let inner = match config.eviction {
-            EvictionConfig::Lru(eviction_config) => Cache::lru(LruCacheConfig {
-                capacity,
-                shards,
-                eviction_config,
-                object_pool_capacity,
-                hash_builder,
-                event_listener,
-            }),
-            EvictionConfig::Lfu(eviction_config) => Cache::lfu(LfuCacheConfig {
-                capacity,
-                shards,
-                eviction_config,
-                object_pool_capacity,
-                hash_builder,
-                event_listener,
-            }),
-            EvictionConfig::S3Fifo(eviction_config) => Cache::s3fifo(S3FifoCacheConfig {
-                capacity,
-                shards,
-                eviction_config,
-                object_pool_capacity,
-                hash_builder,
-                event_listener,
-            }),
-        };
-
-        Self { inner }
-    }
-
-    pub fn get(&self, object_id: HummockSstableObjectId, block_idx: u64) -> Option<BlockHolder> {
-        self.inner
-            .get(&(object_id, block_idx))
-            .map(BlockHolder::from_cached_block)
-    }
-
-    pub fn exists_block(&self, sst_id: HummockSstableObjectId, block_idx: u64) -> bool {
-        // TODO(MrCroxx): optimize me
-        self.get(sst_id, block_idx).is_some()
-    }
-
-    pub fn insert(
-        &self,
-        object_id: HummockSstableObjectId,
-        block_idx: u64,
-        block: Box<Block>,
-        context: CacheContext,
-    ) -> BlockHolder {
-        let charge = block.capacity();
-        BlockHolder::from_cached_block(self.inner.insert_with_context(
-            (object_id, block_idx),
-            block,
-            charge,
-            context,
-        ))
-    }
-
-    pub fn get_or_insert_with<F, Fut>(
-        &self,
-        object_id: HummockSstableObjectId,
-        block_idx: u64,
-        context: CacheContext,
-        mut fetch_block: F,
-    ) -> BlockResponse
-    where
-        F: FnMut() -> Fut,
-        Fut: Future<Output = HummockResult<Box<Block>>> + Send + 'static,
-    {
-        let key = (object_id, block_idx);
-
-        let entry = self.inner.entry(key, || {
-            let f = fetch_block();
-            async move {
-                let block = f.await?;
-                let len = block.capacity();
-                Ok::<_, HummockError>((block, len, context))
-            }
-        });
-
-        BlockResponse::Entry(entry)
-    }
-
-    pub fn size(&self) -> usize {
-        self.inner.usage()
-    }
-
-    #[cfg(any(test, feature = "test"))]
-    pub fn clear(&self) {
-        // This is only a method for test. Therefore it should be safe to call the unsafe method.
-        self.inner.clear();
-    }
-}
diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs
index 9594d7295a03e..63b59366195f0 100644
--- a/src/storage/src/hummock/compactor/compaction_utils.rs
+++ b/src/storage/src/hummock/compactor/compaction_utils.rs
@@ -26,8 +26,9 @@ use risingwave_hummock_sdk::key_range::KeyRange;
 use risingwave_hummock_sdk::prost_key_range::KeyRangeExt;
 use risingwave_hummock_sdk::table_stats::TableStatsMap;
 use risingwave_hummock_sdk::{can_concat, EpochWithGap, KeyComparator};
+use risingwave_pb::hummock::compact_task::TaskType;
 use risingwave_pb::hummock::{
-    compact_task, CompactTask, KeyRange as KeyRange_vec, LevelType, SstableInfo, TableSchema,
+    compact_task, BloomFilterType, CompactTask, LevelType, PbKeyRange, SstableInfo, TableSchema,
 };
 use tokio::time::Instant;
 
@@ -172,24 +173,20 @@ pub fn build_multi_compaction_filter(compact_task: &CompactTask) -> MultiCompact
     multi_filter
 }
 
-const MAX_FILE_COUNT: usize = 32;
-
 fn generate_splits_fast(
     sstable_infos: &Vec<SstableInfo>,
     compaction_size: u64,
-    context: CompactorContext,
-) -> HummockResult<Vec<KeyRange_vec>> {
+    context: &CompactorContext,
+    max_sub_compaction: u32,
+) -> Vec<PbKeyRange> {
     let worker_num = context.compaction_executor.worker_num();
     let parallel_compact_size = (context.storage_opts.parallel_compact_size_mb as u64) << 20;
 
-    let parallelism = (compaction_size + parallel_compact_size - 1) / parallel_compact_size;
-
-    let parallelism = std::cmp::min(
+    let parallelism = calculate_task_parallelism_impl(
         worker_num,
-        std::cmp::min(
-            parallelism as usize,
-            context.storage_opts.max_sub_compaction as usize,
-        ),
+        parallel_compact_size,
+        compaction_size,
+        max_sub_compaction,
     );
     let mut indexes = vec![];
     for sst in sstable_infos {
@@ -212,32 +209,41 @@ fn generate_splits_fast(
     indexes.sort_by(|a, b| KeyComparator::compare_encoded_full_key(a.as_ref(), b.as_ref()));
     indexes.dedup();
     if indexes.len() <= parallelism {
-        return Ok(vec![]);
+        return vec![];
     }
+
     let mut splits = vec![];
-    splits.push(KeyRange_vec::new(vec![], vec![]));
+    splits.push(PbKeyRange::new(vec![], vec![]));
     let parallel_key_count = indexes.len() / parallelism;
     let mut last_split_key_count = 0;
     for key in indexes {
         if last_split_key_count >= parallel_key_count {
             splits.last_mut().unwrap().right.clone_from(&key);
-            splits.push(KeyRange_vec::new(key.clone(), vec![]));
+            splits.push(PbKeyRange::new(key.clone(), vec![]));
             last_split_key_count = 0;
         }
         last_split_key_count += 1;
     }
-    Ok(splits)
+
+    splits
 }
 
 pub async fn generate_splits(
     sstable_infos: &Vec<SstableInfo>,
     compaction_size: u64,
-    context: CompactorContext,
-) -> HummockResult<Vec<KeyRange_vec>> {
+    context: &CompactorContext,
+    max_sub_compaction: u32,
+) -> HummockResult<Vec<PbKeyRange>> {
+    const MAX_FILE_COUNT: usize = 32;
     let parallel_compact_size = (context.storage_opts.parallel_compact_size_mb as u64) << 20;
     if compaction_size > parallel_compact_size {
         if sstable_infos.len() > MAX_FILE_COUNT {
-            return generate_splits_fast(sstable_infos, compaction_size, context);
+            return Ok(generate_splits_fast(
+                sstable_infos,
+                compaction_size,
+                context,
+                max_sub_compaction,
+            ));
         }
         let mut indexes = vec![];
         // preload the meta and get the smallest key to split sub_compaction
@@ -265,20 +271,17 @@ pub async fn generate_splits(
         // sort by key, as for every data block has the same size;
         indexes.sort_by(|a, b| KeyComparator::compare_encoded_full_key(a.1.as_ref(), b.1.as_ref()));
         let mut splits = vec![];
-        splits.push(KeyRange_vec::new(vec![], vec![]));
+        splits.push(PbKeyRange::new(vec![], vec![]));
 
-        let worker_num = context.compaction_executor.worker_num();
-
-        let parallelism = std::cmp::min(
-            worker_num as u64,
-            std::cmp::min(
-                indexes.len() as u64,
-                context.storage_opts.max_sub_compaction as u64,
-            ),
+        let parallelism = calculate_task_parallelism_impl(
+            context.compaction_executor.worker_num(),
+            parallel_compact_size,
+            compaction_size,
+            max_sub_compaction,
         );
+
         let sub_compaction_data_size =
-            std::cmp::max(compaction_size / parallelism, parallel_compact_size);
-        let parallelism = compaction_size / sub_compaction_data_size;
+            std::cmp::max(compaction_size / parallelism as u64, parallel_compact_size);
 
         if parallelism > 1 {
             let mut last_buffer_size = 0;
@@ -290,7 +293,7 @@ pub async fn generate_splits(
                     && remaining_size > parallel_compact_size
                 {
                     splits.last_mut().unwrap().right.clone_from(&key);
-                    splits.push(KeyRange_vec::new(key.clone(), vec![]));
+                    splits.push(PbKeyRange::new(key.clone(), vec![]));
                     last_buffer_size = data_size;
                 } else {
                     last_buffer_size += data_size;
@@ -489,3 +492,191 @@ async fn check_result<
     }
     Ok(true)
 }
+
+pub fn optimize_by_copy_block(compact_task: &CompactTask, context: &CompactorContext) -> bool {
+    let sstable_infos = compact_task
+        .input_ssts
+        .iter()
+        .flat_map(|level| level.table_infos.iter())
+        .filter(|table_info| {
+            let table_ids = &table_info.table_ids;
+            table_ids
+                .iter()
+                .any(|table_id| compact_task.existing_table_ids.contains(table_id))
+        })
+        .cloned()
+        .collect_vec();
+    let compaction_size = sstable_infos
+        .iter()
+        .map(|table_info| table_info.file_size)
+        .sum::<u64>();
+
+    let all_ssts_are_blocked_filter = sstable_infos
+        .iter()
+        .all(|table_info| table_info.bloom_filter_kind() == BloomFilterType::Blocked);
+
+    let delete_key_count = sstable_infos
+        .iter()
+        .map(|table_info| table_info.stale_key_count + table_info.range_tombstone_count)
+        .sum::<u64>();
+    let total_key_count = sstable_infos
+        .iter()
+        .map(|table_info| table_info.total_key_count)
+        .sum::<u64>();
+
+    let has_tombstone = compact_task
+        .input_ssts
+        .iter()
+        .flat_map(|level| level.table_infos.iter())
+        .any(|sst| sst.range_tombstone_count > 0);
+    let has_ttl = compact_task
+        .table_options
+        .iter()
+        .any(|(_, table_option)| table_option.retention_seconds.is_some_and(|ttl| ttl > 0));
+
+    let compact_table_ids: HashSet<u32> = HashSet::from_iter(
+        compact_task
+            .input_ssts
+            .iter()
+            .flat_map(|level| level.table_infos.iter())
+            .flat_map(|sst| sst.table_ids.clone()),
+    );
+    let single_table = compact_table_ids.len() == 1;
+
+    context.storage_opts.enable_fast_compaction
+        && all_ssts_are_blocked_filter
+        && !has_tombstone
+        && !has_ttl
+        && single_table
+        && compact_task.target_level > 0
+        && compact_task.input_ssts.len() == 2
+        && compaction_size < context.storage_opts.compactor_fast_max_compact_task_size
+        && delete_key_count * 100
+            < context.storage_opts.compactor_fast_max_compact_delete_ratio as u64 * total_key_count
+        && compact_task.task_type() == TaskType::Dynamic
+}
+
+pub async fn generate_splits_for_task(
+    compact_task: &mut CompactTask,
+    context: &CompactorContext,
+    optimize_by_copy_block: bool,
+) -> HummockResult<()> {
+    let sstable_infos = compact_task
+        .input_ssts
+        .iter()
+        .flat_map(|level| level.table_infos.iter())
+        .filter(|table_info| {
+            let table_ids = &table_info.table_ids;
+            table_ids
+                .iter()
+                .any(|table_id| compact_task.existing_table_ids.contains(table_id))
+        })
+        .cloned()
+        .collect_vec();
+    let compaction_size = sstable_infos
+        .iter()
+        .map(|table_info| table_info.file_size)
+        .sum::<u64>();
+
+    if !optimize_by_copy_block {
+        let splits = generate_splits(
+            &sstable_infos,
+            compaction_size,
+            context,
+            compact_task.get_max_sub_compaction(),
+        )
+        .await?;
+        if !splits.is_empty() {
+            compact_task.splits = splits;
+        }
+        return Ok(());
+    }
+
+    Ok(())
+}
+
+pub fn metrics_report_for_task(compact_task: &CompactTask, context: &CompactorContext) {
+    let group_label = compact_task.compaction_group_id.to_string();
+    let cur_level_label = compact_task.input_ssts[0].level_idx.to_string();
+    let select_table_infos = compact_task
+        .input_ssts
+        .iter()
+        .filter(|level| level.level_idx != compact_task.target_level)
+        .flat_map(|level| level.table_infos.iter())
+        .collect_vec();
+    let target_table_infos = compact_task
+        .input_ssts
+        .iter()
+        .filter(|level| level.level_idx == compact_task.target_level)
+        .flat_map(|level| level.table_infos.iter())
+        .collect_vec();
+    let select_size = select_table_infos
+        .iter()
+        .map(|table| table.file_size)
+        .sum::<u64>();
+    context
+        .compactor_metrics
+        .compact_read_current_level
+        .with_label_values(&[&group_label, &cur_level_label])
+        .inc_by(select_size);
+    context
+        .compactor_metrics
+        .compact_read_sstn_current_level
+        .with_label_values(&[&group_label, &cur_level_label])
+        .inc_by(select_table_infos.len() as u64);
+
+    let target_level_read_bytes = target_table_infos.iter().map(|t| t.file_size).sum::<u64>();
+    let next_level_label = compact_task.target_level.to_string();
+    context
+        .compactor_metrics
+        .compact_read_next_level
+        .with_label_values(&[&group_label, next_level_label.as_str()])
+        .inc_by(target_level_read_bytes);
+    context
+        .compactor_metrics
+        .compact_read_sstn_next_level
+        .with_label_values(&[&group_label, next_level_label.as_str()])
+        .inc_by(target_table_infos.len() as u64);
+}
+
+pub fn calculate_task_parallelism(compact_task: &CompactTask, context: &CompactorContext) -> usize {
+    let optimize_by_copy_block = optimize_by_copy_block(compact_task, context);
+
+    if optimize_by_copy_block {
+        return 1;
+    }
+
+    let sstable_infos = compact_task
+        .input_ssts
+        .iter()
+        .flat_map(|level| level.table_infos.iter())
+        .filter(|table_info| {
+            let table_ids = &table_info.table_ids;
+            table_ids
+                .iter()
+                .any(|table_id| compact_task.existing_table_ids.contains(table_id))
+        })
+        .cloned()
+        .collect_vec();
+    let compaction_size = sstable_infos
+        .iter()
+        .map(|table_info| table_info.file_size)
+        .sum::<u64>();
+    let parallel_compact_size = (context.storage_opts.parallel_compact_size_mb as u64) << 20;
+    calculate_task_parallelism_impl(
+        context.compaction_executor.worker_num(),
+        parallel_compact_size,
+        compaction_size,
+        compact_task.get_max_sub_compaction(),
+    )
+}
+
+pub fn calculate_task_parallelism_impl(
+    worker_num: usize,
+    parallel_compact_size: u64,
+    compaction_size: u64,
+    max_sub_compaction: u32,
+) -> usize {
+    let parallelism = (compaction_size + parallel_compact_size - 1) / parallel_compact_size;
+    worker_num.min(parallelism.min(max_sub_compaction as u64) as usize)
+}
diff --git a/src/storage/src/hummock/compactor/compactor_runner.rs b/src/storage/src/hummock/compactor/compactor_runner.rs
index 46ccaf0ebb94d..41a55518158de 100644
--- a/src/storage/src/hummock/compactor/compactor_runner.rs
+++ b/src/storage/src/hummock/compactor/compactor_runner.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::collections::{BinaryHeap, HashMap, HashSet};
-use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
 use await_tree::InstrumentAwait;
@@ -30,8 +29,8 @@ use risingwave_hummock_sdk::table_stats::{add_table_stats_map, TableStats, Table
 use risingwave_hummock_sdk::{
     can_concat, compact_task_output_to_string, HummockSstableObjectId, KeyComparator,
 };
-use risingwave_pb::hummock::compact_task::{TaskStatus, TaskType};
-use risingwave_pb::hummock::{BloomFilterType, CompactTask, LevelType, SstableInfo};
+use risingwave_pb::hummock::compact_task::TaskStatus;
+use risingwave_pb::hummock::{CompactTask, LevelType, SstableInfo};
 use thiserror_ext::AsReport;
 use tokio::sync::oneshot::Receiver;
 
@@ -40,7 +39,8 @@ use super::task_progress::TaskProgress;
 use super::{CompactionStatistics, TaskConfig};
 use crate::filter_key_extractor::{FilterKeyExtractorImpl, FilterKeyExtractorManager};
 use crate::hummock::compactor::compaction_utils::{
-    build_multi_compaction_filter, estimate_task_output_capacity, generate_splits,
+    build_multi_compaction_filter, estimate_task_output_capacity, generate_splits_for_task,
+    metrics_report_for_task, optimize_by_copy_block,
 };
 use crate::hummock::compactor::iterator::ConcatSstableIterator;
 use crate::hummock::compactor::task_progress::TaskProgressGuard;
@@ -276,7 +276,9 @@ pub fn partition_overlapping_sstable_infos(
                 &prev_group.max_right_bound,
                 &sst.key_range.as_ref().unwrap().left,
             ) {
-                prev_group.max_right_bound = sst.key_range.as_ref().unwrap().right.clone();
+                prev_group
+                    .max_right_bound
+                    .clone_from(&sst.key_range.as_ref().unwrap().right);
                 prev_group.ssts.push(sst);
                 continue;
             }
@@ -304,46 +306,7 @@ pub async fn compact(
 ) {
     let context = compactor_context.clone();
     let group_label = compact_task.compaction_group_id.to_string();
-    let cur_level_label = compact_task.input_ssts[0].level_idx.to_string();
-    let select_table_infos = compact_task
-        .input_ssts
-        .iter()
-        .filter(|level| level.level_idx != compact_task.target_level)
-        .flat_map(|level| level.table_infos.iter())
-        .collect_vec();
-    let target_table_infos = compact_task
-        .input_ssts
-        .iter()
-        .filter(|level| level.level_idx == compact_task.target_level)
-        .flat_map(|level| level.table_infos.iter())
-        .collect_vec();
-    let select_size = select_table_infos
-        .iter()
-        .map(|table| table.file_size)
-        .sum::<u64>();
-    context
-        .compactor_metrics
-        .compact_read_current_level
-        .with_label_values(&[&group_label, &cur_level_label])
-        .inc_by(select_size);
-    context
-        .compactor_metrics
-        .compact_read_sstn_current_level
-        .with_label_values(&[&group_label, &cur_level_label])
-        .inc_by(select_table_infos.len() as u64);
-
-    let target_level_read_bytes = target_table_infos.iter().map(|t| t.file_size).sum::<u64>();
-    let next_level_label = compact_task.target_level.to_string();
-    context
-        .compactor_metrics
-        .compact_read_next_level
-        .with_label_values(&[&group_label, next_level_label.as_str()])
-        .inc_by(target_level_read_bytes);
-    context
-        .compactor_metrics
-        .compact_read_sstn_next_level
-        .with_label_values(&[&group_label, next_level_label.as_str()])
-        .inc_by(target_table_infos.len() as u64);
+    metrics_report_for_task(&compact_task, &context);
 
     let timer = context
         .compactor_metrics
@@ -356,151 +319,52 @@ pub async fn compact(
 
     let multi_filter = build_multi_compaction_filter(&compact_task);
 
-    let mut compact_table_ids = compact_task
-        .input_ssts
-        .iter()
-        .flat_map(|level| level.table_infos.iter())
-        .flat_map(|sst| sst.table_ids.clone())
-        .collect_vec();
-    compact_table_ids.sort();
-    compact_table_ids.dedup();
-    let single_table = compact_table_ids.len() == 1;
-
     let existing_table_ids: HashSet<u32> =
         HashSet::from_iter(compact_task.existing_table_ids.clone());
     let compact_table_ids = HashSet::from_iter(
-        compact_table_ids
-            .into_iter()
+        compact_task
+            .input_ssts
+            .iter()
+            .flat_map(|level| level.table_infos.iter())
+            .flat_map(|sst| sst.table_ids.clone())
             .filter(|table_id| existing_table_ids.contains(table_id)),
     );
-    let multi_filter_key_extractor = match filter_key_extractor_manager
-        .acquire(compact_table_ids.clone())
-        .await
+
+    let multi_filter_key_extractor = match build_filter_key_extractor(
+        &compact_task,
+        filter_key_extractor_manager,
+        &compact_table_ids,
+    )
+    .await
     {
-        Err(e) => {
-            tracing::error!(error = %e.as_report(), "Failed to fetch filter key extractor tables [{:?}], it may caused by some RPC error", compact_task.existing_table_ids);
+        Some(multi_filter_key_extractor) => multi_filter_key_extractor,
+        None => {
             let task_status = TaskStatus::ExecuteFailed;
             return (
                 compact_done(compact_task, context.clone(), vec![], task_status),
                 None,
             );
         }
-        Ok(extractor) => extractor,
     };
 
-    if let FilterKeyExtractorImpl::Multi(multi) = &multi_filter_key_extractor {
-        let found_tables = multi.get_existing_table_ids();
-        let removed_tables = compact_table_ids
-            .iter()
-            .filter(|table_id| !found_tables.contains(table_id))
-            .collect_vec();
-        if !removed_tables.is_empty() {
-            tracing::error!("Failed to fetch filter key extractor tables [{:?}. [{:?}] may be removed by meta-service. ", compact_table_ids, removed_tables);
-            let task_status = TaskStatus::ExecuteFailed;
-            return (
-                compact_done(compact_task, context.clone(), vec![], task_status),
-                None,
-            );
-        }
-    }
-
-    let multi_filter_key_extractor = Arc::new(multi_filter_key_extractor);
-    let has_tombstone = compact_task
-        .input_ssts
-        .iter()
-        .flat_map(|level| level.table_infos.iter())
-        .any(|sst| sst.range_tombstone_count > 0);
-    let has_ttl = compact_task
-        .table_options
-        .iter()
-        .any(|(_, table_option)| table_option.retention_seconds.is_some_and(|ttl| ttl > 0));
     let mut task_status = TaskStatus::Success;
-    // skip sst related to non-existent able_id to reduce io
-    let sstable_infos = compact_task
-        .input_ssts
-        .iter()
-        .flat_map(|level| level.table_infos.iter())
-        .filter(|table_info| {
-            let table_ids = &table_info.table_ids;
-            table_ids
-                .iter()
-                .any(|table_id| existing_table_ids.contains(table_id))
-        })
-        .cloned()
-        .collect_vec();
-    let compaction_size = sstable_infos
-        .iter()
-        .map(|table_info| table_info.file_size)
-        .sum::<u64>();
-    let all_ssts_are_blocked_filter = sstable_infos
-        .iter()
-        .all(|table_info| table_info.bloom_filter_kind() == BloomFilterType::Blocked);
+    let optimize_by_copy_block = optimize_by_copy_block(&compact_task, &context);
 
-    let delete_key_count = sstable_infos
-        .iter()
-        .map(|table_info| table_info.stale_key_count + table_info.range_tombstone_count)
-        .sum::<u64>();
-    let total_key_count = sstable_infos
-        .iter()
-        .map(|table_info| table_info.total_key_count)
-        .sum::<u64>();
-    let optimize_by_copy_block = context.storage_opts.enable_fast_compaction
-        && all_ssts_are_blocked_filter
-        && !has_tombstone
-        && !has_ttl
-        && single_table
-        && compact_task.target_level > 0
-        && compact_task.input_ssts.len() == 2
-        && compaction_size < context.storage_opts.compactor_fast_max_compact_task_size
-        && delete_key_count * 100
-            < context.storage_opts.compactor_fast_max_compact_delete_ratio as u64 * total_key_count
-        && compact_task.task_type() == TaskType::Dynamic;
-
-    if !optimize_by_copy_block {
-        match generate_splits(&sstable_infos, compaction_size, context.clone()).await {
-            Ok(splits) => {
-                if !splits.is_empty() {
-                    compact_task.splits = splits;
-                }
-            }
-            Err(e) => {
-                tracing::warn!(error = %e.as_report(), "Failed to generate_splits");
-                task_status = TaskStatus::ExecuteFailed;
-                return (
-                    compact_done(compact_task, context.clone(), vec![], task_status),
-                    None,
-                );
-            }
-        }
-    }
-    let compact_task_statistics = statistics_compact_task(&compact_task);
-    // Number of splits (key ranges) is equal to number of compaction tasks
-    let parallelism = compact_task.splits.len();
-    assert_ne!(parallelism, 0, "splits cannot be empty");
-    if !context.acquire_task_quota(parallelism as u32) {
-        tracing::warn!(
-            "Not enough core parallelism to serve the task {} task_parallelism {} running_task_parallelism {} max_task_parallelism {}",
-            compact_task.task_id,
-            parallelism,
-            context.running_task_parallelism.load(Ordering::Relaxed),
-            context.max_task_parallelism.load(Ordering::Relaxed),
-        );
+    if let Err(e) =
+        generate_splits_for_task(&mut compact_task, &context, optimize_by_copy_block).await
+    {
+        tracing::warn!(error = %e.as_report(), "Failed to generate_splits");
+        task_status = TaskStatus::ExecuteFailed;
         return (
-            compact_done(
-                compact_task,
-                context.clone(),
-                vec![],
-                TaskStatus::NoAvailCpuResourceCanceled,
-            ),
+            compact_done(compact_task, context.clone(), vec![], task_status),
             None,
         );
     }
 
-    let _release_quota_guard =
-        scopeguard::guard((parallelism, context.clone()), |(parallelism, context)| {
-            context.release_task_quota(parallelism as u32);
-        });
-
+    let compact_task_statistics = statistics_compact_task(&compact_task);
+    // Number of splits (key ranges) is equal to number of compaction tasks
+    let parallelism = compact_task.splits.len();
+    assert_ne!(parallelism, 0, "splits cannot be empty");
     let mut output_ssts = Vec::with_capacity(parallelism);
     let mut compaction_futures = vec![];
     let mut abort_handles = vec![];
@@ -519,7 +383,6 @@ pub async fn compact(
             .object_store_recv_buffer_size
             .unwrap_or(6 * 1024 * 1024) as u64,
         capacity as u64,
-        context.sstable_store.store().support_streaming_upload(),
     ) * compact_task.splits.len() as u64;
 
     tracing::info!(
@@ -707,7 +570,7 @@ pub async fn compact(
 }
 
 /// Fills in the compact task and tries to report the task result to meta node.
-fn compact_done(
+pub(crate) fn compact_done(
     mut compact_task: CompactTask,
     context: CompactorContext,
     output_ssts: Vec<CompactOutput>,
@@ -921,12 +784,49 @@ where
         table_stats_drop.insert(last_table_id, std::mem::take(&mut last_table_stats));
     }
     iter.collect_local_statistic(&mut local_stats);
+    add_table_stats_map(
+        &mut table_stats_drop,
+        &local_stats.skipped_by_watermark_table_stats,
+    );
     local_stats.report_compactor(compactor_metrics.as_ref());
     compaction_statistics.delta_drop_stat = table_stats_drop;
 
     Ok(compaction_statistics)
 }
 
+async fn build_filter_key_extractor(
+    compact_task: &CompactTask,
+    filter_key_extractor_manager: FilterKeyExtractorManager,
+    compact_table_ids: &HashSet<u32>,
+) -> Option<Arc<FilterKeyExtractorImpl>> {
+    let multi_filter_key_extractor = match filter_key_extractor_manager
+        .acquire(compact_table_ids.clone())
+        .await
+    {
+        Err(e) => {
+            tracing::error!(error = %e.as_report(), "Failed to fetch filter key extractor tables [{:?}], it may caused by some RPC error", compact_task.existing_table_ids);
+            return None;
+        }
+        Ok(extractor) => extractor,
+    };
+
+    if let FilterKeyExtractorImpl::Multi(multi) = &multi_filter_key_extractor {
+        let found_tables = multi.get_existing_table_ids();
+        let removed_tables = compact_table_ids
+            .iter()
+            .filter(|table_id| !found_tables.contains(table_id))
+            .collect_vec();
+        if !removed_tables.is_empty() {
+            tracing::error!("Failed to fetch filter key extractor tables [{:?}. [{:?}] may be removed by meta-service. ", compact_table_ids, removed_tables);
+            return None;
+        }
+    }
+
+    let multi_filter_key_extractor = Arc::new(multi_filter_key_extractor);
+
+    Some(multi_filter_key_extractor)
+}
+
 #[cfg(test)]
 pub mod tests {
     use risingwave_hummock_sdk::can_concat;
@@ -941,7 +841,7 @@ pub mod tests {
     #[tokio::test]
     async fn test_partition_overlapping_level() {
         const TEST_KEYS_COUNT: usize = 10;
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let mut table_infos = vec![];
         for object_id in 0..10 {
             let start_index = object_id * TEST_KEYS_COUNT;
diff --git a/src/storage/src/hummock/compactor/context.rs b/src/storage/src/hummock/compactor/context.rs
index 4525dcdc773fb..e8295effc25ed 100644
--- a/src/storage/src/hummock/compactor/context.rs
+++ b/src/storage/src/hummock/compactor/context.rs
@@ -12,11 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::Arc;
 
-use more_asserts::assert_ge;
-
 use super::task_progress::TaskProgressManagerRef;
 use crate::hummock::compactor::CompactionExecutor;
 use crate::hummock::sstable_store::SstableStoreRef;
@@ -65,10 +62,6 @@ pub struct CompactorContext {
     pub task_progress_manager: TaskProgressManagerRef,
 
     pub await_tree_reg: Option<CompactionAwaitTreeRegRef>,
-
-    pub running_task_parallelism: Arc<AtomicU32>,
-
-    pub max_task_parallelism: Arc<AtomicU32>,
 }
 
 impl CompactorContext {
@@ -97,56 +90,6 @@ impl CompactorContext {
             memory_limiter: MemoryLimiter::unlimit(),
             task_progress_manager: Default::default(),
             await_tree_reg,
-            running_task_parallelism: Arc::new(AtomicU32::new(0)),
-            max_task_parallelism: Arc::new(AtomicU32::new(u32::MAX)),
-        }
-    }
-
-    pub fn acquire_task_quota(&self, parallelism: u32) -> bool {
-        let mut running_u32 = self.running_task_parallelism.load(Ordering::SeqCst);
-        let max_u32 = self.max_task_parallelism.load(Ordering::SeqCst);
-
-        while parallelism + running_u32 <= max_u32 {
-            match self.running_task_parallelism.compare_exchange(
-                running_u32,
-                running_u32 + parallelism,
-                Ordering::SeqCst,
-                Ordering::SeqCst,
-            ) {
-                Ok(_) => {
-                    return true;
-                }
-                Err(old_running_u32) => {
-                    running_u32 = old_running_u32;
-                }
-            }
-        }
-
-        false
-    }
-
-    pub fn release_task_quota(&self, parallelism: u32) {
-        let prev = self
-            .running_task_parallelism
-            .fetch_sub(parallelism, Ordering::SeqCst);
-
-        assert_ge!(
-            prev,
-            parallelism,
-            "running {} parallelism {}",
-            prev,
-            parallelism
-        );
-    }
-
-    pub fn get_free_quota(&self) -> u32 {
-        let running_u32 = self.running_task_parallelism.load(Ordering::SeqCst);
-        let max_u32 = self.max_task_parallelism.load(Ordering::SeqCst);
-
-        if max_u32 > running_u32 {
-            max_u32 - running_u32
-        } else {
-            0
         }
     }
 }
diff --git a/src/storage/src/hummock/compactor/fast_compactor_runner.rs b/src/storage/src/hummock/compactor/fast_compactor_runner.rs
index 7a30a6fd2ef65..81f0210d945c1 100644
--- a/src/storage/src/hummock/compactor/fast_compactor_runner.rs
+++ b/src/storage/src/hummock/compactor/fast_compactor_runner.rs
@@ -316,6 +316,9 @@ impl CompactorRunner {
             context.compactor_metrics.clone(),
             Some(task_progress.clone()),
             task_config.table_vnode_partition.clone(),
+            context
+                .storage_opts
+                .compactor_concurrent_uploading_sst_count,
         );
         assert_eq!(
             task.input_ssts.len(),
@@ -505,17 +508,19 @@ impl CompactorRunner {
         );
 
         let statistic = self.executor.take_statistics();
-        let outputs = self.executor.builder.finish().await?;
-        let ssts = Compactor::report_progress(
+        let output_ssts = self.executor.builder.finish().await?;
+        Compactor::report_progress(
             self.metrics.clone(),
             Some(self.executor.task_progress.clone()),
-            outputs,
+            &output_ssts,
             false,
-        )
-        .await?;
-        let sst_infos = ssts.iter().map(|sst| sst.sst_info.clone()).collect_vec();
+        );
+        let sst_infos = output_ssts
+            .iter()
+            .map(|sst| sst.sst_info.clone())
+            .collect_vec();
         assert!(can_concat(&sst_infos));
-        Ok((ssts, statistic))
+        Ok((output_ssts, statistic))
     }
 }
 
diff --git a/src/storage/src/hummock/compactor/iterator.rs b/src/storage/src/hummock/compactor/iterator.rs
index f89d70a756486..5431c51270efc 100644
--- a/src/storage/src/hummock/compactor/iterator.rs
+++ b/src/storage/src/hummock/compactor/iterator.rs
@@ -584,7 +584,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_concat_iterator() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let mut table_infos = vec![];
         for object_id in 0..3 {
             let start_index = object_id * TEST_KEYS_COUNT;
@@ -704,7 +704,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_concat_iterator_seek_idx() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let mut table_infos = vec![];
         for object_id in 0..3 {
             let start_index = object_id * TEST_KEYS_COUNT + TEST_KEYS_COUNT / 2;
diff --git a/src/storage/src/hummock/compactor/mod.rs b/src/storage/src/hummock/compactor/mod.rs
index c18828bc93e72..898d91d530b22 100644
--- a/src/storage/src/hummock/compactor/mod.rs
+++ b/src/storage/src/hummock/compactor/mod.rs
@@ -20,7 +20,7 @@ use risingwave_pb::hummock::report_compaction_task_request::{
     Event as ReportCompactionTaskEvent, HeartBeat as SharedHeartBeat,
     ReportTask as ReportSharedTask,
 };
-use risingwave_pb::hummock::{ReportFullScanTaskRequest, ReportVacuumTaskRequest};
+use risingwave_pb::hummock::{CompactTask, ReportFullScanTaskRequest, ReportVacuumTaskRequest};
 use risingwave_rpc_client::GrpcCompactorProxyClient;
 use thiserror_ext::AsReport;
 use tokio::sync::mpsc;
@@ -35,7 +35,7 @@ pub(super) mod task_progress;
 
 use std::collections::{HashMap, VecDeque};
 use std::marker::PhantomData;
-use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
 use std::time::{Duration, SystemTime};
 
@@ -51,7 +51,7 @@ pub use context::{
 use futures::{pin_mut, StreamExt};
 pub use iterator::{ConcatSstableIterator, SstableStreamIterator};
 use more_asserts::assert_ge;
-use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map;
+use risingwave_hummock_sdk::table_stats::{to_prost_table_stats_map, TableStatsMap};
 use risingwave_hummock_sdk::{compact_task_to_string, HummockCompactionTaskId, LocalSstableInfo};
 use risingwave_pb::hummock::compact_task::TaskStatus;
 use risingwave_pb::hummock::subscribe_compaction_event_request::{
@@ -80,13 +80,13 @@ use super::{
 use crate::filter_key_extractor::{
     FilterKeyExtractorImpl, FilterKeyExtractorManager, StaticFilterKeyExtractorManager,
 };
-use crate::hummock::compactor::compactor_runner::compact_and_build_sst;
+use crate::hummock::compactor::compaction_utils::calculate_task_parallelism;
+use crate::hummock::compactor::compactor_runner::{compact_and_build_sst, compact_done};
 use crate::hummock::iterator::{Forward, HummockIterator};
-use crate::hummock::multi_builder::SplitTableOutput;
 use crate::hummock::vacuum::Vacuum;
 use crate::hummock::{
-    validate_ssts, BlockedXor16FilterBuilder, FilterBuilder, HummockError,
-    SharedComapctorObjectIdManager, SstableWriterFactory, UnifiedSstableWriterFactory,
+    validate_ssts, BlockedXor16FilterBuilder, FilterBuilder, SharedComapctorObjectIdManager,
+    SstableWriterFactory, UnifiedSstableWriterFactory,
 };
 use crate::monitor::CompactorMetrics;
 
@@ -174,20 +174,19 @@ impl Compactor {
 
         compact_timer.observe_duration();
 
-        let ssts = Self::report_progress(
+        Self::report_progress(
             self.context.compactor_metrics.clone(),
             task_progress,
-            split_table_outputs,
+            &split_table_outputs,
             self.context.is_share_buffer_compact,
-        )
-        .await?;
+        );
 
         self.context
             .compactor_metrics
             .get_table_id_total_time_duration
             .observe(self.get_id_time.load(Ordering::Relaxed) as f64 / 1000.0 / 1000.0);
 
-        debug_assert!(ssts
+        debug_assert!(split_table_outputs
             .iter()
             .all(|table_info| table_info.sst_info.get_table_ids().is_sorted()));
 
@@ -197,33 +196,20 @@ impl Compactor {
                 "Finish Task {:?} split_index {:?} sst count {}",
                 task_id,
                 split_index,
-                ssts.len()
+                split_table_outputs.len()
             );
         }
-        Ok((ssts, table_stats_map))
+        Ok((split_table_outputs, table_stats_map))
     }
 
-    pub async fn report_progress(
+    pub fn report_progress(
         metrics: Arc<CompactorMetrics>,
         task_progress: Option<Arc<TaskProgress>>,
-        split_table_outputs: Vec<SplitTableOutput>,
+        ssts: &Vec<LocalSstableInfo>,
         is_share_buffer_compact: bool,
-    ) -> HummockResult<Vec<LocalSstableInfo>> {
-        let mut ssts = Vec::with_capacity(split_table_outputs.len());
-        let mut rets = vec![];
-
-        for SplitTableOutput {
-            sst_info,
-            upload_join_handle,
-        } in split_table_outputs
-        {
+    ) {
+        for sst_info in ssts {
             let sst_size = sst_info.file_size();
-            ssts.push(sst_info);
-            let ret = upload_join_handle
-                .verbose_instrument_await("upload")
-                .await
-                .map_err(HummockError::sstable_upload_error);
-            rets.push(ret);
             if let Some(tracker) = &task_progress {
                 tracker.inc_ssts_uploaded();
                 tracker.dec_num_pending_write_io();
@@ -234,10 +220,6 @@ impl Compactor {
                 metrics.compaction_upload_sst_counts.inc();
             }
         }
-        for ret in rets {
-            ret??;
-        }
-        Ok(ssts)
     }
 
     async fn compact_key_range_impl<F: SstableWriterFactory, B: FilterBuilder>(
@@ -248,7 +230,7 @@ impl Compactor {
         filter_key_extractor: Arc<FilterKeyExtractorImpl>,
         task_progress: Option<Arc<TaskProgress>>,
         object_id_getter: Box<dyn GetObjectId>,
-    ) -> HummockResult<(Vec<SplitTableOutput>, CompactionStatistics)> {
+    ) -> HummockResult<(Vec<LocalSstableInfo>, CompactionStatistics)> {
         let builder_factory = RemoteBuilderFactory::<F, B> {
             object_id_getter,
             limiter: self.context.memory_limiter.clone(),
@@ -265,6 +247,9 @@ impl Compactor {
             self.context.compactor_metrics.clone(),
             task_progress.clone(),
             self.task_config.table_vnode_partition.clone(),
+            self.context
+                .storage_opts
+                .compactor_concurrent_uploading_sst_count,
         );
         let compaction_statistics = compact_and_build_sst(
             &mut sst_builder,
@@ -299,14 +284,14 @@ pub fn start_compactor(
     let stream_retry_interval = Duration::from_secs(30);
     let task_progress = compactor_context.task_progress_manager.clone();
     let periodic_event_update_interval = Duration::from_millis(1000);
-    let pull_task_ack = Arc::new(AtomicBool::new(true));
+
+    let max_task_parallelism: u32 = (compactor_context.compaction_executor.worker_num() as f32
+        * compactor_context.storage_opts.compactor_max_task_multiplier)
+        .ceil() as u32;
+    let running_task_parallelism = Arc::new(AtomicU32::new(0));
+
     const MAX_PULL_TASK_COUNT: u32 = 4;
-    let max_pull_task_count = std::cmp::min(
-        compactor_context
-            .max_task_parallelism
-            .load(Ordering::SeqCst),
-        MAX_PULL_TASK_COUNT,
-    );
+    let max_pull_task_count = std::cmp::min(max_task_parallelism, MAX_PULL_TASK_COUNT);
 
     assert_ge!(
         compactor_context.storage_opts.compactor_max_task_multiplier,
@@ -321,7 +306,8 @@ pub fn start_compactor(
         // This outer loop is to recreate stream.
         'start_stream: loop {
             // reset state
-            pull_task_ack.store(true, Ordering::SeqCst);
+            // pull_task_ack.store(true, Ordering::SeqCst);
+            let mut pull_task_ack = true;
             tokio::select! {
                 // Wait for interval.
                 _ = min_interval.tick() => {},
@@ -365,7 +351,7 @@ pub fn start_compactor(
                     event_loop_iteration_now = Instant::now();
                 }
 
-                let pull_task_ack = pull_task_ack.clone();
+                let running_task_parallelism = running_task_parallelism.clone();
                 let request_sender = request_sender.clone();
                 let event: Option<Result<SubscribeCompactionEventResponse, _>> = tokio::select! {
                     _ = periodic_event_interval.tick() => {
@@ -389,9 +375,9 @@ pub fn start_compactor(
 
 
                         let mut pending_pull_task_count = 0;
-                        if pull_task_ack.load(Ordering::SeqCst) {
+                        if pull_task_ack {
                             // TODO: Compute parallelism on meta side
-                            pending_pull_task_count = compactor_context.get_free_quota().max(max_pull_task_count);
+                            pending_pull_task_count = (max_task_parallelism - running_task_parallelism.load(Ordering::SeqCst)).min(max_pull_task_count);
 
                             if pending_pull_task_count > 0 {
                                 if let Err(e) = request_sender.send(SubscribeCompactionEventRequest {
@@ -410,14 +396,14 @@ pub fn start_compactor(
                                     // re subscribe stream
                                     continue 'start_stream;
                                 } else {
-                                    pull_task_ack.store(false, Ordering::SeqCst);
+                                    pull_task_ack = false;
                                 }
                             }
                         }
 
                         tracing::info!(
-                            running_parallelism_count = %compactor_context.running_task_parallelism.load(Ordering::Relaxed),
-                            pull_task_ack = %pull_task_ack.load(Ordering::Relaxed),
+                            running_parallelism_count = %running_task_parallelism.load(Ordering::SeqCst),
+                            pull_task_ack = %pull_task_ack,
                             pending_pull_task_count = %pending_pull_task_count
                         );
 
@@ -433,6 +419,28 @@ pub fn start_compactor(
                     }
                 };
 
+                fn send_report_task_event(
+                    compact_task: &CompactTask,
+                    table_stats: TableStatsMap,
+                    request_sender: &mpsc::UnboundedSender<SubscribeCompactionEventRequest>,
+                ) {
+                    if let Err(e) = request_sender.send(SubscribeCompactionEventRequest {
+                        event: Some(RequestEvent::ReportTask(ReportTask {
+                            task_id: compact_task.task_id,
+                            task_status: compact_task.task_status,
+                            sorted_output_ssts: compact_task.sorted_output_ssts.clone(),
+                            table_stats_change: to_prost_table_stats_map(table_stats),
+                        })),
+                        create_at: SystemTime::now()
+                            .duration_since(std::time::UNIX_EPOCH)
+                            .expect("Clock may have gone backwards")
+                            .as_millis() as u64,
+                    }) {
+                        let task_id = compact_task.task_id;
+                        tracing::warn!(error = %e.as_report(), "Failed to report task {task_id:?}");
+                    }
+                }
+
                 match event {
                     Some(Ok(SubscribeCompactionEventResponse { event, create_at })) => {
                         let event = match event {
@@ -454,120 +462,180 @@ pub fn start_compactor(
                         let meta_client = hummock_meta_client.clone();
                         let sstable_object_id_manager = sstable_object_id_manager.clone();
                         let filter_key_extractor_manager = filter_key_extractor_manager.clone();
-                        executor.spawn(async move {
-                                match event {
-                                    ResponseEvent::CompactTask(compact_task)  => {
-                                        let (tx, rx) = tokio::sync::oneshot::channel();
-                                        let task_id = compact_task.task_id;
-                                        shutdown.lock().unwrap().insert(task_id, tx);
-                                        let ((compact_task, table_stats), _memory_tracker) = match sstable_object_id_manager.add_watermark_object_id(None).await
+
+                        match event {
+                            ResponseEvent::CompactTask(compact_task) => {
+                                let parallelism =
+                                    calculate_task_parallelism(&compact_task, &context);
+
+                                assert_ne!(parallelism, 0, "splits cannot be empty");
+
+                                if (max_task_parallelism
+                                    - running_task_parallelism.load(Ordering::SeqCst))
+                                    < parallelism as u32
+                                {
+                                    tracing::warn!(
+                                        "Not enough core parallelism to serve the task {} task_parallelism {} running_task_parallelism {} max_task_parallelism {}",
+                                        compact_task.task_id,
+                                        parallelism,
+                                        max_task_parallelism,
+                                        running_task_parallelism.load(Ordering::Relaxed),
+                                    );
+                                    let (compact_task, table_stats) = compact_done(
+                                        compact_task,
+                                        context.clone(),
+                                        vec![],
+                                        TaskStatus::NoAvailCpuResourceCanceled,
+                                    );
+
+                                    send_report_task_event(
+                                        &compact_task,
+                                        table_stats,
+                                        &request_sender,
+                                    );
+
+                                    continue 'consume_stream;
+                                }
+
+                                running_task_parallelism
+                                    .fetch_add(parallelism as u32, Ordering::SeqCst);
+                                executor.spawn(async move {
+                                    let (tx, rx) = tokio::sync::oneshot::channel();
+                                    let task_id = compact_task.task_id;
+                                    shutdown.lock().unwrap().insert(task_id, tx);
+                                    let ((compact_task, table_stats), _memory_tracker) =
+                                        match sstable_object_id_manager
+                                            .add_watermark_object_id(None)
+                                            .await
                                         {
                                             Ok(tracker_id) => {
-                                                let sstable_object_id_manager_clone = sstable_object_id_manager.clone();
+                                                let sstable_object_id_manager_clone =
+                                                    sstable_object_id_manager.clone();
                                                 let _guard = scopeguard::guard(
                                                     (tracker_id, sstable_object_id_manager_clone),
                                                     |(tracker_id, sstable_object_id_manager)| {
-                                                        sstable_object_id_manager.remove_watermark_object_id(tracker_id);
+                                                        sstable_object_id_manager
+                                                            .remove_watermark_object_id(tracker_id);
                                                     },
                                                 );
-                                                compactor_runner::compact(context.clone(), compact_task, rx, Box::new(sstable_object_id_manager.clone()), filter_key_extractor_manager.clone()).await
-                                            },
+
+                                                compactor_runner::compact(
+                                                    context.clone(),
+                                                    compact_task,
+                                                    rx,
+                                                    Box::new(sstable_object_id_manager.clone()),
+                                                    filter_key_extractor_manager.clone(),
+                                                )
+                                                .await
+                                            }
                                             Err(err) => {
                                                 tracing::warn!(error = %err.as_report(), "Failed to track pending SST object id");
                                                 let mut compact_task = compact_task;
-                                                compact_task.set_task_status(TaskStatus::TrackSstObjectIdFailed);
-                                                ((compact_task, HashMap::default()),None)
+                                                compact_task.set_task_status(
+                                                    TaskStatus::TrackSstObjectIdFailed,
+                                                );
+                                                ((compact_task, HashMap::default()), None)
                                             }
                                         };
-                                        shutdown.lock().unwrap().remove(&task_id);
-
-                                        let enable_check_compaction_result = context.storage_opts.check_compaction_result;
-                                        let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status() == TaskStatus::Success;
-                                        if let Err(e) = request_sender.send(SubscribeCompactionEventRequest {
-                                            event: Some(RequestEvent::ReportTask(
-                                                ReportTask {
-                                                    task_id: compact_task.task_id,
-                                                    task_status: compact_task.task_status,
-                                                    sorted_output_ssts: compact_task.sorted_output_ssts.clone(),
-                                                    table_stats_change:to_prost_table_stats_map(table_stats),
-                                                }
-                                            )),
-                                            create_at: SystemTime::now()
-                                                .duration_since(std::time::UNIX_EPOCH)
-                                                .expect("Clock may have gone backwards")
-                                                .as_millis() as u64,
-                                        }) {
-                                            tracing::warn!(error = %e.as_report(), "Failed to report task {task_id:?}");
-                                        }
-                                        if enable_check_compaction_result && need_check_task {
-                                            match check_compaction_result(&compact_task, context.clone()).await {
-                                                Err(e) => {
-                                                    tracing::warn!(error = %e.as_report(), "Failed to check compaction task {}",compact_task.task_id);
-                                                },
-                                                Ok(true) => (),
-                                                Ok(false) => {
-                                                    panic!("Failed to pass consistency check for result of compaction task:\n{:?}", compact_task_to_string(&compact_task));
-                                                }
-                                            }
-                                        }
-                                    }
-                                    ResponseEvent::VacuumTask(vacuum_task) => {
-                                        match Vacuum::handle_vacuum_task(
-                                            context.sstable_store.clone(),
-                                            &vacuum_task.sstable_object_ids,
-                                        )
-                                        .await{
-                                            Ok(_) => {
-                                                Vacuum::report_vacuum_task(vacuum_task, meta_client).await;
-                                            }
+                                    shutdown.lock().unwrap().remove(&task_id);
+                                    running_task_parallelism.fetch_sub(parallelism as u32, Ordering::SeqCst);
+
+                                    send_report_task_event(
+                                        &compact_task,
+                                        table_stats,
+                                        &request_sender,
+                                    );
+
+                                    let enable_check_compaction_result =
+                                    context.storage_opts.check_compaction_result;
+                                    let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status() == TaskStatus::Success;
+
+                                    if enable_check_compaction_result && need_check_task {
+                                        match check_compaction_result(&compact_task, context.clone())
+                                            .await
+                                        {
                                             Err(e) => {
-                                                tracing::warn!(error = %e.as_report(), "Failed to vacuum task")
+                                                tracing::warn!(error = %e.as_report(), "Failed to check compaction task {}",compact_task.task_id);
                                             }
-                                        }
-                                    }
-                                    ResponseEvent::FullScanTask(full_scan_task) => {
-                                        match Vacuum::handle_full_scan_task(full_scan_task, context.sstable_store.clone()).await {
-                                            Ok((object_ids, total_object_count, total_object_size)) => {
-                                                Vacuum::report_full_scan_task(object_ids, total_object_count, total_object_size, meta_client).await;
-                                            }
-                                            Err(e) => {
-                                                tracing::warn!(error = %e.as_report(), "Failed to iter object");
+                                            Ok(true) => (),
+                                            Ok(false) => {
+                                                panic!("Failed to pass consistency check for result of compaction task:\n{:?}", compact_task_to_string(&compact_task));
                                             }
                                         }
                                     }
-                                    ResponseEvent::ValidationTask(validation_task) => {
-                                        validate_ssts(
-                                            validation_task,
-                                            context.sstable_store.clone(),
-                                        )
-                                        .await;
+                                });
+                            }
+                            ResponseEvent::VacuumTask(vacuum_task) => {
+                                executor.spawn(async move {
+                                    match Vacuum::handle_vacuum_task(
+                                        context.sstable_store.clone(),
+                                        &vacuum_task.sstable_object_ids,
+                                    )
+                                    .await
+                                    {
+                                        Ok(_) => {
+                                            Vacuum::report_vacuum_task(vacuum_task, meta_client).await;
+                                        }
+                                        Err(e) => {
+                                            tracing::warn!(error = %e.as_report(), "Failed to vacuum task")
+                                        }
                                     }
-                                    ResponseEvent::CancelCompactTask(cancel_compact_task) => {
-                                        if let Some(tx) = shutdown
-                                            .lock()
-                                            .unwrap()
-                                            .remove(&cancel_compact_task.task_id)
-                                        {
-                                            if tx.send(()).is_err() {
-                                                tracing::warn!(
-                                                    "Cancellation of compaction task failed. task_id: {}",
-                                                    cancel_compact_task.task_id
-                                                );
-                                            }
-                                        } else {
-                                            tracing::warn!(
-                                                    "Attempting to cancel non-existent compaction task. task_id: {}",
-                                                    cancel_compact_task.task_id
-                                                );
+                                });
+                            }
+                            ResponseEvent::FullScanTask(full_scan_task) => {
+                                executor.spawn(async move {
+                                    match Vacuum::handle_full_scan_task(
+                                        full_scan_task,
+                                        context.sstable_store.clone(),
+                                    )
+                                    .await
+                                    {
+                                        Ok((object_ids, total_object_count, total_object_size)) => {
+                                            Vacuum::report_full_scan_task(
+                                                object_ids,
+                                                total_object_count,
+                                                total_object_size,
+                                                meta_client,
+                                            )
+                                            .await;
+                                        }
+                                        Err(e) => {
+                                            tracing::warn!(error = %e.as_report(), "Failed to iter object");
                                         }
                                     }
-
-                                    ResponseEvent::PullTaskAck(_pull_task_ack) => {
-                                        // set flag
-                                        pull_task_ack.store(true, Ordering::SeqCst);
+                                });
+                            }
+                            ResponseEvent::ValidationTask(validation_task) => {
+                                executor.spawn(async move {
+                                    validate_ssts(validation_task, context.sstable_store.clone())
+                                        .await;
+                                });
+                            }
+                            ResponseEvent::CancelCompactTask(cancel_compact_task) => {
+                                if let Some(tx) = shutdown
+                                    .lock()
+                                    .unwrap()
+                                    .remove(&cancel_compact_task.task_id)
+                                {
+                                    if tx.send(()).is_err() {
+                                        tracing::warn!(
+                                            "Cancellation of compaction task failed. task_id: {}",
+                                            cancel_compact_task.task_id
+                                        );
                                     }
+                                } else {
+                                    tracing::warn!(
+                                        "Attempting to cancel non-existent compaction task. task_id: {}",
+                                            cancel_compact_task.task_id
+                                    );
                                 }
-                            });
+                            }
+
+                            ResponseEvent::PullTaskAck(_pull_task_ack) => {
+                                // set flag
+                                pull_task_ack = true;
+                            }
+                        }
                     }
                     Some(Err(e)) => {
                         tracing::warn!("Failed to consume stream. {}", e.message());
diff --git a/src/storage/src/hummock/compactor/shared_buffer_compact.rs b/src/storage/src/hummock/compactor/shared_buffer_compact.rs
index 9c25eee366bda..52c8cc1710bc2 100644
--- a/src/storage/src/hummock/compactor/shared_buffer_compact.rs
+++ b/src/storage/src/hummock/compactor/shared_buffer_compact.rs
@@ -20,12 +20,11 @@ use std::sync::{Arc, LazyLock};
 
 use await_tree::InstrumentAwait;
 use bytes::Bytes;
-use foyer::memory::CacheContext;
-use futures::future::try_join_all;
+use foyer::CacheContext;
+use futures::future::{try_join, try_join_all};
 use futures::{stream, FutureExt, StreamExt, TryFutureExt};
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
-use risingwave_common::hash::VirtualNode;
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_sdk::key::{FullKey, FullKeyTracker, UserKey, EPOCH_LEN};
 use risingwave_hummock_sdk::key_range::KeyRange;
@@ -38,11 +37,11 @@ use crate::filter_key_extractor::{FilterKeyExtractorImpl, FilterKeyExtractorMana
 use crate::hummock::compactor::compaction_filter::DummyCompactionFilter;
 use crate::hummock::compactor::context::{await_tree_key, CompactorContext};
 use crate::hummock::compactor::{check_flush_result, CompactOutput, Compactor};
-use crate::hummock::event_handler::uploader::UploadTaskPayload;
-use crate::hummock::event_handler::LocalInstanceId;
+use crate::hummock::event_handler::uploader::UploadTaskOutput;
 use crate::hummock::iterator::{Forward, HummockIterator, MergeIterator, UserIterator};
 use crate::hummock::shared_buffer::shared_buffer_batch::{
-    SharedBufferBatch, SharedBufferBatchInner, SharedBufferKeyEntry, VersionedSharedBufferValue,
+    SharedBufferBatch, SharedBufferBatchInner, SharedBufferBatchOldValues, SharedBufferKeyEntry,
+    VersionedSharedBufferValue,
 };
 use crate::hummock::utils::MemoryTracker;
 use crate::hummock::{
@@ -59,12 +58,12 @@ const GC_WATERMARK_FOR_FLUSH: u64 = 0;
 pub async fn compact(
     context: CompactorContext,
     sstable_object_id_manager: SstableObjectIdManagerRef,
-    payload: UploadTaskPayload,
+    payload: Vec<ImmutableMemtable>,
     compaction_group_index: Arc<HashMap<TableId, CompactionGroupId>>,
     filter_key_extractor_manager: FilterKeyExtractorManager,
-) -> HummockResult<Vec<LocalSstableInfo>> {
-    let mut grouped_payload: HashMap<CompactionGroupId, UploadTaskPayload> = HashMap::new();
-    for imm in payload {
+) -> HummockResult<UploadTaskOutput> {
+    let mut grouped_payload: HashMap<CompactionGroupId, Vec<ImmutableMemtable>> = HashMap::new();
+    for imm in &payload {
         let compaction_group_id = match compaction_group_index.get(&imm.table_id) {
             // compaction group id is used only as a hint for grouping different data.
             // If the compaction group id is not found for the table id, we can assign a
@@ -79,14 +78,14 @@ pub async fn compact(
         grouped_payload
             .entry(compaction_group_id)
             .or_default()
-            .push(imm);
+            .push(imm.clone());
     }
 
-    let mut futures = vec![];
+    let mut new_value_futures = vec![];
     for (id, group_payload) in grouped_payload {
         let id_copy = id;
-        futures.push(
-            compact_shared_buffer(
+        new_value_futures.push(
+            compact_shared_buffer::<true>(
                 context.clone(),
                 sstable_object_id_manager.clone(),
                 filter_key_extractor_manager.clone(),
@@ -104,22 +103,51 @@ pub async fn compact(
             .instrument_await(format!("shared_buffer_compact_compaction_group {}", id)),
         );
     }
-    // Note that the output is reordered compared with input `payload`.
-    let result = try_join_all(futures)
-        .await?
+
+    let old_value_payload = payload
         .into_iter()
-        .flatten()
+        .filter(|imm| imm.has_old_value())
         .collect_vec();
-    Ok(result)
+
+    let old_value_future = async {
+        if old_value_payload.is_empty() {
+            Ok(vec![])
+        } else {
+            compact_shared_buffer::<false>(
+                context.clone(),
+                sstable_object_id_manager,
+                filter_key_extractor_manager,
+                old_value_payload,
+            )
+            .await
+        }
+    };
+
+    // Note that the output is reordered compared with input `payload`.
+    let (grouped_new_value_ssts, old_value_ssts) =
+        try_join(try_join_all(new_value_futures), old_value_future).await?;
+
+    let new_value_ssts = grouped_new_value_ssts.into_iter().flatten().collect_vec();
+    Ok(UploadTaskOutput {
+        new_value_ssts,
+        old_value_ssts,
+        wait_poll_timer: None,
+    })
 }
 
 /// For compaction from shared buffer to level 0, this is the only function gets called.
-async fn compact_shared_buffer(
+///
+/// The `IS_NEW_VALUE` flag means for the given payload, we are doing compaction using its new value or old value.
+/// When `IS_NEW_VALUE` is false, we are compacting with old value, and the payload imms should have `old_values` not `None`
+async fn compact_shared_buffer<const IS_NEW_VALUE: bool>(
     context: CompactorContext,
     sstable_object_id_manager: SstableObjectIdManagerRef,
     filter_key_extractor_manager: FilterKeyExtractorManager,
-    mut payload: UploadTaskPayload,
+    mut payload: Vec<ImmutableMemtable>,
 ) -> HummockResult<Vec<LocalSstableInfo>> {
+    if !IS_NEW_VALUE {
+        assert!(payload.iter().all(|imm| imm.has_old_value()));
+    }
     // Local memory compaction looks at all key ranges.
 
     let mut existing_table_ids: HashSet<u32> = payload
@@ -149,7 +177,7 @@ async fn compact_shared_buffer(
     });
 
     let total_key_count = payload.iter().map(|imm| imm.key_count()).sum::<usize>();
-    let (splits, sub_compaction_sstable_size, split_weight_by_vnode) =
+    let (splits, sub_compaction_sstable_size, table_vnode_partition) =
         generate_splits(&payload, &existing_table_ids, context.storage_opts.as_ref());
     let parallelism = splits.len();
     let mut compact_success = true;
@@ -157,14 +185,6 @@ async fn compact_shared_buffer(
     let mut compaction_futures = vec![];
     let use_block_based_filter = BlockedXor16FilterBuilder::is_kv_count_too_large(total_key_count);
 
-    let table_vnode_partition = if existing_table_ids.len() == 1 {
-        let table_id = existing_table_ids.iter().next().unwrap();
-        vec![(*table_id, split_weight_by_vnode)]
-            .into_iter()
-            .collect()
-    } else {
-        BTreeMap::default()
-    };
     for (split_index, key_range) in splits.into_iter().enumerate() {
         let compactor = SharedBufferCompactRunner::new(
             split_index,
@@ -177,7 +197,7 @@ async fn compact_shared_buffer(
         );
         let mut forward_iters = Vec::with_capacity(payload.len());
         for imm in &payload {
-            forward_iters.push(imm.clone().into_forward_iter());
+            forward_iters.push(imm.clone().into_directed_iter::<Forward, IS_NEW_VALUE>());
         }
         let compaction_executor = context.compaction_executor.clone();
         let multi_filter_key_extractor = multi_filter_key_extractor.clone();
@@ -300,7 +320,6 @@ async fn compact_shared_buffer(
 /// Merge multiple batches into a larger one
 pub async fn merge_imms_in_memory(
     table_id: TableId,
-    instance_id: LocalInstanceId,
     imms: Vec<ImmutableMemtable>,
     memory_tracker: Option<MemoryTracker>,
 ) -> ImmutableMemtable {
@@ -309,6 +328,28 @@ pub async fn merge_imms_in_memory(
     assert!(imms.iter().rev().map(|imm| imm.batch_id()).is_sorted());
     let max_imm_id = imms[0].batch_id();
 
+    let has_old_value = imms[0].has_old_value();
+    // TODO: make sure that the corner case on switch_op_consistency is handled
+    // If the imm of a table id contains old value, all other imm of the same table id should have old value
+    assert!(imms.iter().all(|imm| imm.has_old_value() == has_old_value));
+
+    let (old_value_size, global_old_value_size) = if has_old_value {
+        (
+            imms.iter()
+                .map(|imm| imm.old_values().expect("has old value").size)
+                .sum(),
+            Some(
+                imms[0]
+                    .old_values()
+                    .expect("has old value")
+                    .global_old_value_size
+                    .clone(),
+            ),
+        )
+    } else {
+        (0, None)
+    };
+
     let mut imm_iters = Vec::with_capacity(imms.len());
     let key_count = imms.iter().map(|imm| imm.key_count()).sum();
     let value_count = imms.iter().map(|imm| imm.value_count()).sum();
@@ -336,6 +377,11 @@ pub async fn merge_imms_in_memory(
 
     let mut merged_entries: Vec<SharedBufferKeyEntry> = Vec::with_capacity(key_count);
     let mut values: Vec<VersionedSharedBufferValue> = Vec::with_capacity(value_count);
+    let mut old_values: Option<Vec<Bytes>> = if has_old_value {
+        Some(Vec::with_capacity(value_count))
+    } else {
+        None
+    };
 
     merged_entries.push(SharedBufferKeyEntry {
         key: first_item_key.clone(),
@@ -380,34 +426,47 @@ pub async fn merge_imms_in_memory(
                 .iter()
                 .map(|(epoch_with_gap, value)| (*epoch_with_gap, value.clone())),
         );
+        if let Some(old_values) = &mut old_values {
+            old_values.extend(key_entry.old_values.expect("should exist").iter().cloned())
+        }
         mi.advance_peek_to_next_key();
         // Since there is no blocking point in this method, but it is cpu intensive, we call this method
         // to do cooperative scheduling
         tokio::task::consume_budget().await;
     }
 
+    let old_values = old_values.map(|old_values| {
+        SharedBufferBatchOldValues::new(
+            old_values,
+            old_value_size,
+            global_old_value_size.expect("should exist when has old value"),
+        )
+    });
+
     SharedBufferBatch {
         inner: Arc::new(SharedBufferBatchInner::new_with_multi_epoch_batches(
             epochs,
             merged_entries,
             values,
+            old_values,
             merged_size,
             max_imm_id,
             memory_tracker,
         )),
         table_id,
-        instance_id,
     }
 }
 
 ///  Based on the incoming payload and opts, calculate the sharding method and sstable size of shared buffer compaction.
 fn generate_splits(
-    payload: &UploadTaskPayload,
+    payload: &Vec<ImmutableMemtable>,
     existing_table_ids: &HashSet<u32>,
     storage_opts: &StorageOpts,
-) -> (Vec<KeyRange>, u64, u32) {
+) -> (Vec<KeyRange>, u64, BTreeMap<u32, u32>) {
     let mut size_and_start_user_keys = vec![];
     let mut compact_data_size = 0;
+    let mut table_size_infos: HashMap<u32, u64> = HashMap::default();
+    let mut table_vnode_partition = BTreeMap::default();
     for imm in payload {
         let data_size = {
             // calculate encoded bytes of key var length
@@ -415,6 +474,8 @@ fn generate_splits(
         };
         compact_data_size += data_size;
         size_and_start_user_keys.push((data_size, imm.start_user_key()));
+        let v = table_size_infos.entry(imm.table_id.table_id).or_insert(0);
+        *v += data_size;
     }
     size_and_start_user_keys.sort_by(|a, b| a.1.cmp(&b.1));
     let mut splits = Vec::with_capacity(size_and_start_user_keys.len());
@@ -424,6 +485,7 @@ fn generate_splits(
         splits.push(KeyRange::new(key_before_last.clone(), Bytes::new()));
     };
     let sstable_size = (storage_opts.sstable_size_mb as u64) << 20;
+    let min_sstable_size = (storage_opts.min_sstable_size_mb as u64) << 20;
     let parallel_compact_size = (storage_opts.parallel_compact_size_mb as u64) << 20;
     let parallelism = std::cmp::min(
         storage_opts.share_buffers_sync_parallelism as u64,
@@ -435,7 +497,6 @@ fn generate_splits(
         compact_data_size
     };
 
-    let mut vnode_partition_count = 0;
     if existing_table_ids.len() > 1 {
         if parallelism > 1 && compact_data_size > sstable_size {
             let mut last_buffer_size = 0;
@@ -460,23 +521,14 @@ fn generate_splits(
                 }
             }
         }
-    } else {
-        // Collect vnodes in imm
-        let mut vnodes = vec![];
-        for imm in payload {
-            vnodes.extend(imm.collect_vnodes());
-        }
-        vnodes.sort();
-        vnodes.dedup();
-
-        // Based on the estimated `vnode_avg_size`, calculate the required `vnode_partition_count` to avoid small files and further align
-        const MIN_SSTABLE_SIZE: u64 = 16 * 1024 * 1024;
-        if compact_data_size >= MIN_SSTABLE_SIZE && !vnodes.is_empty() {
-            let mut avg_vnode_size = compact_data_size / (vnodes.len() as u64);
-            vnode_partition_count = VirtualNode::COUNT;
-            while avg_vnode_size < MIN_SSTABLE_SIZE && vnode_partition_count > 0 {
-                vnode_partition_count /= 2;
-                avg_vnode_size *= 2;
+
+        // Meta node will calculate size of each state-table in one task in `risingwave_meta::hummock::manager::compaction::calculate_vnode_partition`.
+        // To make the calculate result more accurately we shall split the large state-table from other small ones.
+        for table_id in existing_table_ids {
+            if let Some(table_size) = table_size_infos.get(table_id)
+                && *table_size > min_sstable_size
+            {
+                table_vnode_partition.insert(*table_id, 1);
             }
         }
     }
@@ -484,11 +536,7 @@ fn generate_splits(
     // mul 1.2 for other extra memory usage.
     // Ensure that the size of each sstable is still less than `sstable_size` after optimization to avoid generating a huge size sstable which will affect the object store
     let sub_compaction_sstable_size = std::cmp::min(sstable_size, sub_compaction_data_size * 6 / 5);
-    (
-        splits,
-        sub_compaction_sstable_size,
-        vnode_partition_count as u32,
-    )
+    (splits, sub_compaction_sstable_size, table_vnode_partition)
 }
 
 pub struct SharedBufferCompactRunner {
@@ -564,7 +612,7 @@ mod tests {
     use risingwave_hummock_sdk::key::{prefix_slice_with_vnode, TableKey};
 
     use crate::hummock::compactor::shared_buffer_compact::generate_splits;
-    use crate::hummock::value::HummockValue;
+    use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferValue;
     use crate::mem_table::ImmutableMemtable;
     use crate::opts::StorageOpts;
 
@@ -582,7 +630,7 @@ mod tests {
             0,
             vec![(
                 generate_key("dddd"),
-                HummockValue::put(Bytes::from_static(b"v3")),
+                SharedBufferValue::Insert(Bytes::from_static(b"v3")),
             )],
             1024 * 1024,
             TableId::new(1),
@@ -592,7 +640,7 @@ mod tests {
             0,
             vec![(
                 generate_key("abb"),
-                HummockValue::put(Bytes::from_static(b"v3")),
+                SharedBufferValue::Insert(Bytes::from_static(b"v3")),
             )],
             (1024 + 256) * 1024,
             TableId::new(1),
@@ -603,7 +651,7 @@ mod tests {
             0,
             vec![(
                 generate_key("abc"),
-                HummockValue::put(Bytes::from_static(b"v2")),
+                SharedBufferValue::Insert(Bytes::from_static(b"v2")),
             )],
             (1024 + 512) * 1024,
             TableId::new(1),
@@ -613,7 +661,7 @@ mod tests {
             0,
             vec![(
                 generate_key("aaa"),
-                HummockValue::put(Bytes::from_static(b"v3")),
+                SharedBufferValue::Insert(Bytes::from_static(b"v3")),
             )],
             (1024 + 512) * 1024,
             TableId::new(1),
@@ -624,7 +672,7 @@ mod tests {
             0,
             vec![(
                 generate_key("aaa"),
-                HummockValue::put(Bytes::from_static(b"v3")),
+                SharedBufferValue::Insert(Bytes::from_static(b"v3")),
             )],
             (1024 + 256) * 1024,
             TableId::new(2),
@@ -637,13 +685,13 @@ mod tests {
             ..Default::default()
         };
         let payload = vec![imm1, imm2, imm3, imm4, imm5];
-        let (splits, _sstable_capacity, vnode) =
+        let (splits, _sstable_capacity, vnodes) =
             generate_splits(&payload, &HashSet::from_iter([1, 2]), &storage_opts);
         assert_eq!(
             splits.len(),
             storage_opts.share_buffers_sync_parallelism as usize
         );
-        assert_eq!(vnode, 0);
+        assert!(vnodes.is_empty());
         for i in 1..splits.len() {
             assert_eq!(splits[i].left, splits[i - 1].right);
             assert!(splits[i].left > splits[i - 1].left);
diff --git a/src/storage/src/hummock/error.rs b/src/storage/src/hummock/error.rs
index 10e7ffbf325a8..48f71b9199332 100644
--- a/src/storage/src/hummock/error.rs
+++ b/src/storage/src/hummock/error.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use risingwave_common::catalog::TableId;
 use risingwave_object_store::object::ObjectError;
 use thiserror::Error;
 use thiserror_ext::AsReport;
@@ -48,7 +49,11 @@ pub enum HummockErrorInner {
     #[error("Barrier read is unavailable for now. Likely the cluster is recovering")]
     ReadCurrentEpoch,
     #[error("Expired Epoch: watermark {safe_epoch}, epoch {epoch}")]
-    ExpiredEpoch { safe_epoch: u64, epoch: u64 },
+    ExpiredEpoch {
+        table_id: u32,
+        safe_epoch: u64,
+        epoch: u64,
+    },
     #[error("CompactionExecutor error: {0}")]
     CompactionExecutor(String),
     #[error("FileCache error: {0}")]
@@ -61,6 +66,8 @@ pub enum HummockErrorInner {
     SstableUploadError(String),
     #[error("Read backup error: {0}")]
     ReadBackupError(String),
+    #[error("Foyer error: {0}")]
+    FoyerError(anyhow::Error),
     #[error("Other error: {0}")]
     Other(String),
 }
@@ -106,8 +113,13 @@ impl HummockError {
         HummockErrorInner::ReadCurrentEpoch.into()
     }
 
-    pub fn expired_epoch(safe_epoch: u64, epoch: u64) -> HummockError {
-        HummockErrorInner::ExpiredEpoch { safe_epoch, epoch }.into()
+    pub fn expired_epoch(table_id: TableId, safe_epoch: u64, epoch: u64) -> HummockError {
+        HummockErrorInner::ExpiredEpoch {
+            table_id: table_id.table_id,
+            safe_epoch,
+            epoch,
+        }
+        .into()
     }
 
     pub fn is_expired_epoch(&self) -> bool {
@@ -146,6 +158,10 @@ impl HummockError {
         HummockErrorInner::ReadBackupError(error.to_string()).into()
     }
 
+    pub fn foyer_error(error: anyhow::Error) -> HummockError {
+        HummockErrorInner::FoyerError(error).into()
+    }
+
     pub fn other(error: impl ToString) -> HummockError {
         HummockErrorInner::Other(error.to_string()).into()
     }
diff --git a/src/storage/src/hummock/event_handler/hummock_event_handler.rs b/src/storage/src/hummock/event_handler/hummock_event_handler.rs
index 0b96aa19cb837..277984d3545dd 100644
--- a/src/storage/src/hummock/event_handler/hummock_event_handler.rs
+++ b/src/storage/src/hummock/event_handler/hummock_event_handler.rs
@@ -12,12 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashMap};
-use std::ops::DerefMut;
+use std::collections::{HashMap, HashSet, VecDeque};
 use std::pin::pin;
 use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering::Relaxed;
 use std::sync::{Arc, LazyLock};
+use std::time::Duration;
 
 use arc_swap::ArcSwap;
 use await_tree::InstrumentAwait;
@@ -28,7 +28,7 @@ use prometheus::core::{AtomicU64, GenericGauge};
 use prometheus::{Histogram, IntGauge};
 use risingwave_common::catalog::TableId;
 use risingwave_hummock_sdk::compaction_group::hummock_version_ext::SstDeltaInfo;
-use risingwave_hummock_sdk::{HummockEpoch, LocalSstableInfo, SyncResult};
+use risingwave_hummock_sdk::{HummockEpoch, SyncResult};
 use thiserror_ext::AsReport;
 use tokio::spawn;
 use tokio::sync::mpsc::error::SendError;
@@ -43,8 +43,7 @@ use crate::hummock::compactor::{await_tree_key, compact, CompactorContext};
 use crate::hummock::conflict_detector::ConflictDetector;
 use crate::hummock::event_handler::refiller::{CacheRefillerEvent, SpawnRefillTask};
 use crate::hummock::event_handler::uploader::{
-    default_spawn_merging_task, HummockUploader, SpawnMergingTask, SpawnUploadTask, SyncedData,
-    UploadTaskInfo, UploadTaskOutput, UploadTaskPayload, UploaderEvent,
+    HummockUploader, SpawnUploadTask, SyncedData, UploadTaskInfo, UploadTaskOutput, UploaderEvent,
 };
 use crate::hummock::event_handler::{
     HummockEvent, HummockReadVersionRef, HummockVersionUpdate, ReadOnlyReadVersionMapping,
@@ -56,14 +55,16 @@ use crate::hummock::store::version::{
 };
 use crate::hummock::utils::validate_table_key_range;
 use crate::hummock::{
-    HummockError, HummockResult, MemoryLimiter, SstableObjectIdManager, SstableStoreRef, TrackerId,
+    HummockResult, MemoryLimiter, SstableObjectIdManager, SstableStoreRef, TrackerId,
 };
+use crate::mem_table::ImmutableMemtable;
 use crate::monitor::HummockStateStoreMetrics;
 use crate::opts::StorageOpts;
 
 #[derive(Clone)]
 pub struct BufferTracker {
     flush_threshold: usize,
+    min_batch_flush_size: usize,
     global_buffer: Arc<MemoryLimiter>,
     global_upload_task_size: GenericGauge<AtomicU64>,
 }
@@ -75,25 +76,34 @@ impl BufferTracker {
     ) -> Self {
         let capacity = config.shared_buffer_capacity_mb * (1 << 20);
         let flush_threshold = (capacity as f32 * config.shared_buffer_flush_ratio) as usize;
+        let shared_buffer_min_batch_flush_size =
+            config.shared_buffer_min_batch_flush_size_mb * (1 << 20);
         assert!(
             flush_threshold < capacity,
             "flush_threshold {} should be less or equal to capacity {}",
             flush_threshold,
             capacity
         );
-        Self::new(capacity, flush_threshold, global_upload_task_size)
+        Self::new(
+            capacity,
+            flush_threshold,
+            global_upload_task_size,
+            shared_buffer_min_batch_flush_size,
+        )
     }
 
-    pub fn new(
+    fn new(
         capacity: usize,
         flush_threshold: usize,
         global_upload_task_size: GenericGauge<AtomicU64>,
+        min_batch_flush_size: usize,
     ) -> Self {
         assert!(capacity >= flush_threshold);
         Self {
             flush_threshold,
             global_buffer: Arc::new(MemoryLimiter::new(capacity as u64)),
             global_upload_task_size,
+            min_batch_flush_size,
         }
     }
 
@@ -118,9 +128,18 @@ impl BufferTracker {
 
     /// Return true when the buffer size minus current upload task size is still greater than the
     /// flush threshold.
-    pub fn need_more_flush(&self) -> bool {
+    pub fn need_flush(&self) -> bool {
         self.get_buffer_size() > self.flush_threshold + self.global_upload_task_size.get() as usize
     }
+
+    pub fn need_more_flush(&self, curr_batch_flush_size: usize) -> bool {
+        curr_batch_flush_size < self.min_batch_flush_size || self.need_flush()
+    }
+
+    #[cfg(test)]
+    pub(crate) fn flush_threshold(&self) -> usize {
+        self.flush_threshold
+    }
 }
 
 #[derive(Clone)]
@@ -175,10 +194,9 @@ pub struct HummockEventHandler {
     hummock_event_tx: HummockEventSender,
     hummock_event_rx: HummockEventReceiver,
     version_update_rx: UnboundedReceiver<HummockVersionUpdate>,
-    pending_sync_requests: BTreeMap<HummockEpoch, oneshot::Sender<HummockResult<SyncResult>>>,
     read_version_mapping: Arc<RwLock<ReadVersionMappingType>>,
     /// A copy of `read_version_mapping` but owned by event handler
-    local_read_version_mapping: HashMap<LocalInstanceId, HummockReadVersionRef>,
+    local_read_version_mapping: HashMap<LocalInstanceId, (TableId, HummockReadVersionRef)>,
 
     version_update_notifier_tx: Arc<tokio::sync::watch::Sender<HummockEpoch>>,
     pinned_version: Arc<ArcSwap<PinnedVersion>>,
@@ -194,12 +212,12 @@ pub struct HummockEventHandler {
 }
 
 async fn flush_imms(
-    payload: UploadTaskPayload,
+    payload: Vec<ImmutableMemtable>,
     task_info: UploadTaskInfo,
     compactor_context: CompactorContext,
     filter_key_extractor_manager: FilterKeyExtractorManager,
     sstable_object_id_manager: Arc<SstableObjectIdManager>,
-) -> HummockResult<Vec<LocalSstableInfo>> {
+) -> HummockResult<UploadTaskOutput> {
     for epoch in &task_info.epochs {
         let _ = sstable_object_id_manager
             .add_watermark_object_id(Some(*epoch))
@@ -257,18 +275,25 @@ impl HummockEventHandler {
                 spawn({
                     let future = async move {
                         let _timer = upload_task_latency.start_timer();
-                        let ssts = flush_imms(
-                            payload,
+                        let mut output = flush_imms(
+                            payload
+                                .into_values()
+                                .flat_map(|imms| imms.into_iter())
+                                .collect(),
                             task_info,
                             upload_compactor_context.clone(),
                             filter_key_extractor_manager.clone(),
                             sstable_object_id_manager.clone(),
                         )
                         .await?;
-                        Ok(UploadTaskOutput {
-                            ssts,
-                            wait_poll_timer: Some(wait_poll_latency.start_timer()),
-                        })
+                        assert!(
+                            output
+                                .wait_poll_timer
+                                .replace(wait_poll_latency.start_timer())
+                                .is_none(),
+                            "should not set timer before"
+                        );
+                        Ok(output)
                     };
                     if let Some(tree_root) = tree_root {
                         tree_root.instrument(future).left_future()
@@ -277,10 +302,6 @@ impl HummockEventHandler {
                     }
                 })
             }),
-            default_spawn_merging_task(
-                compactor_context.compaction_executor.clone(),
-                compactor_context.await_tree_reg.clone(),
-            ),
             CacheRefiller::default_spawn_refill_task(),
         )
     }
@@ -293,7 +314,6 @@ impl HummockEventHandler {
         state_store_metrics: Arc<HummockStateStoreMetrics>,
         storage_opts: &StorageOpts,
         spawn_upload_task: SpawnUploadTask,
-        spawn_merging_task: SpawnMergingTask,
         spawn_refill_task: SpawnRefillTask,
     ) -> Self {
         let (hummock_event_tx, hummock_event_rx) =
@@ -327,7 +347,6 @@ impl HummockEventHandler {
             state_store_metrics,
             pinned_version.clone(),
             spawn_upload_task,
-            spawn_merging_task,
             buffer_tracker,
             storage_opts,
         );
@@ -341,7 +360,6 @@ impl HummockEventHandler {
             hummock_event_tx,
             hummock_event_rx,
             version_update_rx,
-            pending_sync_requests: Default::default(),
             version_update_notifier_tx,
             pinned_version: Arc::new(ArcSwap::from_pointee(pinned_version)),
             write_conflict_detector,
@@ -378,137 +396,123 @@ impl HummockEventHandler {
 
 // Handler for different events
 impl HummockEventHandler {
-    fn handle_epoch_synced(
-        &mut self,
-        epoch: HummockEpoch,
-        newly_uploaded_sstables: Vec<StagingSstableInfo>,
-    ) {
+    fn handle_epoch_synced(&mut self, epoch: HummockEpoch, data: SyncedData) {
         debug!("epoch has been synced: {}.", epoch);
-        if !newly_uploaded_sstables.is_empty() {
-            newly_uploaded_sstables
+        let SyncedData {
+            newly_upload_ssts: newly_uploaded_sstables,
+            ..
+        } = data;
+        {
+            let newly_uploaded_sstables = newly_uploaded_sstables
                 .into_iter()
-                // Take rev because newer data come first in `newly_uploaded_sstables` but we apply
-                // older data first
-                .rev()
-                .for_each(|staging_sstable_info| {
-                    let staging_sstable_info_ref = Arc::new(staging_sstable_info);
-                    self.for_each_read_version(|read_version| {
-                        read_version.update(VersionUpdate::Staging(StagingData::Sst(
-                            staging_sstable_info_ref.clone(),
-                        )))
-                    });
+                .map(Arc::new)
+                .collect_vec();
+            if !newly_uploaded_sstables.is_empty() {
+                let related_instance_ids: HashSet<_> = newly_uploaded_sstables
+                    .iter()
+                    .flat_map(|sst| sst.imm_ids().keys().cloned())
+                    .collect();
+                self.for_each_read_version(related_instance_ids, |instance_id, read_version| {
+                    newly_uploaded_sstables
+                        .iter()
+                        // Take rev because newer data come first in `newly_uploaded_sstables` but we apply
+                        // older data first
+                        .rev()
+                        .for_each(|staging_sstable_info| {
+                            if staging_sstable_info.imm_ids().contains_key(&instance_id) {
+                                read_version.update(VersionUpdate::Staging(StagingData::Sst(
+                                    staging_sstable_info.clone(),
+                                )));
+                            }
+                        });
                 });
-        }
-        let result = self
-            .uploader
-            .get_synced_data(epoch)
-            .expect("data just synced. must exist");
-        // clear the pending sync epoch that is older than newly synced epoch
-        while let Some((smallest_pending_sync_epoch, _)) =
-            self.pending_sync_requests.first_key_value()
-        {
-            if *smallest_pending_sync_epoch > epoch {
-                // The smallest pending sync epoch has not synced yet. Wait later
-                break;
-            }
-            let (pending_sync_epoch, result_sender) =
-                self.pending_sync_requests.pop_first().expect("must exist");
-            if pending_sync_epoch == epoch {
-                send_sync_result(result_sender, to_sync_result(result));
-                break;
-            } else {
-                send_sync_result(
-                    result_sender,
-                    Err(HummockError::other(format!(
-                        "epoch {} is not a checkpoint epoch",
-                        pending_sync_epoch
-                    ))),
-                );
             }
-        }
+        };
     }
 
     /// This function will be performed under the protection of the `read_version_mapping` read
     /// lock, and add write lock on each `read_version` operation
-    fn for_each_read_version(&self, mut f: impl FnMut(&mut HummockReadVersion)) {
-        self.local_read_version_mapping
-            .values()
-            .for_each(|read_version: &HummockReadVersionRef| f(read_version.write().deref_mut()));
+    fn for_each_read_version(
+        &self,
+        instances: impl IntoIterator<Item = LocalInstanceId>,
+        mut f: impl FnMut(LocalInstanceId, &mut HummockReadVersion),
+    ) {
+        let instances = {
+            #[cfg(debug_assertions)]
+            {
+                // check duplication on debug_mode
+                let mut id_set = HashSet::new();
+                for instance in instances {
+                    assert!(id_set.insert(instance));
+                }
+                id_set
+            }
+            #[cfg(not(debug_assertions))]
+            {
+                instances
+            }
+        };
+        let mut pending = VecDeque::new();
+        let mut total_count = 0;
+        for instance_id in instances {
+            let Some((_, read_version)) = self.local_read_version_mapping.get(&instance_id) else {
+                continue;
+            };
+            total_count += 1;
+            if let Some(mut write_guard) = read_version.try_write() {
+                f(instance_id, &mut write_guard);
+            } else {
+                pending.push_back(instance_id);
+            }
+        }
+        if !pending.is_empty() {
+            warn!(
+                pending_count = pending.len(),
+                total_count, "cannot acquire lock for all read version"
+            );
+        }
+
+        const TRY_LOCK_TIMEOUT: Duration = Duration::from_millis(1);
+
+        while let Some(instance_id) = pending.pop_front() {
+            let (_, read_version) = self
+                .local_read_version_mapping
+                .get(&instance_id)
+                .expect("have checked exist before");
+            if let Some(mut write_guard) = read_version.try_write_for(TRY_LOCK_TIMEOUT) {
+                f(instance_id, &mut write_guard);
+            } else {
+                warn!(instance_id, "failed to get lock again for instance");
+                pending.push_back(instance_id);
+            }
+        }
     }
 
     fn handle_data_spilled(&mut self, staging_sstable_info: StagingSstableInfo) {
-        // todo: do some prune for version update
         let staging_sstable_info = Arc::new(staging_sstable_info);
-        self.for_each_read_version(|read_version| {
-            trace!("data_spilled. SST size {}", staging_sstable_info.imm_size());
-            read_version.update(VersionUpdate::Staging(StagingData::Sst(
-                staging_sstable_info.clone(),
-            )))
-        })
+        trace!("data_spilled. SST size {}", staging_sstable_info.imm_size());
+        self.for_each_read_version(
+            staging_sstable_info.imm_ids().keys().cloned(),
+            |_, read_version| {
+                read_version.update(VersionUpdate::Staging(StagingData::Sst(
+                    staging_sstable_info.clone(),
+                )))
+            },
+        )
     }
 
-    fn handle_await_sync_epoch(
+    fn handle_sync_epoch(
         &mut self,
         new_sync_epoch: HummockEpoch,
         sync_result_sender: oneshot::Sender<HummockResult<SyncResult>>,
     ) {
-        debug!("receive await sync epoch: {}", new_sync_epoch);
-        // The epoch to sync has been committed already.
-        if new_sync_epoch <= self.uploader.max_committed_epoch() {
-            send_sync_result(
-                sync_result_sender,
-                Err(HummockError::other(format!(
-                    "epoch {} has been committed. {}",
-                    new_sync_epoch,
-                    self.uploader.max_committed_epoch()
-                ))),
-            );
-            return;
-        }
-        // The epoch has been synced
-        if new_sync_epoch <= self.uploader.max_synced_epoch() {
-            debug!(
-                "epoch {} has been synced. Current max_sync_epoch {}",
-                new_sync_epoch,
-                self.uploader.max_synced_epoch()
-            );
-            if let Some(result) = self.uploader.get_synced_data(new_sync_epoch) {
-                let result = to_sync_result(result);
-                send_sync_result(sync_result_sender, result);
-            } else {
-                send_sync_result(
-                    sync_result_sender,
-                    Err(HummockError::other(
-                        "the requested sync epoch is not a checkpoint epoch",
-                    )),
-                );
-            }
-            return;
-        }
-
         debug!(
             "awaiting for epoch to be synced: {}, max_synced_epoch: {}",
             new_sync_epoch,
             self.uploader.max_synced_epoch()
         );
-
-        // If the epoch is not synced, we add to the `pending_sync_requests` anyway. If the epoch is
-        // not a checkpoint epoch, it will be clear with the max synced epoch bumps up.
-        if let Some(old_sync_result_sender) = self
-            .pending_sync_requests
-            .insert(new_sync_epoch, sync_result_sender)
-        {
-            let _ = old_sync_result_sender
-                .send(Err(HummockError::other(
-                    "the sync rx is overwritten by an new rx",
-                )))
-                .inspect_err(|e| {
-                    error!(
-                        "unable to send sync result: {}. Err: {:?}",
-                        new_sync_epoch, e
-                    );
-                });
-        }
+        self.uploader
+            .start_sync_epoch(new_sync_epoch, sync_result_sender);
     }
 
     async fn handle_clear(&mut self, notifier: oneshot::Sender<()>, prev_epoch: u64) {
@@ -516,7 +520,6 @@ impl HummockEventHandler {
             prev_epoch,
             max_committed_epoch = self.uploader.max_committed_epoch(),
             max_synced_epoch = self.uploader.max_synced_epoch(),
-            max_sealed_epoch = self.uploader.max_sealed_epoch(),
             "handle clear event"
         );
 
@@ -579,22 +582,12 @@ impl HummockEventHandler {
             );
         }
 
-        for (epoch, result_sender) in self.pending_sync_requests.extract_if(|_, _| true) {
-            send_sync_result(
-                result_sender,
-                Err(HummockError::other(format!(
-                    "the sync epoch {} has been cleared",
-                    epoch
-                ))),
-            );
-        }
-
         assert!(
             self.local_read_version_mapping.is_empty(),
             "read version mapping not empty when clear. remaining tables: {:?}",
             self.local_read_version_mapping
                 .values()
-                .map(|read_version| read_version.read().table_id())
+                .map(|(_, read_version)| read_version.read().table_id())
                 .collect_vec()
         );
 
@@ -654,7 +647,7 @@ impl HummockEventHandler {
 
                 version_to_apply
             }
-            HummockVersionUpdate::PinnedVersion(version) => version,
+            HummockVersionUpdate::PinnedVersion(version) => *version,
         };
 
         validate_table_key_range(&newly_pinned_version);
@@ -675,9 +668,13 @@ impl HummockEventHandler {
             .store(Arc::new(new_pinned_version.clone()));
 
         {
-            self.for_each_read_version(|read_version| {
-                read_version.update(VersionUpdate::CommittedSnapshot(new_pinned_version.clone()))
-            });
+            self.for_each_read_version(
+                self.local_read_version_mapping.keys().cloned(),
+                |_, read_version| {
+                    read_version
+                        .update(VersionUpdate::CommittedSnapshot(new_pinned_version.clone()))
+                },
+            );
         }
 
         let prev_max_committed_epoch = pinned_version.max_committed_epoch();
@@ -753,38 +750,18 @@ impl HummockEventHandler {
 
     fn handle_uploader_event(&mut self, event: UploaderEvent) {
         match event {
-            UploaderEvent::SyncFinish(epoch, newly_uploaded_sstables) => {
+            UploaderEvent::SyncFinish(epoch, data) => {
                 let _timer = self
                     .metrics
                     .event_handler_on_sync_finish_latency
                     .start_timer();
-                self.handle_epoch_synced(epoch, newly_uploaded_sstables);
+                self.handle_epoch_synced(epoch, data);
             }
 
             UploaderEvent::DataSpilled(staging_sstable_info) => {
                 let _timer = self.metrics.event_handler_on_spilled_latency.start_timer();
                 self.handle_data_spilled(staging_sstable_info);
             }
-
-            UploaderEvent::ImmMerged(merge_output) => {
-                // update read version for corresponding table shards
-                if let Some(read_version) = self
-                    .local_read_version_mapping
-                    .get(&merge_output.instance_id)
-                {
-                    read_version
-                        .write()
-                        .update(VersionUpdate::Staging(StagingData::MergedImmMem(
-                            merge_output.merged_imm,
-                            merge_output.imm_ids,
-                        )));
-                } else {
-                    warn!(
-                        "handle ImmMerged: table instance not found. table {:?}, instance {}",
-                        &merge_output.table_id, &merge_output.instance_id
-                    )
-                }
-            }
         }
     }
 
@@ -794,11 +771,11 @@ impl HummockEventHandler {
             HummockEvent::BufferMayFlush => {
                 self.uploader.may_flush();
             }
-            HummockEvent::AwaitSyncEpoch {
+            HummockEvent::SyncEpoch {
                 new_sync_epoch,
                 sync_result_sender,
             } => {
-                self.handle_await_sync_epoch(new_sync_epoch, sync_result_sender);
+                self.handle_sync_epoch(new_sync_epoch, sync_result_sender);
             }
             HummockEvent::Clear(_, _) => {
                 unreachable!("clear is handled in separated async context")
@@ -806,48 +783,36 @@ impl HummockEventHandler {
             HummockEvent::Shutdown => {
                 unreachable!("shutdown is handled specially")
             }
-            HummockEvent::ImmToUploader(imm) => {
+            HummockEvent::InitEpoch {
+                instance_id,
+                init_epoch,
+            } => {
+                let table_id = self
+                    .local_read_version_mapping
+                    .get(&instance_id)
+                    .expect("should exist")
+                    .0;
+                self.uploader
+                    .init_instance(instance_id, table_id, init_epoch);
+            }
+            HummockEvent::ImmToUploader { instance_id, imm } => {
                 assert!(
-                    self.local_read_version_mapping
-                        .contains_key(&imm.instance_id),
+                    self.local_read_version_mapping.contains_key(&instance_id),
                     "add imm from non-existing read version instance: instance_id: {}, table_id {}",
-                    imm.instance_id,
+                    instance_id,
                     imm.table_id,
                 );
-                self.uploader.add_imm(imm);
+                self.uploader.add_imm(instance_id, imm);
                 self.uploader.may_flush();
             }
 
-            HummockEvent::SealEpoch {
-                epoch,
-                is_checkpoint,
-            } => {
-                self.uploader.seal_epoch(epoch);
-
-                if is_checkpoint {
-                    self.uploader.start_sync_epoch(epoch);
-                } else {
-                    // start merging task on non-checkpoint epochs sealed
-                    self.uploader.start_merge_imms(epoch);
-                }
-            }
-
             HummockEvent::LocalSealEpoch {
-                epoch,
+                next_epoch,
                 opts,
-                table_id,
                 instance_id,
             } => {
-                assert!(
-                    self.local_read_version_mapping
-                        .contains_key(&instance_id),
-                    "seal epoch from non-existing read version instance: instance_id: {}, table_id: {}, epoch: {}",
-                    instance_id, table_id, epoch,
-                );
-                if let Some((direction, watermarks)) = opts.table_watermarks {
-                    self.uploader
-                        .add_table_watermarks(epoch, table_id, watermarks, direction)
-                }
+                self.uploader
+                    .local_seal_epoch(instance_id, next_epoch, opts);
             }
 
             #[cfg(any(test, feature = "test"))]
@@ -864,17 +829,17 @@ impl HummockEventHandler {
                 vnodes,
             } => {
                 let pinned_version = self.pinned_version.load();
+                let instance_id = self.generate_instance_id();
                 let basic_read_version = Arc::new(RwLock::new(
                     HummockReadVersion::new_with_replication_option(
                         table_id,
+                        instance_id,
                         (**pinned_version).clone(),
                         is_replicated,
                         vnodes,
                     ),
                 ));
 
-                let instance_id = self.generate_instance_id();
-
                 debug!(
                     "new read version registered: table_id: {}, instance_id: {}",
                     table_id, instance_id
@@ -882,7 +847,7 @@ impl HummockEventHandler {
 
                 {
                     self.local_read_version_mapping
-                        .insert(instance_id, basic_read_version.clone());
+                        .insert(instance_id, (table_id, basic_read_version.clone()));
                     let mut read_version_mapping_guard = self.read_version_mapping.write();
 
                     read_version_mapping_guard
@@ -906,33 +871,29 @@ impl HummockEventHandler {
                             table_id, instance_id
                         );
                         guard.event_sender.take().expect("sender is just set");
-                        self.destroy_read_version(table_id, instance_id);
+                        self.destroy_read_version(instance_id);
                     }
                 }
             }
 
-            HummockEvent::DestroyReadVersion {
-                table_id,
-                instance_id,
-            } => {
-                self.destroy_read_version(table_id, instance_id);
+            HummockEvent::DestroyReadVersion { instance_id } => {
+                self.uploader.may_destroy_instance(instance_id);
+                self.destroy_read_version(instance_id);
             }
         }
     }
 
-    fn destroy_read_version(&mut self, table_id: TableId, instance_id: LocalInstanceId) {
+    fn destroy_read_version(&mut self, instance_id: LocalInstanceId) {
         {
             {
-                debug!(
-                    "read version deregister: table_id: {}, instance_id: {}",
-                    table_id, instance_id
-                );
-                self.local_read_version_mapping
+                debug!("read version deregister: instance_id: {}", instance_id);
+                let (table_id, _) = self
+                    .local_read_version_mapping
                     .remove(&instance_id)
                     .unwrap_or_else(|| {
                         panic!(
-                            "DestroyHummockInstance inexist instance table_id {} instance_id {}",
-                            table_id, instance_id
+                            "DestroyHummockInstance inexist instance instance_id {}",
+                            instance_id
                         )
                     });
                 let mut read_version_mapping_guard = self.read_version_mapping.write();
@@ -963,243 +924,68 @@ impl HummockEventHandler {
     }
 }
 
-fn send_sync_result(
+pub(super) fn send_sync_result(
     sender: oneshot::Sender<HummockResult<SyncResult>>,
-    result: HummockResult<SyncResult>,
+    result: HummockResult<&SyncedData>,
 ) {
+    let result = result.map(
+        |SyncedData {
+             newly_upload_ssts,
+             uploaded_ssts,
+             table_watermarks,
+         }| {
+            let mut sync_size = 0;
+            let mut uncommitted_ssts = Vec::new();
+            let mut old_value_ssts = Vec::new();
+            // The newly uploaded `sstable_infos` contains newer data. Therefore,
+            // `newly_upload_ssts` at the front
+            for sst in newly_upload_ssts.iter().chain(uploaded_ssts.iter()) {
+                sync_size += sst.imm_size();
+                uncommitted_ssts.extend(sst.sstable_infos().iter().cloned());
+                old_value_ssts.extend(sst.old_value_sstable_infos().iter().cloned());
+            }
+            SyncResult {
+                sync_size,
+                uncommitted_ssts,
+                table_watermarks: table_watermarks.clone(),
+                old_value_ssts,
+            }
+        },
+    );
+
     let _ = sender.send(result).inspect_err(|e| {
         error!("unable to send sync result. Err: {:?}", e);
     });
 }
 
-fn to_sync_result(result: &HummockResult<SyncedData>) -> HummockResult<SyncResult> {
-    match result {
-        Ok(sync_data) => {
-            let sync_size = sync_data
-                .staging_ssts
-                .iter()
-                .map(StagingSstableInfo::imm_size)
-                .sum();
-            Ok(SyncResult {
-                sync_size,
-                uncommitted_ssts: sync_data
-                    .staging_ssts
-                    .iter()
-                    .flat_map(|staging_sstable_info| staging_sstable_info.sstable_infos().clone())
-                    .collect(),
-                table_watermarks: sync_data.table_watermarks.clone(),
-            })
-        }
-        Err(e) => Err(HummockError::other(format!(
-            "sync task failed: {}",
-            e.as_report()
-        ))),
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use std::future::{poll_fn, Future};
-    use std::iter::once;
     use std::sync::Arc;
     use std::task::Poll;
 
-    use bytes::Bytes;
     use futures::FutureExt;
-    use itertools::Itertools;
-    use risingwave_common::buffer::Bitmap;
-    use risingwave_common::catalog::TableId;
+    use parking_lot::Mutex;
+    use risingwave_common::buffer::BitmapBuilder;
     use risingwave_common::hash::VirtualNode;
     use risingwave_common::util::epoch::{test_epoch, EpochExt};
-    use risingwave_common::util::iter_util::ZipEqDebug;
-    use risingwave_hummock_sdk::key::TableKey;
     use risingwave_hummock_sdk::version::HummockVersion;
     use risingwave_pb::hummock::PbHummockVersion;
     use tokio::spawn;
     use tokio::sync::mpsc::unbounded_channel;
     use tokio::sync::oneshot;
-    use tokio::task::yield_now;
 
     use crate::hummock::event_handler::refiller::CacheRefiller;
+    use crate::hummock::event_handler::uploader::tests::{gen_imm, TEST_TABLE_ID};
+    use crate::hummock::event_handler::uploader::UploadTaskOutput;
     use crate::hummock::event_handler::{HummockEvent, HummockEventHandler, HummockVersionUpdate};
     use crate::hummock::iterator::test_utils::mock_sstable_store;
     use crate::hummock::local_version::pinned_version::PinnedVersion;
-    use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferBatch;
     use crate::hummock::store::version::{StagingData, VersionUpdate};
     use crate::hummock::test_utils::default_opts_for_test;
-    use crate::hummock::value::HummockValue;
     use crate::hummock::HummockError;
     use crate::monitor::HummockStateStoreMetrics;
-
-    #[tokio::test]
-    async fn test_event_handler_merging_task() {
-        let table_id = TableId::new(123);
-        let epoch0 = test_epoch(233);
-        let pinned_version = PinnedVersion::new(
-            HummockVersion::from_rpc_protobuf(&PbHummockVersion {
-                id: 1,
-                max_committed_epoch: epoch0,
-                ..Default::default()
-            }),
-            unbounded_channel().0,
-        );
-
-        let mut storage_opts = default_opts_for_test();
-        storage_opts.imm_merge_threshold = 5;
-
-        let (_version_update_tx, version_update_rx) = unbounded_channel();
-
-        let (spawn_upload_task_tx, mut spawn_upload_task_rx) = unbounded_channel();
-        let (spawn_merging_task_tx, mut spawn_merging_task_rx) = unbounded_channel();
-        let event_handler = HummockEventHandler::new_inner(
-            version_update_rx,
-            pinned_version,
-            None,
-            mock_sstable_store(),
-            Arc::new(HummockStateStoreMetrics::unused()),
-            &storage_opts,
-            Arc::new(move |_, _| {
-                let (tx, rx) = oneshot::channel::<()>();
-                spawn_upload_task_tx.send(tx).unwrap();
-                spawn(async move {
-                    // wait for main thread to notify returning error
-                    rx.await.unwrap();
-                    Err(HummockError::other("".to_string()))
-                })
-            }),
-            Arc::new(move |_, _, imms, _| {
-                let (tx, rx) = oneshot::channel::<()>();
-                let (finish_tx, finish_rx) = oneshot::channel::<()>();
-                spawn_merging_task_tx.send((tx, finish_rx)).unwrap();
-                spawn(async move {
-                    rx.await.unwrap();
-                    finish_tx.send(()).unwrap();
-                    imms[0].clone()
-                })
-            }),
-            CacheRefiller::default_spawn_refill_task(),
-        );
-
-        let tx = event_handler.event_sender();
-
-        let _join_handle = spawn(event_handler.start_hummock_event_handler_worker());
-
-        let (read_version_tx, read_version_rx) = oneshot::channel();
-
-        tx.send(HummockEvent::RegisterReadVersion {
-            table_id,
-            new_read_version_sender: read_version_tx,
-            is_replicated: false,
-            vnodes: Arc::new(Bitmap::ones(VirtualNode::COUNT)),
-        })
-        .unwrap();
-        let (read_version, guard) = read_version_rx.await.unwrap();
-        let instance_id = guard.instance_id;
-
-        let build_batch = |epoch, spill_offset| {
-            SharedBufferBatch::build_shared_buffer_batch(
-                epoch,
-                spill_offset,
-                vec![(TableKey(Bytes::from("key")), HummockValue::Delete)],
-                10,
-                table_id,
-                instance_id,
-                None,
-            )
-        };
-
-        let epoch1 = epoch0.next_epoch();
-        let imm1 = build_batch(epoch1, 0);
-        read_version
-            .write()
-            .update(VersionUpdate::Staging(StagingData::ImmMem(imm1.clone())));
-        tx.send(HummockEvent::ImmToUploader(imm1.clone())).unwrap();
-        tx.send(HummockEvent::SealEpoch {
-            epoch: epoch1,
-            is_checkpoint: true,
-        })
-        .unwrap();
-        let (sync_tx, mut sync_rx) = oneshot::channel();
-        tx.send(HummockEvent::AwaitSyncEpoch {
-            new_sync_epoch: epoch1,
-            sync_result_sender: sync_tx,
-        })
-        .unwrap();
-
-        let upload_finish_tx = spawn_upload_task_rx.recv().await.unwrap();
-        assert!(poll_fn(|cx| Poll::Ready(sync_rx.poll_unpin(cx)))
-            .await
-            .is_pending());
-
-        let epoch2 = epoch1.next_epoch();
-        let mut imm_ids = Vec::new();
-        for i in 0..10 {
-            let imm = build_batch(epoch2, i);
-            imm_ids.push(imm.batch_id());
-            read_version
-                .write()
-                .update(VersionUpdate::Staging(StagingData::ImmMem(imm.clone())));
-            tx.send(HummockEvent::ImmToUploader(imm)).unwrap();
-        }
-
-        for (staging_imm, imm_id) in read_version
-            .read()
-            .staging()
-            .imm
-            .iter()
-            .zip_eq_debug(imm_ids.iter().copied().rev().chain(once(imm1.batch_id())))
-        {
-            assert_eq!(staging_imm.batch_id(), imm_id);
-        }
-
-        // should start merging task
-        tx.send(HummockEvent::SealEpoch {
-            epoch: epoch2,
-            is_checkpoint: false,
-        })
-        .unwrap();
-
-        println!("before wait spawn merging task");
-
-        let (merging_start_tx, merging_finish_rx) = spawn_merging_task_rx.recv().await.unwrap();
-        merging_start_tx.send(()).unwrap();
-
-        println!("after wait spawn merging task");
-
-        // yield to possibly poll the merging task, though it shouldn't poll it because there is unfinished syncing task
-        yield_now().await;
-
-        for (staging_imm, imm_id) in read_version
-            .read()
-            .staging()
-            .imm
-            .iter()
-            .zip_eq_debug(imm_ids.iter().copied().rev().chain(once(imm1.batch_id())))
-        {
-            assert_eq!(staging_imm.batch_id(), imm_id);
-        }
-
-        upload_finish_tx.send(()).unwrap();
-        assert!(sync_rx.await.unwrap().is_err());
-
-        merging_finish_rx.await.unwrap();
-
-        // yield to poll the merging task, and then it should have finished.
-        for _ in 0..10 {
-            yield_now().await;
-        }
-
-        assert_eq!(
-            read_version
-                .read()
-                .staging()
-                .imm
-                .iter()
-                .map(|imm| imm.batch_id())
-                .collect_vec(),
-            vec![*imm_ids.last().unwrap(), imm1.batch_id()]
-        );
-    }
+    use crate::store::SealCurrentEpochOptions;
 
     #[tokio::test]
     async fn test_clear_shared_buffer() {
@@ -1226,11 +1012,10 @@ mod tests {
             version_update_rx,
             initial_version.clone(),
             None,
-            mock_sstable_store(),
+            mock_sstable_store().await,
             Arc::new(HummockStateStoreMetrics::unused()),
             &default_opts_for_test(),
             Arc::new(|_, _| unreachable!("should not spawn upload task")),
-            Arc::new(|_, _, _, _| unreachable!("should not spawn merging task")),
             Arc::new(move |_, _, old_version, new_version| {
                 let (tx, rx) = oneshot::channel();
                 refill_task_tx_clone
@@ -1262,7 +1047,9 @@ mod tests {
         let version1 = make_new_version(epoch1);
         {
             version_update_tx
-                .send(HummockVersionUpdate::PinnedVersion(version1.clone()))
+                .send(HummockVersionUpdate::PinnedVersion(Box::new(
+                    version1.clone(),
+                )))
                 .unwrap();
             let (old_version, new_version, refill_finish_tx) = refill_task_rx.recv().await.unwrap();
             assert_eq!(old_version.version(), initial_version.version());
@@ -1282,10 +1069,14 @@ mod tests {
         let version3 = make_new_version(epoch3);
         {
             version_update_tx
-                .send(HummockVersionUpdate::PinnedVersion(version2.clone()))
+                .send(HummockVersionUpdate::PinnedVersion(Box::new(
+                    version2.clone(),
+                )))
                 .unwrap();
             version_update_tx
-                .send(HummockVersionUpdate::PinnedVersion(version3.clone()))
+                .send(HummockVersionUpdate::PinnedVersion(Box::new(
+                    version3.clone(),
+                )))
                 .unwrap();
             let (old_version2, new_version2, _refill_finish_tx2) =
                 refill_task_rx.recv().await.unwrap();
@@ -1315,14 +1106,142 @@ mod tests {
             let mut rx = send_clear(epoch5);
             assert_pending(&mut rx).await;
             version_update_tx
-                .send(HummockVersionUpdate::PinnedVersion(version4.clone()))
+                .send(HummockVersionUpdate::PinnedVersion(Box::new(
+                    version4.clone(),
+                )))
                 .unwrap();
             assert_pending(&mut rx).await;
             version_update_tx
-                .send(HummockVersionUpdate::PinnedVersion(version5.clone()))
+                .send(HummockVersionUpdate::PinnedVersion(Box::new(
+                    version5.clone(),
+                )))
                 .unwrap();
             rx.await.unwrap();
             assert_eq!(latest_version.load().version(), &version5);
         }
     }
+
+    #[tokio::test]
+    async fn test_old_epoch_sync_fail() {
+        let epoch0 = test_epoch(233);
+
+        let initial_version = PinnedVersion::new(
+            HummockVersion::from_rpc_protobuf(&PbHummockVersion {
+                id: 1,
+                max_committed_epoch: epoch0,
+                ..Default::default()
+            }),
+            unbounded_channel().0,
+        );
+
+        let (_version_update_tx, version_update_rx) = unbounded_channel();
+
+        let epoch1 = epoch0.next_epoch();
+        let epoch2 = epoch1.next_epoch();
+        let (tx, rx) = oneshot::channel();
+        let rx = Arc::new(Mutex::new(Some(rx)));
+
+        let event_handler = HummockEventHandler::new_inner(
+            version_update_rx,
+            initial_version.clone(),
+            None,
+            mock_sstable_store().await,
+            Arc::new(HummockStateStoreMetrics::unused()),
+            &default_opts_for_test(),
+            Arc::new(move |_, info| {
+                assert_eq!(info.epochs.len(), 1);
+                let epoch = info.epochs[0];
+                match epoch {
+                    epoch if epoch == epoch1 => {
+                        let rx = rx.lock().take().unwrap();
+                        spawn(async move {
+                            rx.await.unwrap();
+                            Err(HummockError::other("fail"))
+                        })
+                    }
+                    epoch if epoch == epoch2 => spawn(async move {
+                        Ok(UploadTaskOutput {
+                            new_value_ssts: vec![],
+                            old_value_ssts: vec![],
+                            wait_poll_timer: None,
+                        })
+                    }),
+                    _ => unreachable!(),
+                }
+            }),
+            CacheRefiller::default_spawn_refill_task(),
+        );
+
+        let event_tx = event_handler.event_sender();
+
+        let send_event = |event| event_tx.send(event).unwrap();
+
+        let _join_handle = spawn(event_handler.start_hummock_event_handler_worker());
+
+        let (read_version, guard) = {
+            let (tx, rx) = oneshot::channel();
+            send_event(HummockEvent::RegisterReadVersion {
+                table_id: TEST_TABLE_ID,
+                new_read_version_sender: tx,
+                is_replicated: false,
+                vnodes: Arc::new(BitmapBuilder::filled(VirtualNode::COUNT).finish()),
+            });
+            rx.await.unwrap()
+        };
+
+        send_event(HummockEvent::InitEpoch {
+            instance_id: guard.instance_id,
+            init_epoch: epoch1,
+        });
+
+        let imm1 = gen_imm(epoch1).await;
+        read_version
+            .write()
+            .update(VersionUpdate::Staging(StagingData::ImmMem(imm1.clone())));
+
+        send_event(HummockEvent::ImmToUploader {
+            instance_id: guard.instance_id,
+            imm: imm1,
+        });
+
+        send_event(HummockEvent::LocalSealEpoch {
+            instance_id: guard.instance_id,
+            next_epoch: epoch2,
+            opts: SealCurrentEpochOptions::for_test(),
+        });
+
+        let imm2 = gen_imm(epoch2).await;
+        read_version
+            .write()
+            .update(VersionUpdate::Staging(StagingData::ImmMem(imm2.clone())));
+
+        send_event(HummockEvent::ImmToUploader {
+            instance_id: guard.instance_id,
+            imm: imm2,
+        });
+
+        let epoch3 = epoch2.next_epoch();
+        send_event(HummockEvent::LocalSealEpoch {
+            instance_id: guard.instance_id,
+            next_epoch: epoch3,
+            opts: SealCurrentEpochOptions::for_test(),
+        });
+
+        let (tx1, mut rx1) = oneshot::channel();
+        send_event(HummockEvent::SyncEpoch {
+            new_sync_epoch: epoch1,
+            sync_result_sender: tx1,
+        });
+        assert!(poll_fn(|cx| Poll::Ready(rx1.poll_unpin(cx).is_pending())).await);
+        let (tx2, mut rx2) = oneshot::channel();
+        send_event(HummockEvent::SyncEpoch {
+            new_sync_epoch: epoch2,
+            sync_result_sender: tx2,
+        });
+        assert!(poll_fn(|cx| Poll::Ready(rx2.poll_unpin(cx).is_pending())).await);
+
+        tx.send(()).unwrap();
+        rx1.await.unwrap().unwrap_err();
+        rx2.await.unwrap().unwrap_err();
+    }
 }
diff --git a/src/storage/src/hummock/event_handler/mod.rs b/src/storage/src/hummock/event_handler/mod.rs
index 9e98fac4ee3b1..bbf69ae194f72 100644
--- a/src/storage/src/hummock/event_handler/mod.rs
+++ b/src/storage/src/hummock/event_handler/mod.rs
@@ -22,7 +22,7 @@ use risingwave_hummock_sdk::{HummockEpoch, SyncResult};
 use thiserror_ext::AsReport;
 use tokio::sync::oneshot;
 
-use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferBatch;
+use crate::hummock::shared_buffer::shared_buffer_batch::{SharedBufferBatch, SharedBufferBatchId};
 use crate::hummock::HummockResult;
 use crate::mem_table::ImmutableMemtable;
 use crate::store::SealCurrentEpochOptions;
@@ -47,7 +47,7 @@ pub struct BufferWriteRequest {
 #[derive(Debug)]
 pub enum HummockVersionUpdate {
     VersionDeltas(Vec<HummockVersionDelta>),
-    PinnedVersion(HummockVersion),
+    PinnedVersion(Box<HummockVersion>),
 }
 
 pub enum HummockEvent {
@@ -57,7 +57,7 @@ pub enum HummockEvent {
     /// An epoch is going to be synced. Once the event is processed, there will be no more flush
     /// task on this epoch. Previous concurrent flush task join handle will be returned by the join
     /// handle sender.
-    AwaitSyncEpoch {
+    SyncEpoch {
         new_sync_epoch: HummockEpoch,
         sync_result_sender: oneshot::Sender<HummockResult<SyncResult>>,
     },
@@ -67,17 +67,19 @@ pub enum HummockEvent {
 
     Shutdown,
 
-    ImmToUploader(ImmutableMemtable),
+    ImmToUploader {
+        instance_id: SharedBufferBatchId,
+        imm: ImmutableMemtable,
+    },
 
-    SealEpoch {
-        epoch: HummockEpoch,
-        is_checkpoint: bool,
+    InitEpoch {
+        instance_id: LocalInstanceId,
+        init_epoch: HummockEpoch,
     },
 
     LocalSealEpoch {
         instance_id: LocalInstanceId,
-        table_id: TableId,
-        epoch: HummockEpoch,
+        next_epoch: HummockEpoch,
         opts: SealCurrentEpochOptions,
     },
 
@@ -94,7 +96,6 @@ pub enum HummockEvent {
     },
 
     DestroyReadVersion {
-        table_id: TableId,
         instance_id: LocalInstanceId,
     },
 }
@@ -104,7 +105,7 @@ impl HummockEvent {
         match self {
             HummockEvent::BufferMayFlush => "BufferMayFlush".to_string(),
 
-            HummockEvent::AwaitSyncEpoch {
+            HummockEvent::SyncEpoch {
                 new_sync_epoch,
                 sync_result_sender: _,
             } => format!("AwaitSyncEpoch epoch {} ", new_sync_epoch),
@@ -113,27 +114,25 @@ impl HummockEvent {
 
             HummockEvent::Shutdown => "Shutdown".to_string(),
 
-            HummockEvent::ImmToUploader(imm) => {
-                format!("ImmToUploader {:?}", imm)
+            HummockEvent::InitEpoch {
+                instance_id,
+                init_epoch,
+            } => {
+                format!("InitEpoch {} {}", instance_id, init_epoch)
             }
 
-            HummockEvent::SealEpoch {
-                epoch,
-                is_checkpoint,
-            } => format!(
-                "SealEpoch epoch {:?} is_checkpoint {:?}",
-                epoch, is_checkpoint
-            ),
+            HummockEvent::ImmToUploader { instance_id, imm } => {
+                format!("ImmToUploader {} {}", instance_id, imm.batch_id())
+            }
 
             HummockEvent::LocalSealEpoch {
-                epoch,
                 instance_id,
-                table_id,
+                next_epoch,
                 opts,
             } => {
                 format!(
-                    "LocalSealEpoch epoch: {}, table_id: {}, instance_id: {}, opts: {:?}",
-                    epoch, table_id.table_id, instance_id, opts
+                    "LocalSealEpoch next_epoch: {}, instance_id: {}, opts: {:?}",
+                    next_epoch, instance_id, opts
                 )
             }
 
@@ -147,13 +146,9 @@ impl HummockEvent {
                 table_id, is_replicated
             ),
 
-            HummockEvent::DestroyReadVersion {
-                table_id,
-                instance_id,
-            } => format!(
-                "DestroyReadVersion table_id {:?} instance_id {:?}",
-                table_id, instance_id
-            ),
+            HummockEvent::DestroyReadVersion { instance_id } => {
+                format!("DestroyReadVersion instance_id {:?}", instance_id)
+            }
 
             #[cfg(any(test, feature = "test"))]
             HummockEvent::FlushEvent(_) => "FlushEvent".to_string(),
@@ -170,6 +165,7 @@ impl std::fmt::Debug for HummockEvent {
 }
 
 pub type LocalInstanceId = u64;
+pub const TEST_LOCAL_INSTANCE_ID: LocalInstanceId = 233;
 pub type HummockReadVersionRef = Arc<RwLock<HummockReadVersion>>;
 pub type ReadVersionMappingType = HashMap<TableId, HashMap<LocalInstanceId, HummockReadVersionRef>>;
 pub type ReadOnlyReadVersionMapping = ReadOnlyRwLockRef<ReadVersionMappingType>;
@@ -206,7 +202,6 @@ impl Drop for LocalInstanceGuard {
             // need to handle failure
             sender
                 .send(HummockEvent::DestroyReadVersion {
-                    table_id: self.table_id,
                     instance_id: self.instance_id,
                 })
                 .unwrap_or_else(|err| {
diff --git a/src/storage/src/hummock/event_handler/refiller.rs b/src/storage/src/hummock/event_handler/refiller.rs
index 1c592fac85a2d..10ed4484917b6 100644
--- a/src/storage/src/hummock/event_handler/refiller.rs
+++ b/src/storage/src/hummock/event_handler/refiller.rs
@@ -14,13 +14,12 @@
 
 use std::collections::{HashMap, HashSet, VecDeque};
 use std::future::poll_fn;
-use std::ops::{Deref, Range};
+use std::ops::Range;
 use std::sync::{Arc, LazyLock};
 use std::task::{ready, Poll};
 use std::time::{Duration, Instant};
 
-use foyer::common::code::Key;
-use foyer::common::range::RangeBoundsExt;
+use foyer::{HybridCacheEntry, RangeBoundsExt};
 use futures::future::{join_all, try_join_all};
 use futures::{Future, FutureExt};
 use itertools::Itertools;
@@ -30,18 +29,15 @@ use prometheus::{
     register_int_gauge_with_registry, Histogram, HistogramVec, IntGauge, Registry,
 };
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
-use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_hummock_sdk::compaction_group::hummock_version_ext::SstDeltaInfo;
 use risingwave_hummock_sdk::{HummockSstableObjectId, KeyComparator};
 use thiserror_ext::AsReport;
 use tokio::sync::Semaphore;
 use tokio::task::JoinHandle;
 
-use crate::hummock::file_cache::preclude::*;
 use crate::hummock::local_version::pinned_version::PinnedVersion;
 use crate::hummock::{
-    CachedBlock, FileCacheCompression, HummockError, HummockResult, Sstable, SstableBlockIndex,
-    SstableStoreRef, TableHolder,
+    Block, HummockError, HummockResult, Sstable, SstableBlockIndex, SstableStoreRef, TableHolder,
 };
 use crate::monitor::StoreLocalStatistic;
 use crate::opts::StorageOpts;
@@ -295,9 +291,7 @@ impl CacheRefiller {
     /// Clear the queue for cache refill and return an event that merges all pending cache refill events
     /// into a single event that takes the earliest and latest version.
     pub(crate) fn clear(&mut self) -> Option<CacheRefillerEvent> {
-        let Some(last_item) = self.queue.pop_back() else {
-            return None;
-        };
+        let last_item = self.queue.pop_back()?;
         let mut event = last_item.event;
         while let Some(item) = self.queue.pop_back() {
             assert_eq!(
@@ -392,7 +386,7 @@ impl CacheRefillTask {
     fn get_units_to_refill_by_inheritance(
         context: &CacheRefillContext,
         ssts: &[TableHolder],
-        parent_ssts: &[impl Deref<Target = Sstable>],
+        parent_ssts: impl IntoIterator<Item = HybridCacheEntry<HummockSstableObjectId, Box<Sstable>>>,
     ) -> HashSet<SstableUnit> {
         let mut res = HashSet::default();
 
@@ -561,12 +555,12 @@ impl CacheRefillTask {
             }
         });
         let parent_ssts = match try_join_all(futures).await {
-            Ok(parent_ssts) => parent_ssts.into_iter().flatten().collect_vec(),
+            Ok(parent_ssts) => parent_ssts.into_iter().flatten(),
             Err(e) => {
                 return tracing::error!(error = %e.as_report(), "get old meta from cache error")
             }
         };
-        let units = Self::get_units_to_refill_by_inheritance(context, &holders, &parent_ssts);
+        let units = Self::get_units_to_refill_by_inheritance(context, &holders, parent_ssts);
 
         let ssts: HashMap<HummockSstableObjectId, TableHolder> =
             holders.into_iter().map(|meta| (meta.id, meta)).collect();
@@ -598,8 +592,7 @@ impl CacheRefillTask {
         let blocks = unit.blks.size().unwrap();
 
         let mut tasks = vec![];
-        let mut writers = Vec::with_capacity(blocks);
-        let mut ranges = Vec::with_capacity(blocks);
+        let mut contexts = Vec::with_capacity(blocks);
         let mut admits = 0;
 
         let (range_first, _) = sst.calculate_block_info(unit.blks.start);
@@ -611,25 +604,22 @@ impl CacheRefillTask {
             .inc_by(range.size().unwrap() as u64);
 
         for blk in unit.blks {
-            let (range, _uncompressed_capacity) = sst.calculate_block_info(blk);
+            let (range, uncompressed_capacity) = sst.calculate_block_info(blk);
             let key = SstableBlockIndex {
                 sst_id: sst.id,
                 block_idx: blk as u64,
             };
-            // see `CachedBlock::serialized_len()`
-            let mut writer = sstable_store
-                .data_file_cache()
-                .writer(key, key.serialized_len() + 1 + 8 + range.size().unwrap());
 
-            if writer.judge() {
+            let mut writer = sstable_store.block_cache().storage_writer(key);
+
+            if writer.pick() {
                 admits += 1;
             }
 
-            writers.push(writer);
-            ranges.push(range);
+            contexts.push((writer, range, uncompressed_capacity))
         }
 
-        if admits as f64 / writers.len() as f64 >= threshold {
+        if admits as f64 / contexts.len() as f64 >= threshold {
             let task = async move {
                 GLOBAL_CACHE_REFILL_METRICS.data_refill_attempts_total.inc();
 
@@ -646,23 +636,14 @@ impl CacheRefillTask {
                     .read(&sstable_store.get_sst_data_path(sst.id), range.clone())
                     .await?;
                 let mut futures = vec![];
-                for (mut writer, r) in writers.into_iter().zip_eq_fast(ranges) {
+                for (w, r, uc) in contexts {
                     let offset = r.start - range.start;
                     let len = r.end - r.start;
                     let bytes = data.slice(offset..offset + len);
-
                     let future = async move {
-                        let value = CachedBlock::Fetched {
-                            bytes,
-                            uncompressed_capacity: writer.weight() - writer.key().serialized_len(),
-                        };
-
-                        writer.force();
-                        // TODO(MrCroxx): compress if raw is not compressed?
-                        // skip compression for it may already be compressed.
-                        writer.set_compression(FileCacheCompression::None);
-                        let res = writer.finish(value).await.map_err(HummockError::file_cache);
-                        if matches!(res, Ok(true)) {
+                        let value = Box::new(Block::decode(bytes, uc)?);
+                        // The entry should always be `Some(..)`, use if here for compatible.
+                        if let Some(_entry) = w.force().insert(value) {
                             GLOBAL_CACHE_REFILL_METRICS
                                 .data_refill_success_bytes
                                 .inc_by(len as u64);
@@ -670,7 +651,7 @@ impl CacheRefillTask {
                                 .data_refill_block_success_total
                                 .inc();
                         }
-                        res
+                        Ok::<_, HummockError>(())
                     };
                     futures.push(future);
                 }
diff --git a/src/storage/src/hummock/event_handler/uploader.rs b/src/storage/src/hummock/event_handler/uploader.rs
index 30be483a72f61..f768aa23dcd89 100644
--- a/src/storage/src/hummock/event_handler/uploader.rs
+++ b/src/storage/src/hummock/event_handler/uploader.rs
@@ -12,52 +12,64 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::hash_map::Entry;
-use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
+use std::cmp::Ordering;
+use std::collections::btree_map::Entry;
+use std::collections::{BTreeMap, HashMap, VecDeque};
 use std::fmt::{Debug, Display, Formatter};
 use std::future::{poll_fn, Future};
-use std::mem::swap;
+use std::mem::{replace, swap, take};
 use std::pin::Pin;
-use std::sync::atomic::AtomicUsize;
-use std::sync::atomic::Ordering::Relaxed;
-use std::sync::{Arc, LazyLock};
+use std::sync::Arc;
 use std::task::{ready, Context, Poll};
 
 use futures::future::{try_join_all, TryJoinAll};
 use futures::FutureExt;
 use itertools::Itertools;
-use more_asserts::{assert_ge, assert_gt, assert_le};
+use more_asserts::{assert_ge, assert_gt};
 use prometheus::core::{AtomicU64, GenericGauge};
 use prometheus::{HistogramTimer, IntGauge};
 use risingwave_common::buffer::BitmapBuilder;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
-use risingwave_hummock_sdk::key::EPOCH_LEN;
+use risingwave_common::must_match;
 use risingwave_hummock_sdk::table_watermark::{
     TableWatermarks, VnodeWatermark, WatermarkDirection,
 };
-use risingwave_hummock_sdk::{CompactionGroupId, HummockEpoch, LocalSstableInfo};
+use risingwave_hummock_sdk::{CompactionGroupId, HummockEpoch, LocalSstableInfo, SyncResult};
 use thiserror_ext::AsReport;
+use tokio::sync::oneshot;
 use tokio::task::JoinHandle;
-use tracing::{debug, error, info};
+use tracing::{debug, error, info, warn};
 
-use crate::hummock::compactor::{
-    await_tree_key, merge_imms_in_memory, CompactionAwaitTreeRegRef, CompactionExecutor,
-};
-use crate::hummock::event_handler::hummock_event_handler::BufferTracker;
+use crate::hummock::event_handler::hummock_event_handler::{send_sync_result, BufferTracker};
+use crate::hummock::event_handler::uploader::uploader_imm::UploaderImm;
 use crate::hummock::event_handler::LocalInstanceId;
 use crate::hummock::local_version::pinned_version::PinnedVersion;
+use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferBatchId;
 use crate::hummock::store::version::StagingSstableInfo;
-use crate::hummock::utils::MemoryTracker;
 use crate::hummock::{HummockError, HummockResult, ImmutableMemtable};
 use crate::mem_table::ImmId;
 use crate::monitor::HummockStateStoreMetrics;
 use crate::opts::StorageOpts;
+use crate::store::SealCurrentEpochOptions;
+
+/// Take epoch data inclusively before `epoch` out from `data`
+fn take_before_epoch<T>(
+    data: &mut BTreeMap<HummockEpoch, T>,
+    epoch: HummockEpoch,
+) -> BTreeMap<HummockEpoch, T> {
+    let mut before_epoch_data = data.split_off(&(epoch + 1));
+    swap(&mut before_epoch_data, data);
+    before_epoch_data
+}
 
-pub type UploadTaskPayload = Vec<ImmutableMemtable>;
+type UploadTaskInput = HashMap<LocalInstanceId, Vec<UploaderImm>>;
+pub type UploadTaskPayload = HashMap<LocalInstanceId, Vec<ImmutableMemtable>>;
 
+#[derive(Debug)]
 pub struct UploadTaskOutput {
-    pub ssts: Vec<LocalSstableInfo>,
+    pub new_value_ssts: Vec<LocalSstableInfo>,
+    pub old_value_ssts: Vec<LocalSstableInfo>,
     pub wait_poll_timer: Option<HistogramTimer>,
 }
 pub type SpawnUploadTask = Arc<
@@ -67,58 +79,11 @@ pub type SpawnUploadTask = Arc<
         + 'static,
 >;
 
-pub type SpawnMergingTask = Arc<
-    dyn Fn(
-            TableId,
-            LocalInstanceId,
-            Vec<ImmutableMemtable>,
-            Option<MemoryTracker>,
-        ) -> JoinHandle<ImmutableMemtable>
-        + Send
-        + Sync
-        + 'static,
->;
-
-pub(crate) fn default_spawn_merging_task(
-    compaction_executor: Arc<CompactionExecutor>,
-    await_tree_reg: Option<CompactionAwaitTreeRegRef>,
-) -> SpawnMergingTask {
-    Arc::new(move |table_id, instance_id, imms, tracker| {
-        compaction_executor.spawn({
-            static NEXT_MERGING_TASK_ID: LazyLock<AtomicUsize> =
-                LazyLock::new(|| AtomicUsize::new(0));
-            let tree_root = await_tree_reg.as_ref().map(|reg| {
-                let merging_task_id = NEXT_MERGING_TASK_ID.fetch_add(1, Relaxed);
-                reg.register(
-                    await_tree_key::MergingTask {
-                        id: merging_task_id,
-                    },
-                    format!(
-                        "Merging Imm {:?} {:?} {:?}",
-                        table_id,
-                        instance_id,
-                        imms.iter()
-                            .flat_map(|imm| imm.epochs().iter())
-                            .copied()
-                            .collect::<BTreeSet<_>>()
-                    ),
-                )
-            });
-            let future = merge_imms_in_memory(table_id, instance_id, imms, tracker);
-            if let Some(root) = tree_root {
-                root.instrument(future).left_future()
-            } else {
-                future.right_future()
-            }
-        })
-    })
-}
-
 #[derive(Clone)]
 pub struct UploadTaskInfo {
     pub task_size: usize,
     pub epochs: Vec<HummockEpoch>,
-    pub imm_ids: Vec<ImmId>,
+    pub imm_ids: HashMap<LocalInstanceId, Vec<ImmId>>,
     pub compaction_group_index: Arc<HashMap<TableId, CompactionGroupId>>,
 }
 
@@ -142,86 +107,73 @@ impl Debug for UploadTaskInfo {
     }
 }
 
-/// A wrapper for a uploading task that compacts and uploads the imm payload. Task context are
-/// stored so that when the task fails, it can be re-tried.
-struct UploadingTask {
-    payload: UploadTaskPayload,
-    join_handle: JoinHandle<HummockResult<UploadTaskOutput>>,
-    task_info: UploadTaskInfo,
-    spawn_upload_task: SpawnUploadTask,
-    task_size_guard: GenericGauge<AtomicU64>,
-    task_count_guard: IntGauge,
-}
+mod uploader_imm {
+    use std::fmt::Formatter;
+    use std::ops::Deref;
 
-pub struct MergeImmTaskOutput {
-    /// Input imm ids of the merging task. Larger imm ids at the front.
-    pub imm_ids: Vec<ImmId>,
-    pub table_id: TableId,
-    pub instance_id: LocalInstanceId,
-    pub merged_imm: ImmutableMemtable,
-}
+    use prometheus::core::{AtomicU64, GenericGauge};
 
-// A future that merges multiple immutable memtables to a single immutable memtable.
-struct MergingImmTask {
-    table_id: TableId,
-    instance_id: LocalInstanceId,
-    input_imms: Vec<ImmutableMemtable>,
-    join_handle: JoinHandle<ImmutableMemtable>,
-}
+    use crate::hummock::event_handler::uploader::UploaderContext;
+    use crate::mem_table::ImmutableMemtable;
 
-impl Debug for MergingImmTask {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("MergingImmTask")
-            .field("table_id", &self.table_id)
-            .field("instance_id", &self.instance_id)
-            .field("input_imms", &self.input_imms)
-            .finish()
+    pub(super) struct UploaderImm {
+        inner: ImmutableMemtable,
+        size_guard: GenericGauge<AtomicU64>,
     }
-}
 
-impl MergingImmTask {
-    fn new(
-        table_id: TableId,
-        instance_id: LocalInstanceId,
-        imms: Vec<ImmutableMemtable>,
-        memory_tracker: Option<MemoryTracker>,
-        context: &UploaderContext,
-    ) -> Self {
-        assert!(imms.iter().rev().map(|imm| imm.batch_id()).is_sorted());
-        let input_imms = imms.clone();
-        let join_handle = (context.spawn_merging_task)(table_id, instance_id, imms, memory_tracker);
+    impl UploaderImm {
+        pub(super) fn new(imm: ImmutableMemtable, context: &UploaderContext) -> Self {
+            let size = imm.size();
+            let size_guard = context.stats.uploader_imm_size.clone();
+            size_guard.add(size as _);
+            Self {
+                inner: imm,
+                size_guard,
+            }
+        }
 
-        MergingImmTask {
-            table_id,
-            instance_id,
-            input_imms,
-            join_handle,
+        #[cfg(test)]
+        pub(super) fn for_test(imm: ImmutableMemtable) -> Self {
+            Self {
+                inner: imm,
+                size_guard: GenericGauge::new("test", "test").unwrap(),
+            }
         }
     }
 
-    /// Poll the result of the merge task
-    fn poll_result(&mut self, cx: &mut Context<'_>) -> Poll<ImmutableMemtable> {
-        Poll::Ready(match ready!(self.join_handle.poll_unpin(cx)) {
-            Ok(task_result) => task_result,
-            Err(err) => {
-                panic!(
-                    "failed to join merging task: {:?} {:?}",
-                    err.as_report(),
-                    self
-                );
-            }
-        })
+    impl std::fmt::Debug for UploaderImm {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            self.inner.fmt(f)
+        }
     }
-}
 
-impl Future for MergingImmTask {
-    type Output = ImmutableMemtable;
+    impl Deref for UploaderImm {
+        type Target = ImmutableMemtable;
 
-    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        self.poll_result(cx)
+        fn deref(&self) -> &Self::Target {
+            &self.inner
+        }
+    }
+
+    impl Drop for UploaderImm {
+        fn drop(&mut self) {
+            self.size_guard.sub(self.inner.size() as _);
+        }
     }
 }
 
+/// A wrapper for a uploading task that compacts and uploads the imm payload. Task context are
+/// stored so that when the task fails, it can be re-tried.
+struct UploadingTask {
+    // newer data at the front
+    input: UploadTaskInput,
+    join_handle: JoinHandle<HummockResult<UploadTaskOutput>>,
+    task_info: UploadTaskInfo,
+    spawn_upload_task: SpawnUploadTask,
+    task_size_guard: GenericGauge<AtomicU64>,
+    task_count_guard: IntGauge,
+}
+
 impl Drop for UploadingTask {
     fn drop(&mut self) {
         self.task_size_guard.sub(self.task_info.task_size as u64);
@@ -232,29 +184,59 @@ impl Drop for UploadingTask {
 impl Debug for UploadingTask {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("UploadingTask")
-            .field("payload", &self.payload)
+            .field("input", &self.input)
             .field("task_info", &self.task_info)
             .finish()
     }
 }
 
+fn get_payload_imm_ids(
+    payload: &UploadTaskPayload,
+) -> HashMap<LocalInstanceId, Vec<SharedBufferBatchId>> {
+    payload
+        .iter()
+        .map(|(instance_id, imms)| {
+            (
+                *instance_id,
+                imms.iter().map(|imm| imm.batch_id()).collect_vec(),
+            )
+        })
+        .collect()
+}
+
 impl UploadingTask {
     // INFO logs will be enabled for task with size exceeding 50MB.
     const LOG_THRESHOLD_FOR_UPLOAD_TASK_SIZE: usize = 50 * (1 << 20);
 
-    fn new(payload: UploadTaskPayload, context: &UploaderContext) -> Self {
-        assert!(!payload.is_empty());
-        let mut epochs = payload
+    fn input_to_payload(input: &UploadTaskInput) -> UploadTaskPayload {
+        input
+            .iter()
+            .map(|(instance_id, imms)| {
+                (
+                    *instance_id,
+                    imms.iter().map(|imm| (**imm).clone()).collect(),
+                )
+            })
+            .collect()
+    }
+
+    fn new(input: UploadTaskInput, context: &UploaderContext) -> Self {
+        assert!(!input.is_empty());
+        let mut epochs = input
             .iter()
-            .flat_map(|imm| imm.epochs().clone())
+            .flat_map(|(_, imms)| imms.iter().flat_map(|imm| imm.epochs().iter().cloned()))
             .sorted()
             .dedup()
             .collect_vec();
 
         // reverse to make newer epochs comes first
         epochs.reverse();
-        let imm_ids = payload.iter().map(|imm| imm.batch_id()).collect_vec();
-        let task_size = payload.iter().map(|imm| imm.size()).sum();
+        let payload = Self::input_to_payload(&input);
+        let imm_ids = get_payload_imm_ids(&payload);
+        let task_size = input
+            .values()
+            .map(|imms| imms.iter().map(|imm| imm.size()).sum::<usize>())
+            .sum();
         let task_info = UploadTaskInfo {
             task_size,
             epochs,
@@ -270,10 +252,10 @@ impl UploadingTask {
         } else {
             debug!("start upload task: {:?}", task_info);
         }
-        let join_handle = (context.spawn_upload_task)(payload.clone(), task_info.clone());
+        let join_handle = (context.spawn_upload_task)(payload, task_info.clone());
         context.stats.uploader_uploading_task_count.inc();
         Self {
-            payload,
+            input,
             join_handle,
             task_info,
             spawn_upload_task: context.spawn_upload_task.clone(),
@@ -296,7 +278,8 @@ impl UploadingTask {
                 .inspect_err(|e| error!(task_info = ?self.task_info, err = ?e.as_report(), "upload task failed"))
                 .map(|output| {
                     StagingSstableInfo::new(
-                        output.ssts,
+                        output.new_value_ssts,
+                        output.old_value_ssts,
                         self.task_info.epochs.clone(),
                         self.task_info.imm_ids.clone(),
                         self.task_info.task_size,
@@ -322,8 +305,10 @@ impl UploadingTask {
                         task_info = ?self.task_info,
                         "a flush task failed, start retry",
                     );
-                    self.join_handle =
-                        (self.spawn_upload_task)(self.payload.clone(), self.task_info.clone());
+                    self.join_handle = (self.spawn_upload_task)(
+                        Self::input_to_payload(&self.input),
+                        self.task_info.clone(),
+                    );
                     // It is important not to return Poll::pending here immediately, because the new
                     // join_handle is not polled yet, and will not awake the current task when
                     // succeed. It will be polled in the next loop iteration.
@@ -356,11 +341,6 @@ struct SpilledData {
 }
 
 impl SpilledData {
-    #[cfg(test)]
-    fn is_empty(&self) -> bool {
-        self.uploading_tasks.is_empty() && self.uploaded_data.is_empty()
-    }
-
     fn add_task(&mut self, task: UploadingTask) {
         self.uploading_tasks.push_front(task);
     }
@@ -379,26 +359,24 @@ impl SpilledData {
         }
     }
 
-    fn clear(&mut self) {
-        for task in self.uploading_tasks.drain(..) {
+    fn abort(self) {
+        for task in self.uploading_tasks {
             task.join_handle.abort();
         }
-        self.uploaded_data.clear();
     }
 }
 
 #[derive(Default, Debug)]
-struct UnsealedEpochData {
-    // newer data at the front
-    imms: VecDeque<ImmutableMemtable>,
+struct EpochData {
     spilled_data: SpilledData,
-
-    table_watermarks: HashMap<TableId, (WatermarkDirection, Vec<VnodeWatermark>, BitmapBuilder)>,
 }
 
-impl UnsealedEpochData {
-    fn flush(&mut self, context: &UploaderContext) {
-        let imms = self.imms.drain(..).collect_vec();
+impl EpochData {
+    fn flush(
+        &mut self,
+        context: &UploaderContext,
+        imms: HashMap<LocalInstanceId, Vec<UploaderImm>>,
+    ) -> usize {
         if !imms.is_empty() {
             let task = UploadingTask::new(imms, context);
             context.stats.spill_task_counts_from_unsealed.inc();
@@ -407,13 +385,19 @@ impl UnsealedEpochData {
                 .spill_task_size_from_unsealed
                 .inc_by(task.task_info.task_size as u64);
             info!("Spill unsealed data. Task: {}", task.get_task_info());
+            let size = task.task_info.task_size;
             self.spilled_data.add_task(task);
+            size
+        } else {
+            0
         }
     }
+}
 
+impl TableUnsyncData {
     fn add_table_watermarks(
         &mut self,
-        table_id: TableId,
+        epoch: HummockEpoch,
         table_watermarks: Vec<VnodeWatermark>,
         direction: WatermarkDirection,
     ) {
@@ -432,65 +416,50 @@ impl UnsealedEpochData {
                 }
             }
         }
-        match self.table_watermarks.entry(table_id) {
-            Entry::Occupied(mut entry) => {
-                let (prev_direction, prev_watermarks, vnode_bitmap) = entry.get_mut();
+        match &mut self.table_watermarks {
+            Some((prev_direction, prev_watermarks)) => {
                 assert_eq!(
                     *prev_direction, direction,
                     "table id {} new watermark direction not match with previous",
-                    table_id
+                    self.table_id
                 );
-                apply_new_vnodes(vnode_bitmap, &table_watermarks);
-                prev_watermarks.extend(table_watermarks);
+                match prev_watermarks.entry(epoch) {
+                    Entry::Occupied(mut entry) => {
+                        let (prev_watermarks, vnode_bitmap) = entry.get_mut();
+                        apply_new_vnodes(vnode_bitmap, &table_watermarks);
+                        prev_watermarks.extend(table_watermarks);
+                    }
+                    Entry::Vacant(entry) => {
+                        let mut vnode_bitmap = BitmapBuilder::zeroed(VirtualNode::COUNT);
+                        apply_new_vnodes(&mut vnode_bitmap, &table_watermarks);
+                        entry.insert((table_watermarks, vnode_bitmap));
+                    }
+                }
             }
-            Entry::Vacant(entry) => {
+            None => {
                 let mut vnode_bitmap = BitmapBuilder::zeroed(VirtualNode::COUNT);
                 apply_new_vnodes(&mut vnode_bitmap, &table_watermarks);
-                entry.insert((direction, table_watermarks, vnode_bitmap));
+                self.table_watermarks = Some((
+                    direction,
+                    BTreeMap::from_iter([(epoch, (table_watermarks, vnode_bitmap))]),
+                ));
             }
         }
     }
 }
 
 #[derive(Default)]
-/// Data at the sealed stage. We will ensure that data in `imms` are newer than the data in the
-/// `spilled_data`, and that data in the `uploading_tasks` in `spilled_data` are newer than data in
-/// the `uploaded_data` in `spilled_data`.
-struct SealedData {
+struct SyncDataBuilder {
     // newer epochs come first
     epochs: VecDeque<HummockEpoch>,
 
-    // store the output of merge task that will be feed into `flush_imms` procedure
-    merged_imms: VecDeque<ImmutableMemtable>,
-
-    // Sealed imms grouped by table shard.
-    // newer data (larger imm id) at the front
-    imms_by_table_shard: HashMap<(TableId, LocalInstanceId), VecDeque<ImmutableMemtable>>,
-
-    // Merging tasks generated from sealed imms
-    // it should be safe to directly drop these tasks
-    merging_tasks: VecDeque<MergingImmTask>,
-
     spilled_data: SpilledData,
 
     table_watermarks: HashMap<TableId, TableWatermarks>,
 }
 
-impl SealedData {
-    fn clear(&mut self) {
-        self.epochs.clear();
-
-        self.spilled_data.clear();
-        self.imms_by_table_shard.clear();
-        self.merged_imms.clear();
-        self.merging_tasks
-            .iter()
-            .for_each(|task| task.join_handle.abort());
-        self.merging_tasks.clear();
-        self.table_watermarks.clear();
-    }
-
-    /// Add the data of a newly sealed epoch.
+impl SyncDataBuilder {
+    /// Add the data of a new epoch.
     ///
     /// Note: it may happen that, for example, currently we hold `imms` and `spilled_data` of epoch
     /// 3,  and after we add the spilled data of epoch 4, both `imms` and `spilled_data` hold data
@@ -500,9 +469,9 @@ impl SealedData {
     /// data of `imms` must not overlap with the epoch 4 data of `spilled_data`. The explanation is
     /// as followed:
     ///
-    /// First, unsealed data has 3 stages, from earlier to later, imms, uploading task, and
-    /// uploaded. When we try to spill unsealed data, we first pick the imms of older epoch until
-    /// the imms of older epoch are all picked. When we try to poll the uploading tasks of unsealed
+    /// First, unsync data has 3 stages, from earlier to later, imms, uploading task, and
+    /// uploaded. When we try to spill unsync data, we first pick the imms of older epoch until
+    /// the imms of older epoch are all picked. When we try to poll the uploading tasks of unsync
     /// data, we first poll the task of older epoch, until there is no uploading task in older
     /// epoch. Therefore, we can reach that, if two data are in the same stage, but
     /// different epochs, data in the older epoch will always enter the next stage earlier than data
@@ -516,29 +485,19 @@ impl SealedData {
     /// Based on the two points above, we can reach that, if two data of a same key appear in
     /// different epochs, the data of older epoch will not appear at a later stage than the data
     /// of newer epoch. Therefore, we can safely merge the data of each stage when we seal an epoch.
-    fn seal_new_epoch(&mut self, epoch: HummockEpoch, mut unseal_epoch_data: UnsealedEpochData) {
-        if let Some(prev_max_sealed_epoch) = self.epochs.front() {
+    fn add_new_epoch(&mut self, epoch: HummockEpoch, mut unseal_epoch_data: EpochData) {
+        if let Some(prev_max_epoch) = self.epochs.front() {
             assert!(
-                epoch > *prev_max_sealed_epoch,
-                "epoch {} to seal not greater than prev max sealed epoch {}",
+                epoch > *prev_max_epoch,
+                "epoch {} to seal not greater than prev max epoch {}",
                 epoch,
-                prev_max_sealed_epoch
+                prev_max_epoch
             );
         }
 
-        // rearrange sealed imms by table shard and in epoch descending order
-        for imm in unseal_epoch_data.imms.into_iter().rev() {
-            let queue = self
-                .imms_by_table_shard
-                .entry((imm.table_id, imm.instance_id))
-                .or_default();
-            if let Some(front) = queue.front() {
-                assert_gt!(imm.batch_id(), front.batch_id());
-            }
-            queue.push_front(imm);
-        }
-
         self.epochs.push_front(epoch);
+        // for each local instance, earlier data must be spilled at earlier epoch. Therefore, since we add spill data from old epoch
+        // to new epoch,
         unseal_epoch_data
             .spilled_data
             .uploading_tasks
@@ -549,135 +508,403 @@ impl SealedData {
             .append(&mut self.spilled_data.uploaded_data);
         self.spilled_data.uploading_tasks = unseal_epoch_data.spilled_data.uploading_tasks;
         self.spilled_data.uploaded_data = unseal_epoch_data.spilled_data.uploaded_data;
-        for (table_id, (direction, watermarks, _)) in unseal_epoch_data.table_watermarks {
-            match self.table_watermarks.entry(table_id) {
-                Entry::Occupied(mut entry) => {
-                    entry.get_mut().add_new_epoch_watermarks(
+    }
+
+    fn add_table_watermarks(
+        &mut self,
+        table_id: TableId,
+        direction: WatermarkDirection,
+        watermarks: impl Iterator<Item = (HummockEpoch, Vec<VnodeWatermark>)>,
+    ) {
+        let mut table_watermarks: Option<TableWatermarks> = None;
+        for (epoch, watermarks) in watermarks {
+            match &mut table_watermarks {
+                Some(prev_watermarks) => {
+                    prev_watermarks.add_new_epoch_watermarks(
                         epoch,
                         Arc::from(watermarks),
                         direction,
                     );
                 }
-                Entry::Vacant(entry) => {
-                    entry.insert(TableWatermarks::single_epoch(epoch, watermarks, direction));
+                None => {
+                    table_watermarks =
+                        Some(TableWatermarks::single_epoch(epoch, watermarks, direction));
                 }
-            };
+            }
+        }
+        if let Some(table_watermarks) = table_watermarks {
+            assert!(self
+                .table_watermarks
+                .insert(table_id, table_watermarks)
+                .is_none());
         }
     }
 
-    fn add_merged_imm(&mut self, merged_imm: &ImmutableMemtable) {
-        // add merged_imm to merged_imms
-        self.merged_imms.push_front(merged_imm.clone());
+    fn flush(&mut self, context: &UploaderContext, payload: UploadTaskInput) {
+        if !payload.is_empty() {
+            let task = UploadingTask::new(payload, context);
+            self.spilled_data.add_task(task);
+        }
     }
+}
 
-    fn drop_merging_tasks(&mut self) {
-        // pop from newest merging task to restore candidate imms back
-        while let Some(task) = self.merging_tasks.pop_front() {
-            // cancel the task
-            task.join_handle.abort();
-            self.imms_by_table_shard
-                .get_mut(&(task.table_id, task.instance_id))
-                .unwrap()
-                .extend(task.input_imms.into_iter());
+struct LocalInstanceEpochData {
+    epoch: HummockEpoch,
+    // newer data comes first.
+    imms: VecDeque<UploaderImm>,
+    has_spilled: bool,
+}
+
+impl LocalInstanceEpochData {
+    fn new(epoch: HummockEpoch) -> Self {
+        Self {
+            epoch,
+            imms: VecDeque::new(),
+            has_spilled: false,
         }
     }
 
-    // Flush can be triggered by either a sync_epoch or a spill (`may_flush`) request.
-    fn flush(&mut self, context: &UploaderContext, is_spilled: bool) {
-        // drop unfinished merging tasks
-        self.drop_merging_tasks();
+    fn epoch(&self) -> HummockEpoch {
+        self.epoch
+    }
 
-        // group imms by epoch and order by epoch
-        let mut imms_by_epoch: BTreeMap<HummockEpoch, Vec<ImmutableMemtable>> = BTreeMap::new();
-        self.imms_by_table_shard.drain().for_each(|(_, imms)| {
-            for imm in imms {
-                debug_assert_eq!(imm.max_epoch(), imm.min_epoch());
-                imms_by_epoch.entry(imm.max_epoch()).or_default().push(imm);
-            }
-        });
+    fn add_imm(&mut self, imm: UploaderImm) {
+        assert_eq!(imm.max_epoch(), imm.min_epoch());
+        assert_eq!(self.epoch, imm.min_epoch());
+        if let Some(prev_imm) = self.imms.front() {
+            assert_gt!(imm.batch_id(), prev_imm.batch_id());
+        }
+        self.imms.push_front(imm);
+    }
 
-        // When ImmMerged, we have removed those imms that have been merged from the
-        // `self.imms`, thus we need to feed merged_imms into the `flush` procedure
-        // to complete a checkpoint.
-        let merged_imms = self.merged_imms.drain(..);
-
-        // newer epoch comes first, the order is matter for cleaning the staging imms
-        // when adding a staging sst to the `StagingVersion`
-        let payload = imms_by_epoch
-            .into_iter()
-            .rev()
-            // in `imms`, newer data comes first
-            .flat_map(|(_epoch, imms)| imms)
-            .chain(merged_imms)
-            .collect_vec();
+    fn is_empty(&self) -> bool {
+        self.imms.is_empty()
+    }
+}
 
-        if !payload.is_empty() {
-            let task = UploadingTask::new(payload, context);
-            if is_spilled {
-                context.stats.spill_task_counts_from_sealed.inc();
-                context
-                    .stats
-                    .spill_task_size_from_sealed
-                    .inc_by(task.task_info.task_size as u64);
-                info!("Spill sealed data. Task: {}", task.get_task_info());
-            }
-            self.spilled_data.add_task(task);
+struct LocalInstanceUnsyncData {
+    table_id: TableId,
+    instance_id: LocalInstanceId,
+    // None means that the current instance should have stopped advancing
+    current_epoch_data: Option<LocalInstanceEpochData>,
+    // newer data comes first.
+    sealed_data: VecDeque<LocalInstanceEpochData>,
+    // newer data comes first
+    flushing_imms: VecDeque<SharedBufferBatchId>,
+}
+
+impl LocalInstanceUnsyncData {
+    fn new(table_id: TableId, instance_id: LocalInstanceId, init_epoch: HummockEpoch) -> Self {
+        Self {
+            table_id,
+            instance_id,
+            current_epoch_data: Some(LocalInstanceEpochData::new(init_epoch)),
+            sealed_data: VecDeque::new(),
+            flushing_imms: Default::default(),
         }
     }
 
-    fn poll_success_merge_imm(&mut self, cx: &mut Context<'_>) -> Poll<Option<MergeImmTaskOutput>> {
-        // only poll the oldest merge task if there is any
-        if let Some(task) = self.merging_tasks.back_mut() {
-            let merged_imm = ready!(task.poll_unpin(cx));
+    fn add_imm(&mut self, imm: UploaderImm) {
+        assert_eq!(self.table_id, imm.table_id);
+        self.current_epoch_data
+            .as_mut()
+            .expect("should be Some when adding new imm")
+            .add_imm(imm);
+    }
+
+    fn local_seal_epoch(&mut self, next_epoch: HummockEpoch) -> HummockEpoch {
+        let data = self
+            .current_epoch_data
+            .as_mut()
+            .expect("should be Some when seal new epoch");
+        let current_epoch = data.epoch;
+        debug!(
+            instance_id = self.instance_id,
+            next_epoch, current_epoch, "local seal epoch"
+        );
+        assert_gt!(next_epoch, current_epoch);
+        let epoch_data = replace(data, LocalInstanceEpochData::new(next_epoch));
+        if !epoch_data.is_empty() {
+            self.sealed_data.push_front(epoch_data);
+        }
+        current_epoch
+    }
 
-            // pop the finished task
-            let task = self.merging_tasks.pop_back().expect("must exist");
+    // imm_ids from old to new, which means in ascending order
+    fn ack_flushed(&mut self, imm_ids: impl Iterator<Item = SharedBufferBatchId>) {
+        for imm_id in imm_ids {
+            assert_eq!(self.flushing_imms.pop_back().expect("should exist"), imm_id);
+        }
+    }
 
-            Poll::Ready(Some(MergeImmTaskOutput {
-                imm_ids: task.input_imms.iter().map(|imm| imm.batch_id()).collect(),
-                table_id: task.table_id,
-                instance_id: task.instance_id,
-                merged_imm,
-            }))
+    fn spill(&mut self, epoch: HummockEpoch) -> Vec<UploaderImm> {
+        let imms = if let Some(oldest_sealed_epoch) = self.sealed_data.back() {
+            match oldest_sealed_epoch.epoch.cmp(&epoch) {
+                Ordering::Less => {
+                    unreachable!(
+                        "should not spill at this epoch because there \
+                    is unspilled data in previous epoch: prev epoch {}, spill epoch {}",
+                        oldest_sealed_epoch.epoch, epoch
+                    );
+                }
+                Ordering::Equal => {
+                    let epoch_data = self.sealed_data.pop_back().unwrap();
+                    assert_eq!(epoch, epoch_data.epoch);
+                    epoch_data.imms
+                }
+                Ordering::Greater => VecDeque::new(),
+            }
         } else {
-            Poll::Ready(None)
+            let Some(current_epoch_data) = &mut self.current_epoch_data else {
+                return Vec::new();
+            };
+            match current_epoch_data.epoch.cmp(&epoch) {
+                Ordering::Less => {
+                    assert!(
+                        current_epoch_data.imms.is_empty(),
+                        "should not spill at this epoch because there \
+                    is unspilled data in current epoch epoch {}, spill epoch {}",
+                        current_epoch_data.epoch,
+                        epoch
+                    );
+                    VecDeque::new()
+                }
+                Ordering::Equal => {
+                    if !current_epoch_data.imms.is_empty() {
+                        current_epoch_data.has_spilled = true;
+                        take(&mut current_epoch_data.imms)
+                    } else {
+                        VecDeque::new()
+                    }
+                }
+                Ordering::Greater => VecDeque::new(),
+            }
+        };
+        self.add_flushing_imm(imms.iter().rev().map(|imm| imm.batch_id()));
+        imms.into_iter().collect()
+    }
+
+    fn add_flushing_imm(&mut self, imm_ids: impl Iterator<Item = SharedBufferBatchId>) {
+        for imm_id in imm_ids {
+            if let Some(prev_imm_id) = self.flushing_imms.front() {
+                assert_gt!(imm_id, *prev_imm_id);
+            }
+            self.flushing_imms.push_front(imm_id);
         }
     }
 
-    /// Clear self and return the current sealed data
-    fn drain(&mut self) -> SealedData {
-        let mut ret = SealedData::default();
-        swap(&mut ret, self);
+    // start syncing the imm inclusively before the `epoch`
+    // returning data with newer data coming first
+    fn sync(&mut self, epoch: HummockEpoch) -> Vec<UploaderImm> {
+        // firstly added from old to new
+        let mut ret = Vec::new();
+        while let Some(epoch_data) = self.sealed_data.back()
+            && epoch_data.epoch() <= epoch
+        {
+            let imms = self.sealed_data.pop_back().expect("checked exist").imms;
+            self.add_flushing_imm(imms.iter().rev().map(|imm| imm.batch_id()));
+            ret.extend(imms.into_iter().rev());
+        }
+        // reverse so that newer data comes first
+        ret.reverse();
+        if let Some(latest_epoch_data) = &self.current_epoch_data {
+            if latest_epoch_data.epoch <= epoch {
+                assert!(self.sealed_data.is_empty());
+                assert!(latest_epoch_data.is_empty());
+                assert!(!latest_epoch_data.has_spilled);
+                if cfg!(debug_assertions) {
+                    panic!("sync epoch exceeds latest epoch, and the current instance should have be archived");
+                }
+                warn!(
+                    instance_id = self.instance_id,
+                    table_id = self.table_id.table_id,
+                    "sync epoch exceeds latest epoch, and the current instance should have be archived"
+                );
+                self.current_epoch_data = None;
+            }
+        }
         ret
     }
+}
 
-    #[cfg(test)]
-    fn imm_count(&self) -> usize {
-        self.imms_by_table_shard
-            .values()
-            .map(|imms| imms.len())
-            .sum()
-    }
-
-    #[cfg(test)]
-    fn imms_by_epoch(&self) -> BTreeMap<HummockEpoch, Vec<ImmutableMemtable>> {
-        let mut imms_by_epoch: BTreeMap<HummockEpoch, Vec<ImmutableMemtable>> = BTreeMap::new();
-        self.imms_by_table_shard.iter().for_each(|(_, imms)| {
-            for imm in imms {
-                debug_assert!(imm.max_epoch() == imm.min_epoch());
-                imms_by_epoch
-                    .entry(imm.max_epoch())
-                    .or_default()
-                    .push(imm.clone());
+struct TableUnsyncData {
+    table_id: TableId,
+    instance_data: HashMap<LocalInstanceId, LocalInstanceUnsyncData>,
+    #[expect(clippy::type_complexity)]
+    table_watermarks: Option<(
+        WatermarkDirection,
+        BTreeMap<HummockEpoch, (Vec<VnodeWatermark>, BitmapBuilder)>,
+    )>,
+}
+
+impl TableUnsyncData {
+    fn new(table_id: TableId) -> Self {
+        Self {
+            table_id,
+            instance_data: Default::default(),
+            table_watermarks: None,
+        }
+    }
+
+    fn sync(
+        &mut self,
+        epoch: HummockEpoch,
+    ) -> (
+        impl Iterator<Item = (LocalInstanceId, Vec<UploaderImm>)> + '_,
+        Option<(
+            WatermarkDirection,
+            impl Iterator<Item = (HummockEpoch, Vec<VnodeWatermark>)>,
+        )>,
+    ) {
+        (
+            self.instance_data
+                .iter_mut()
+                .map(move |(instance_id, data)| (*instance_id, data.sync(epoch))),
+            self.table_watermarks
+                .as_mut()
+                .map(|(direction, watermarks)| {
+                    let watermarks = take_before_epoch(watermarks, epoch);
+                    (
+                        *direction,
+                        watermarks
+                            .into_iter()
+                            .map(|(epoch, (watermarks, _))| (epoch, watermarks)),
+                    )
+                }),
+        )
+    }
+}
+
+#[derive(Default)]
+/// Unsync data, can be either imm or spilled sst, and some aggregated epoch information.
+///
+/// `instance_data` holds the imm of each individual local instance, and data are first added here.
+/// The aggregated epoch information (table watermarks, etc.) and the spilled sst will be added to `epoch_data`.
+struct UnsyncData {
+    table_data: HashMap<TableId, TableUnsyncData>,
+    // An index as a mapping from instance id to its table id
+    instance_table_id: HashMap<LocalInstanceId, TableId>,
+    epoch_data: BTreeMap<HummockEpoch, EpochData>,
+}
+
+impl UnsyncData {
+    fn init_instance(
+        &mut self,
+        table_id: TableId,
+        instance_id: LocalInstanceId,
+        init_epoch: HummockEpoch,
+    ) {
+        debug!(
+            table_id = table_id.table_id,
+            instance_id, init_epoch, "init epoch"
+        );
+        let table_data = self
+            .table_data
+            .entry(table_id)
+            .or_insert_with(|| TableUnsyncData::new(table_id));
+        assert!(table_data
+            .instance_data
+            .insert(
+                instance_id,
+                LocalInstanceUnsyncData::new(table_id, instance_id, init_epoch)
+            )
+            .is_none());
+        assert!(self
+            .instance_table_id
+            .insert(instance_id, table_id)
+            .is_none());
+        self.epoch_data.entry(init_epoch).or_default();
+    }
+
+    fn instance_data(
+        &mut self,
+        instance_id: LocalInstanceId,
+    ) -> Option<&mut LocalInstanceUnsyncData> {
+        self.instance_table_id
+            .get_mut(&instance_id)
+            .cloned()
+            .map(move |table_id| {
+                self.table_data
+                    .get_mut(&table_id)
+                    .expect("should exist")
+                    .instance_data
+                    .get_mut(&instance_id)
+                    .expect("should exist")
+            })
+    }
+
+    fn add_imm(&mut self, instance_id: LocalInstanceId, imm: UploaderImm) {
+        self.instance_data(instance_id)
+            .expect("should exist")
+            .add_imm(imm);
+    }
+
+    fn local_seal_epoch(
+        &mut self,
+        instance_id: LocalInstanceId,
+        next_epoch: HummockEpoch,
+        opts: SealCurrentEpochOptions,
+    ) {
+        let table_id = self.instance_table_id[&instance_id];
+        let table_data = self.table_data.get_mut(&table_id).expect("should exist");
+        let instance_data = table_data
+            .instance_data
+            .get_mut(&instance_id)
+            .expect("should exist");
+        let epoch = instance_data.local_seal_epoch(next_epoch);
+        self.epoch_data.entry(next_epoch).or_default();
+        if let Some((direction, table_watermarks)) = opts.table_watermarks {
+            table_data.add_table_watermarks(epoch, table_watermarks, direction);
+        }
+    }
+
+    fn may_destroy_instance(&mut self, instance_id: LocalInstanceId) {
+        if let Some(table_id) = self.instance_table_id.remove(&instance_id) {
+            debug!(instance_id, "destroy instance");
+            let table_data = self.table_data.get_mut(&table_id).expect("should exist");
+            assert!(table_data.instance_data.remove(&instance_id).is_some());
+            if table_data.instance_data.is_empty() {
+                self.table_data.remove(&table_id);
             }
-        });
-        imms_by_epoch
+        }
+    }
+
+    fn sync(&mut self, epoch: HummockEpoch, context: &UploaderContext) -> SyncDataBuilder {
+        let sync_epoch_data = take_before_epoch(&mut self.epoch_data, epoch);
+
+        let mut sync_data = SyncDataBuilder::default();
+        for (epoch, epoch_data) in sync_epoch_data {
+            sync_data.add_new_epoch(epoch, epoch_data);
+        }
+
+        let mut flush_payload = HashMap::new();
+        for (table_id, table_data) in &mut self.table_data {
+            let (unflushed_payload, table_watermarks) = table_data.sync(epoch);
+            for (instance_id, payload) in unflushed_payload {
+                if !payload.is_empty() {
+                    flush_payload.insert(instance_id, payload);
+                }
+            }
+            if let Some((direction, watermarks)) = table_watermarks {
+                sync_data.add_table_watermarks(*table_id, direction, watermarks);
+            }
+        }
+        sync_data.flush(context, flush_payload);
+        sync_data
+    }
+
+    fn ack_flushed(&mut self, sstable_info: &StagingSstableInfo) {
+        for (instance_id, imm_ids) in sstable_info.imm_ids() {
+            if let Some(instance_data) = self.instance_data(*instance_id) {
+                // take `rev` to let old imm id goes first
+                instance_data.ack_flushed(imm_ids.iter().rev().cloned());
+            }
+        }
     }
 }
 
 struct SyncingData {
-    // newer epochs come first
-    epochs: Vec<HummockEpoch>,
+    sync_epoch: HummockEpoch,
     // TODO: may replace `TryJoinAll` with a future that will abort other join handles once
     // one join handle failed.
     // None means there is no pending uploading tasks
@@ -685,32 +912,20 @@ struct SyncingData {
     // newer data at the front
     uploaded: VecDeque<StagingSstableInfo>,
     table_watermarks: HashMap<TableId, TableWatermarks>,
-}
-
-impl SyncingData {
-    fn sync_epoch(&self) -> HummockEpoch {
-        *self.epochs.first().expect("non-empty")
-    }
+    sync_result_sender: oneshot::Sender<HummockResult<SyncResult>>,
 }
 
 pub struct SyncedData {
-    pub staging_ssts: Vec<StagingSstableInfo>,
+    pub newly_upload_ssts: Vec<StagingSstableInfo>,
+    pub uploaded_ssts: VecDeque<StagingSstableInfo>,
     pub table_watermarks: HashMap<TableId, TableWatermarks>,
 }
 
-// newer staging sstable info at the front
-type SyncedDataState = HummockResult<SyncedData>;
-
 struct UploaderContext {
     pinned_version: PinnedVersion,
     /// When called, it will spawn a task to flush the imm into sst and return the join handle.
     spawn_upload_task: SpawnUploadTask,
-    spawn_merging_task: SpawnMergingTask,
     buffer_tracker: BufferTracker,
-    /// The number of immutable memtables that will be merged into a new imm.
-    /// When the number of imms of a table shard exceeds this threshold, uploader will generate
-    /// merging tasks to merge them.
-    imm_merge_threshold: usize,
 
     stats: Arc<HummockStateStoreMetrics>,
 }
@@ -719,84 +934,91 @@ impl UploaderContext {
     fn new(
         pinned_version: PinnedVersion,
         spawn_upload_task: SpawnUploadTask,
-        spawn_merging_task: SpawnMergingTask,
         buffer_tracker: BufferTracker,
-        config: &StorageOpts,
+        _config: &StorageOpts,
         stats: Arc<HummockStateStoreMetrics>,
     ) -> Self {
         UploaderContext {
             pinned_version,
             spawn_upload_task,
-            spawn_merging_task,
             buffer_tracker,
-            imm_merge_threshold: config.imm_merge_threshold,
             stats,
         }
     }
 }
 
+#[derive(Default)]
+struct UploaderData {
+    unsync_data: UnsyncData,
+
+    /// Data that has started syncing but not synced yet. `epoch` satisfies
+    /// `max_synced_epoch < epoch <= max_syncing_epoch`.
+    /// Newer epoch at the front
+    syncing_data: VecDeque<SyncingData>,
+}
+
+impl UploaderData {
+    fn abort(self, err: impl Fn() -> HummockError) {
+        for (_, epoch_data) in self.unsync_data.epoch_data {
+            epoch_data.spilled_data.abort();
+        }
+        // TODO: call `abort` on the uploading task join handle of syncing_data
+        for syncing_data in self.syncing_data {
+            send_sync_result(syncing_data.sync_result_sender, Err(err()));
+        }
+    }
+}
+
+enum UploaderState {
+    Working(UploaderData),
+    Err {
+        failed_epoch: HummockEpoch,
+        reason: String,
+    },
+}
+
 /// An uploader for hummock data.
 ///
-/// Data have 4 sequential stages: unsealed, sealed, syncing, synced.
+/// Data have 3 sequential stages: unsync (inside each local instance, data can be unsealed, sealed), syncing, synced.
 ///
-/// The 4 stages are divided by 3 marginal epochs: `max_sealed_epoch`, `max_syncing_epoch`,
+/// The 3 stages are divided by 2 marginal epochs: `max_syncing_epoch`,
 /// `max_synced_epoch`. Epochs satisfy the following inequality.
 ///
 /// (epochs of `synced_data`) <= `max_synced_epoch` < (epochs of `syncing_data`) <=
-/// `max_syncing_epoch` < (epochs of `sealed_data`) <= `max_sealed_epoch` < (epochs of
-/// `unsealed_data`)
+/// `max_syncing_epoch` < (epochs of `unsync_data`)
 ///
 /// Data are mostly stored in `VecDeque`, and the order stored in the `VecDeque` indicates the data
 /// order. Data at the front represents ***newer*** data.
 pub struct HummockUploader {
-    /// The maximum epoch that is sealed
-    max_sealed_epoch: HummockEpoch,
     /// The maximum epoch that has started syncing
     max_syncing_epoch: HummockEpoch,
     /// The maximum epoch that has been synced
     max_synced_epoch: HummockEpoch,
 
-    /// Data that are not sealed yet. `epoch` satisfies `epoch > max_sealed_epoch`.
-    unsealed_data: BTreeMap<HummockEpoch, UnsealedEpochData>,
-
-    /// Data that are sealed but not synced yet. `epoch` satisfies
-    /// `max_syncing_epoch < epoch <= max_sealed_epoch`.
-    sealed_data: SealedData,
-
-    /// Data that has started syncing but not synced yet. `epoch` satisfies
-    /// `max_synced_epoch < epoch <= max_syncing_epoch`.
-    /// Newer epoch at the front
-    syncing_data: VecDeque<SyncingData>,
-
-    /// Data that has been synced already. `epoch` satisfies
-    /// `epoch <= max_synced_epoch`.
-    synced_data: BTreeMap<HummockEpoch, SyncedDataState>,
+    state: UploaderState,
 
     context: UploaderContext,
 }
 
 impl HummockUploader {
-    pub(crate) fn new(
+    pub(super) fn new(
         state_store_metrics: Arc<HummockStateStoreMetrics>,
         pinned_version: PinnedVersion,
         spawn_upload_task: SpawnUploadTask,
-        spawn_merging_task: SpawnMergingTask,
         buffer_tracker: BufferTracker,
         config: &StorageOpts,
     ) -> Self {
         let initial_epoch = pinned_version.version().max_committed_epoch;
         Self {
-            max_sealed_epoch: initial_epoch,
             max_syncing_epoch: initial_epoch,
             max_synced_epoch: initial_epoch,
-            unsealed_data: Default::default(),
-            sealed_data: Default::default(),
-            syncing_data: Default::default(),
-            synced_data: Default::default(),
+            state: UploaderState::Working(UploaderData {
+                unsync_data: Default::default(),
+                syncing_data: Default::default(),
+            }),
             context: UploaderContext::new(
                 pinned_version,
                 spawn_upload_task,
-                spawn_merging_task,
                 buffer_tracker,
                 config,
                 state_store_metrics,
@@ -804,159 +1026,77 @@ impl HummockUploader {
         }
     }
 
-    #[cfg(test)]
-    pub(crate) fn imm_merge_threshold(&self) -> usize {
-        self.context.imm_merge_threshold
-    }
-
-    pub(crate) fn buffer_tracker(&self) -> &BufferTracker {
+    pub(super) fn buffer_tracker(&self) -> &BufferTracker {
         &self.context.buffer_tracker
     }
 
-    pub(crate) fn max_sealed_epoch(&self) -> HummockEpoch {
-        self.max_sealed_epoch
-    }
-
-    pub(crate) fn max_synced_epoch(&self) -> HummockEpoch {
+    pub(super) fn max_synced_epoch(&self) -> HummockEpoch {
         self.max_synced_epoch
     }
 
-    pub(crate) fn max_committed_epoch(&self) -> HummockEpoch {
+    pub(super) fn max_committed_epoch(&self) -> HummockEpoch {
         self.context.pinned_version.max_committed_epoch()
     }
 
-    pub(crate) fn hummock_version(&self) -> &PinnedVersion {
+    pub(super) fn hummock_version(&self) -> &PinnedVersion {
         &self.context.pinned_version
     }
 
-    pub(crate) fn get_synced_data(&self, epoch: HummockEpoch) -> Option<&SyncedDataState> {
-        assert!(self.max_committed_epoch() < epoch && epoch <= self.max_synced_epoch);
-        self.synced_data.get(&epoch)
-    }
-
-    pub(crate) fn add_imm(&mut self, imm: ImmutableMemtable) {
-        let epoch = imm.min_epoch();
-        assert!(
-            epoch > self.max_sealed_epoch,
-            "imm epoch {} older than max sealed epoch {}",
-            epoch,
-            self.max_sealed_epoch
-        );
-        self.unsealed_data
-            .entry(epoch)
-            .or_default()
-            .imms
-            .push_front(imm);
+    pub(super) fn add_imm(&mut self, instance_id: LocalInstanceId, imm: ImmutableMemtable) {
+        let UploaderState::Working(data) = &mut self.state else {
+            return;
+        };
+        let imm = UploaderImm::new(imm, &self.context);
+        data.unsync_data.add_imm(instance_id, imm);
     }
 
-    pub(crate) fn add_table_watermarks(
+    pub(super) fn init_instance(
         &mut self,
-        epoch: u64,
+        instance_id: LocalInstanceId,
         table_id: TableId,
-        table_watermarks: Vec<VnodeWatermark>,
-        direction: WatermarkDirection,
+        init_epoch: HummockEpoch,
     ) {
-        assert!(
-            epoch > self.max_sealed_epoch,
-            "imm epoch {} older than max sealed epoch {}",
-            epoch,
-            self.max_sealed_epoch
-        );
-        self.unsealed_data
-            .entry(epoch)
-            .or_default()
-            .add_table_watermarks(table_id, table_watermarks, direction);
-    }
-
-    pub(crate) fn seal_epoch(&mut self, epoch: HummockEpoch) {
-        debug!("epoch {} is sealed", epoch);
-        assert!(
-            epoch > self.max_sealed_epoch,
-            "sealing a sealed epoch {}. {}",
-            epoch,
-            self.max_sealed_epoch
-        );
-        self.max_sealed_epoch = epoch;
-        let unsealed_data =
-            if let Some((&smallest_unsealed_epoch, _)) = self.unsealed_data.first_key_value() {
-                assert!(
-                    smallest_unsealed_epoch >= epoch,
-                    "some epoch {} older than epoch to seal {}",
-                    smallest_unsealed_epoch,
-                    epoch
-                );
-                if smallest_unsealed_epoch == epoch {
-                    let (_, unsealed_data) = self
-                        .unsealed_data
-                        .pop_first()
-                        .expect("we have checked non-empty");
-                    unsealed_data
-                } else {
-                    debug!("epoch {} to seal has no data", epoch);
-                    UnsealedEpochData::default()
-                }
-            } else {
-                debug!("epoch {} to seal has no data", epoch);
-                UnsealedEpochData::default()
-            };
-        self.sealed_data.seal_new_epoch(epoch, unsealed_data);
-    }
-
-    pub(crate) fn start_merge_imms(&mut self, sealed_epoch: HummockEpoch) {
-        // skip merging if merge threshold is 1
-        if self.context.imm_merge_threshold <= 1 {
+        let UploaderState::Working(data) = &mut self.state else {
             return;
-        }
-
-        let memory_limiter = self.context.buffer_tracker.get_memory_limiter();
-        // scan imms of each table shard to generate merging task
-        // when the number of imms exceeds the merge threshold
-        for ((table_id, shard_id), imms) in self
-            .sealed_data
-            .imms_by_table_shard
-            .iter_mut()
-            .filter(|(_, imms)| imms.len() >= self.context.imm_merge_threshold)
-        {
-            let imms_to_merge = imms.drain(..).collect_vec();
-            let mut value_count = 0;
-            let mut imm_size = 0;
-            imms_to_merge.iter().for_each(|imm| {
-                // ensure imms are sealed
-                assert_le!(imm.max_epoch(), sealed_epoch);
-                value_count += imm.value_count();
-                imm_size += imm.size();
-            });
-
-            // acquire memory before generate merge task
-            // if acquire memory failed, the task will not be generated
-            let memory_sz = (imm_size + value_count * EPOCH_LEN) as u64;
-            if let Some(tracker) = memory_limiter.try_require_memory(memory_sz) {
-                self.sealed_data
-                    .merging_tasks
-                    .push_front(MergingImmTask::new(
-                        *table_id,
-                        *shard_id,
-                        imms_to_merge,
-                        Some(tracker),
-                        &self.context,
-                    ));
-            } else {
-                tracing::warn!(
-                    "fail to acqiure memory {} B, skip merging imms for table {}, shard {}",
-                    memory_sz,
-                    table_id,
-                    shard_id
-                );
-                imms.extend(imms_to_merge);
-            }
-        }
+        };
+        assert_gt!(init_epoch, self.max_syncing_epoch);
+        data.unsync_data
+            .init_instance(table_id, instance_id, init_epoch);
     }
 
-    pub(crate) fn update_sealed_data(&mut self, merged_imm: &ImmutableMemtable) {
-        self.sealed_data.add_merged_imm(merged_imm);
+    pub(super) fn local_seal_epoch(
+        &mut self,
+        instance_id: LocalInstanceId,
+        next_epoch: HummockEpoch,
+        opts: SealCurrentEpochOptions,
+    ) {
+        let UploaderState::Working(data) = &mut self.state else {
+            return;
+        };
+        assert_gt!(next_epoch, self.max_syncing_epoch);
+        data.unsync_data
+            .local_seal_epoch(instance_id, next_epoch, opts);
     }
 
-    pub(crate) fn start_sync_epoch(&mut self, epoch: HummockEpoch) {
+    pub(super) fn start_sync_epoch(
+        &mut self,
+        epoch: HummockEpoch,
+        sync_result_sender: oneshot::Sender<HummockResult<SyncResult>>,
+    ) {
+        let data = match &mut self.state {
+            UploaderState::Working(data) => data,
+            UploaderState::Err {
+                failed_epoch,
+                reason,
+            } => {
+                let result = Err(HummockError::other(format!(
+                    "previous epoch {} failed due to [{}]",
+                    failed_epoch, reason
+                )));
+                send_sync_result(sync_result_sender, result);
+                return;
+            }
+        };
         debug!("start sync epoch: {}", epoch);
         assert!(
             epoch > self.max_syncing_epoch,
@@ -964,20 +1104,12 @@ impl HummockUploader {
             epoch,
             self.max_syncing_epoch
         );
-        assert_eq!(
-            epoch, self.max_sealed_epoch,
-            "we must start syncing all the sealed data",
-        );
 
         self.max_syncing_epoch = epoch;
 
-        // flush imms to SST file, the output SSTs will be uploaded to object store
-        // return unfinished merging task
-        self.sealed_data.flush(&self.context, false);
+        let sync_data = data.unsync_data.sync(epoch, &self.context);
 
-        let SealedData {
-            epochs,
-            imms_by_table_shard,
+        let SyncDataBuilder {
             spilled_data:
                 SpilledData {
                     uploading_tasks,
@@ -985,14 +1117,7 @@ impl HummockUploader {
                 },
             table_watermarks,
             ..
-        } = self.sealed_data.drain();
-
-        assert!(
-            imms_by_table_shard.is_empty(),
-            "after flush, imms must be empty"
-        );
-
-        assert_eq!(epoch, *epochs.front().expect("non-empty epoch"));
+        } = sync_data;
 
         let try_join_all_upload_task = if uploading_tasks.is_empty() {
             None
@@ -1000,20 +1125,21 @@ impl HummockUploader {
             Some(try_join_all(uploading_tasks))
         };
 
-        self.syncing_data.push_front(SyncingData {
-            epochs: epochs.into_iter().collect(),
+        data.syncing_data.push_front(SyncingData {
+            sync_epoch: epoch,
             uploading_tasks: try_join_all_upload_task,
             uploaded: uploaded_data,
             table_watermarks,
+            sync_result_sender,
         });
 
         self.context
             .stats
             .uploader_syncing_epoch_count
-            .set(self.syncing_data.len() as _);
+            .set(data.syncing_data.len() as _);
     }
 
-    fn add_synced_data(&mut self, epoch: HummockEpoch, synced_state: SyncedDataState) {
+    fn add_synced_data(&mut self, epoch: HummockEpoch) {
         assert!(
             epoch <= self.max_syncing_epoch,
             "epoch {} that has been synced has not started syncing yet.  previous max syncing epoch {}",
@@ -1027,7 +1153,6 @@ impl HummockUploader {
             self.max_synced_epoch
         );
         self.max_synced_epoch = epoch;
-        assert!(self.synced_data.insert(epoch, synced_state).is_none());
     }
 
     pub(crate) fn update_pinned_version(&mut self, pinned_version: PinnedVersion) {
@@ -1037,50 +1162,60 @@ impl HummockUploader {
         );
         let max_committed_epoch = pinned_version.max_committed_epoch();
         self.context.pinned_version = pinned_version;
-        self.synced_data
-            .retain(|epoch, _| *epoch > max_committed_epoch);
         if self.max_synced_epoch < max_committed_epoch {
             self.max_synced_epoch = max_committed_epoch;
-            if let Some(syncing_data) = self.syncing_data.back() {
-                // there must not be any syncing data below MCE
-                assert_gt!(
-                    *syncing_data
-                        .epochs
-                        .last()
-                        .expect("epoch should not be empty"),
-                    max_committed_epoch
-                );
-            }
         }
         if self.max_syncing_epoch < max_committed_epoch {
             self.max_syncing_epoch = max_committed_epoch;
-            // there must not be any sealed data below MCE
-            if let Some(&epoch) = self.sealed_data.epochs.back() {
-                assert_gt!(epoch, max_committed_epoch);
-            }
-        }
-        if self.max_sealed_epoch < max_committed_epoch {
-            self.max_sealed_epoch = max_committed_epoch;
-            // there must not be any unsealed data below MCE
-            if let Some((&epoch, _)) = self.unsealed_data.first_key_value() {
-                assert_gt!(epoch, max_committed_epoch);
+            if let UploaderState::Working(data) = &self.state {
+                for instance_data in data
+                    .unsync_data
+                    .table_data
+                    .values()
+                    .flat_map(|data| data.instance_data.values())
+                {
+                    if let Some(oldest_epoch) = instance_data.sealed_data.back() {
+                        assert_gt!(oldest_epoch.epoch, max_committed_epoch);
+                    } else if let Some(current_epoch) = &instance_data.current_epoch_data {
+                        assert_gt!(current_epoch.epoch, max_committed_epoch);
+                    }
+                }
             }
         }
     }
 
-    pub(crate) fn may_flush(&mut self) {
-        if self.context.buffer_tracker.need_more_flush() {
-            self.sealed_data.flush(&self.context, true);
-        }
-
-        if self.context.buffer_tracker.need_more_flush() {
+    pub(crate) fn may_flush(&mut self) -> bool {
+        let UploaderState::Working(data) = &mut self.state else {
+            return false;
+        };
+        if self.context.buffer_tracker.need_flush() {
+            let mut curr_batch_flush_size = 0;
             // iterate from older epoch to newer epoch
-            for unsealed_data in self.unsealed_data.values_mut() {
-                unsealed_data.flush(&self.context);
-                if !self.context.buffer_tracker.need_more_flush() {
+            for (epoch, epoch_data) in &mut data.unsync_data.epoch_data {
+                if !self
+                    .context
+                    .buffer_tracker
+                    .need_more_flush(curr_batch_flush_size)
+                {
                     break;
                 }
+                let mut payload = HashMap::new();
+                for (instance_id, instance_data) in data
+                    .unsync_data
+                    .table_data
+                    .values_mut()
+                    .flat_map(|data| data.instance_data.iter_mut())
+                {
+                    let instance_payload = instance_data.spill(*epoch);
+                    if !instance_payload.is_empty() {
+                        payload.insert(*instance_id, instance_payload);
+                    }
+                }
+                curr_batch_flush_size += epoch_data.flush(&self.context, payload);
             }
+            curr_batch_flush_size > 0
+        } else {
+            false
         }
     }
 
@@ -1088,25 +1223,41 @@ impl HummockUploader {
         let max_committed_epoch = self.context.pinned_version.max_committed_epoch();
         self.max_synced_epoch = max_committed_epoch;
         self.max_syncing_epoch = max_committed_epoch;
-        self.max_sealed_epoch = max_committed_epoch;
-        self.synced_data.clear();
-        self.syncing_data.clear();
-        self.sealed_data.clear();
-        self.unsealed_data.clear();
+        if let UploaderState::Working(data) = replace(
+            &mut self.state,
+            UploaderState::Working(UploaderData::default()),
+        ) {
+            data.abort(|| {
+                HummockError::other(format!("uploader is reset to {}", max_committed_epoch))
+            });
+        }
 
         self.context.stats.uploader_syncing_epoch_count.set(0);
+    }
 
-        // TODO: call `abort` on the uploading task join handle
+    pub(crate) fn may_destroy_instance(&mut self, instance_id: LocalInstanceId) {
+        let UploaderState::Working(data) = &mut self.state else {
+            return;
+        };
+        data.unsync_data.may_destroy_instance(instance_id);
     }
 }
 
-impl HummockUploader {
+impl UploaderData {
     /// Poll the syncing task of the syncing data of the oldest epoch. Return `Poll::Ready(None)` if
     /// there is no syncing data.
+    #[expect(clippy::type_complexity)]
     fn poll_syncing_task(
         &mut self,
         cx: &mut Context<'_>,
-    ) -> Poll<Option<(HummockEpoch, Vec<StagingSstableInfo>)>> {
+        context: &UploaderContext,
+    ) -> Poll<
+        Option<(
+            HummockEpoch,
+            HummockResult<SyncedData>,
+            oneshot::Sender<HummockResult<SyncResult>>,
+        )>,
+    > {
         // Only poll the oldest epoch if there is any so that the syncing epoch are finished in
         // order
         if let Some(syncing_data) = self.syncing_data.back_mut() {
@@ -1117,118 +1268,102 @@ impl HummockUploader {
                 Ok(Vec::new())
             };
             let syncing_data = self.syncing_data.pop_back().expect("must exist");
-            self.context
+            context
                 .stats
                 .uploader_syncing_epoch_count
                 .set(self.syncing_data.len() as _);
-            let epoch = syncing_data.sync_epoch();
-
-            let newly_uploaded_sstable_infos = match &result {
-                Ok(sstable_infos) => sstable_infos.clone(),
-                Err(_) => vec![],
-            };
+            let epoch = syncing_data.sync_epoch;
 
-            let result = result.map(|mut sstable_infos| {
-                // The newly uploaded `sstable_infos` contains newer data. Therefore,
-                // `sstable_infos` at the front
-                sstable_infos.extend(syncing_data.uploaded);
+            let result = result.map(|newly_uploaded_sstable_infos| {
+                // take `rev` so that old data is acked first
+                for sstable_info in newly_uploaded_sstable_infos.iter().rev() {
+                    self.unsync_data.ack_flushed(sstable_info);
+                }
                 SyncedData {
-                    staging_ssts: sstable_infos,
+                    newly_upload_ssts: newly_uploaded_sstable_infos,
+                    uploaded_ssts: syncing_data.uploaded,
                     table_watermarks: syncing_data.table_watermarks,
                 }
             });
-            self.add_synced_data(epoch, result);
-            Poll::Ready(Some((epoch, newly_uploaded_sstable_infos)))
+
+            Poll::Ready(Some((epoch, result, syncing_data.sync_result_sender)))
         } else {
             Poll::Ready(None)
         }
     }
 
-    /// Poll the success of the oldest spilled task of sealed data. Return `Poll::Ready(None)` if
-    /// there is no spilling task.
-    fn poll_sealed_spill_task(&mut self, cx: &mut Context<'_>) -> Poll<Option<StagingSstableInfo>> {
-        self.sealed_data.spilled_data.poll_success_spill(cx)
-    }
-
-    /// Poll the success of the oldest spilled task of unsealed data. Return `Poll::Ready(None)` if
+    /// Poll the success of the oldest spilled task of unsync spill data. Return `Poll::Ready(None)` if
     /// there is no spilling task.
-    fn poll_unsealed_spill_task(
-        &mut self,
-        cx: &mut Context<'_>,
-    ) -> Poll<Option<StagingSstableInfo>> {
+    fn poll_spill_task(&mut self, cx: &mut Context<'_>) -> Poll<Option<StagingSstableInfo>> {
         // iterator from older epoch to new epoch so that the spill task are finished in epoch order
-        for unsealed_data in self.unsealed_data.values_mut() {
-            // if None, there is no spilling task. Search for the unsealed data of the next epoch in
+        for epoch_data in self.unsync_data.epoch_data.values_mut() {
+            // if None, there is no spilling task. Search for the unsync data of the next epoch in
             // the next iteration.
-            if let Some(sstable_info) = ready!(unsealed_data.spilled_data.poll_success_spill(cx)) {
+            if let Some(sstable_info) = ready!(epoch_data.spilled_data.poll_success_spill(cx)) {
+                self.unsync_data.ack_flushed(&sstable_info);
                 return Poll::Ready(Some(sstable_info));
             }
         }
         Poll::Ready(None)
     }
-
-    fn poll_sealed_merge_imm_task(
-        &mut self,
-        cx: &mut Context<'_>,
-    ) -> Poll<Option<MergeImmTaskOutput>> {
-        let poll_ret = self.sealed_data.poll_success_merge_imm(cx);
-        if let Poll::Ready(Some(output)) = &poll_ret {
-            let table_id_label = output.table_id.to_string();
-
-            // monitor finished task
-            self.context
-                .stats
-                .merge_imm_task_counts
-                .with_label_values(&[table_id_label.as_str()])
-                .inc();
-            // monitor merge imm memory size
-            // we should also add up the size of EPOCH stored in each entry
-            self.context
-                .stats
-                .merge_imm_batch_memory_sz
-                .with_label_values(&[table_id_label.as_str()])
-                .inc_by(
-                    (output.merged_imm.size() + output.merged_imm.value_count() * EPOCH_LEN) as _,
-                );
-        }
-        poll_ret
-    }
 }
 
-pub(crate) enum UploaderEvent {
+pub(super) enum UploaderEvent {
     // staging sstable info of newer data comes first
-    SyncFinish(HummockEpoch, Vec<StagingSstableInfo>),
+    SyncFinish(HummockEpoch, SyncedData),
     DataSpilled(StagingSstableInfo),
-    ImmMerged(MergeImmTaskOutput),
 }
 
 impl HummockUploader {
-    pub(crate) fn next_event(&mut self) -> impl Future<Output = UploaderEvent> + '_ {
+    pub(super) fn next_event(&mut self) -> impl Future<Output = UploaderEvent> + '_ {
         poll_fn(|cx| {
-            if let Some((epoch, newly_uploaded_sstables)) = ready!(self.poll_syncing_task(cx)) {
-                return Poll::Ready(UploaderEvent::SyncFinish(epoch, newly_uploaded_sstables));
-            }
-
-            if let Some(sstable_info) = ready!(self.poll_sealed_spill_task(cx)) {
-                return Poll::Ready(UploaderEvent::DataSpilled(sstable_info));
+            let UploaderState::Working(data) = &mut self.state else {
+                return Poll::Pending;
+            };
+            if let Some((epoch, result, result_sender)) =
+                ready!(data.poll_syncing_task(cx, &self.context))
+            {
+                match result {
+                    Ok(data) => {
+                        self.add_synced_data(epoch);
+                        send_sync_result(result_sender, Ok(&data));
+                        return Poll::Ready(UploaderEvent::SyncFinish(epoch, data));
+                    }
+                    Err(e) => {
+                        send_sync_result(
+                            result_sender,
+                            Err(HummockError::other(format!(
+                                "failed sync task: {:?}",
+                                e.as_report()
+                            ))),
+                        );
+                        let data = must_match!(replace(
+                            &mut self.state,
+                            UploaderState::Err {
+                                failed_epoch: epoch,
+                                reason: format!("{:?}", e.as_report()),
+                            },
+                        ), UploaderState::Working(data) => data);
+
+                        data.abort(|| {
+                            HummockError::other(format!("previous epoch {} failed to sync", epoch))
+                        });
+                        return Poll::Pending;
+                    }
+                };
             }
 
-            if let Some(sstable_info) = ready!(self.poll_unsealed_spill_task(cx)) {
+            if let Some(sstable_info) = ready!(data.poll_spill_task(cx)) {
                 return Poll::Ready(UploaderEvent::DataSpilled(sstable_info));
             }
 
-            if let Some(merge_output) = ready!(self.poll_sealed_merge_imm_task(cx)) {
-                // add the merged imm into sealed data
-                self.update_sealed_data(&merge_output.merged_imm);
-                return Poll::Ready(UploaderEvent::ImmMerged(merge_output));
-            }
             Poll::Pending
         })
     }
 }
 
 #[cfg(test)]
-mod tests {
+pub(crate) mod tests {
     use std::collections::{HashMap, VecDeque};
     use std::future::{poll_fn, Future};
     use std::ops::Deref;
@@ -1236,13 +1371,14 @@ mod tests {
     use std::sync::atomic::Ordering::SeqCst;
     use std::sync::Arc;
     use std::task::Poll;
-    use std::time::Duration;
 
     use bytes::Bytes;
     use futures::future::BoxFuture;
     use futures::FutureExt;
+    use itertools::Itertools;
     use prometheus::core::GenericGauge;
     use risingwave_common::catalog::TableId;
+    use risingwave_common::must_match;
     use risingwave_common::util::epoch::{test_epoch, EpochExt};
     use risingwave_hummock_sdk::key::{FullKey, TableKey};
     use risingwave_hummock_sdk::version::HummockVersion;
@@ -1254,44 +1390,49 @@ mod tests {
     use tokio::sync::oneshot;
     use tokio::task::yield_now;
 
-    use crate::hummock::compactor::CompactionExecutor;
     use crate::hummock::event_handler::hummock_event_handler::BufferTracker;
+    use crate::hummock::event_handler::uploader::uploader_imm::UploaderImm;
     use crate::hummock::event_handler::uploader::{
-        default_spawn_merging_task, HummockUploader, MergingImmTask, UploadTaskInfo,
-        UploadTaskOutput, UploadTaskPayload, UploaderContext, UploaderEvent, UploadingTask,
-    };
-    use crate::hummock::event_handler::LocalInstanceId;
-    use crate::hummock::iterator::test_utils::{
-        iterator_test_table_key_of, transform_shared_buffer,
+        get_payload_imm_ids, HummockUploader, SyncedData, UploadTaskInfo, UploadTaskOutput,
+        UploadTaskPayload, UploaderContext, UploaderData, UploaderEvent, UploaderState,
+        UploadingTask,
     };
+    use crate::hummock::event_handler::{LocalInstanceId, TEST_LOCAL_INSTANCE_ID};
     use crate::hummock::local_version::pinned_version::PinnedVersion;
-    use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferBatch;
-    use crate::hummock::value::HummockValue;
+    use crate::hummock::shared_buffer::shared_buffer_batch::{
+        SharedBufferBatch, SharedBufferBatchId, SharedBufferValue,
+    };
     use crate::hummock::{HummockError, HummockResult, MemoryLimiter};
     use crate::mem_table::{ImmId, ImmutableMemtable};
     use crate::monitor::HummockStateStoreMetrics;
     use crate::opts::StorageOpts;
-    use crate::storage_value::StorageValue;
+    use crate::store::SealCurrentEpochOptions;
 
     const INITIAL_EPOCH: HummockEpoch = test_epoch(5);
-    const TEST_TABLE_ID: TableId = TableId { table_id: 233 };
+    pub(crate) const TEST_TABLE_ID: TableId = TableId { table_id: 233 };
 
     pub trait UploadOutputFuture =
         Future<Output = HummockResult<UploadTaskOutput>> + Send + 'static;
     pub trait UploadFn<Fut: UploadOutputFuture> =
         Fn(UploadTaskPayload, UploadTaskInfo) -> Fut + Send + Sync + 'static;
 
-    fn test_hummock_version(epoch: HummockEpoch) -> HummockVersion {
-        HummockVersion {
-            id: epoch,
-            levels: Default::default(),
-            max_committed_epoch: epoch,
-            safe_epoch: 0,
-            table_watermarks: HashMap::new(),
-            table_change_log: HashMap::new(),
+    impl HummockUploader {
+        fn data(&self) -> &UploaderData {
+            must_match!(&self.state, UploaderState::Working(data) => data)
+        }
+
+        fn start_sync_epoch_for_test(&mut self, epoch: HummockEpoch) {
+            self.start_sync_epoch(epoch, oneshot::channel().0)
         }
     }
 
+    fn test_hummock_version(epoch: HummockEpoch) -> HummockVersion {
+        let mut version = HummockVersion::default();
+        version.id = epoch;
+        version.max_committed_epoch = epoch;
+        version
+    }
+
     fn initial_pinned_version() -> PinnedVersion {
         PinnedVersion::new(test_hummock_version(INITIAL_EPOCH), unbounded_channel().0)
     }
@@ -1304,11 +1445,11 @@ mod tests {
         epoch: HummockEpoch,
         limiter: Option<&MemoryLimiter>,
     ) -> ImmutableMemtable {
-        let sorted_items = SharedBufferBatch::build_shared_buffer_item_batches(vec![(
+        let sorted_items = vec![(
             TableKey(Bytes::from(dummy_table_key())),
-            StorageValue::new_delete(),
-        )]);
-        let size = SharedBufferBatch::measure_batch_size(&sorted_items);
+            SharedBufferValue::Delete,
+        )];
+        let size = SharedBufferBatch::measure_batch_size(&sorted_items, None).0;
         let tracker = match limiter {
             Some(limiter) => Some(limiter.require_memory(size as u64).await),
             None => None,
@@ -1317,14 +1458,14 @@ mod tests {
             epoch,
             0,
             sorted_items,
+            None,
             size,
             TEST_TABLE_ID,
-            LocalInstanceId::default(),
             tracker,
         )
     }
 
-    async fn gen_imm(epoch: HummockEpoch) -> ImmutableMemtable {
+    pub(crate) async fn gen_imm(epoch: HummockEpoch) -> ImmutableMemtable {
         gen_imm_with_limiter(epoch, None).await
     }
 
@@ -1354,11 +1495,9 @@ mod tests {
         F: UploadFn<Fut>,
     {
         let config = StorageOpts::default();
-        let compaction_executor = Arc::new(CompactionExecutor::new(None));
         UploaderContext::new(
             initial_pinned_version(),
             Arc::new(move |payload, task_info| spawn(upload_fn(payload, task_info))),
-            default_spawn_merging_task(compaction_executor, None),
             BufferTracker::for_test(),
             &config,
             Arc::new(HummockStateStoreMetrics::unused()),
@@ -1371,22 +1510,23 @@ mod tests {
         F: UploadFn<Fut>,
     {
         let config = StorageOpts {
-            imm_merge_threshold: 4,
             ..Default::default()
         };
-        let compaction_executor = Arc::new(CompactionExecutor::new(None));
         HummockUploader::new(
             Arc::new(HummockStateStoreMetrics::unused()),
             initial_pinned_version(),
             Arc::new(move |payload, task_info| spawn(upload_fn(payload, task_info))),
-            default_spawn_merging_task(compaction_executor, None),
             BufferTracker::for_test(),
             &config,
         )
     }
 
-    fn dummy_success_upload_output() -> Vec<LocalSstableInfo> {
-        gen_sstable_info(INITIAL_EPOCH, INITIAL_EPOCH)
+    fn dummy_success_upload_output() -> UploadTaskOutput {
+        UploadTaskOutput {
+            new_value_ssts: gen_sstable_info(INITIAL_EPOCH, INITIAL_EPOCH),
+            old_value_ssts: vec![],
+            wait_poll_timer: None,
+        }
     }
 
     #[allow(clippy::unused_async)]
@@ -1394,10 +1534,7 @@ mod tests {
         _: UploadTaskPayload,
         _: UploadTaskInfo,
     ) -> HummockResult<UploadTaskOutput> {
-        Ok(UploadTaskOutput {
-            ssts: dummy_success_upload_output(),
-            wait_poll_timer: None,
-        })
+        Ok(dummy_success_upload_output())
     }
 
     #[allow(clippy::unused_async)]
@@ -1408,37 +1545,75 @@ mod tests {
         Err(HummockError::other("failed"))
     }
 
+    impl UploadingTask {
+        fn from_vec(imms: Vec<ImmutableMemtable>, context: &UploaderContext) -> Self {
+            let input = HashMap::from_iter([(
+                TEST_LOCAL_INSTANCE_ID,
+                imms.into_iter().map(UploaderImm::for_test).collect_vec(),
+            )]);
+            Self::new(input, context)
+        }
+    }
+
+    fn get_imm_ids<'a>(
+        imms: impl IntoIterator<Item = &'a ImmutableMemtable>,
+    ) -> HashMap<LocalInstanceId, Vec<SharedBufferBatchId>> {
+        HashMap::from_iter([(
+            TEST_LOCAL_INSTANCE_ID,
+            imms.into_iter().map(|imm| imm.batch_id()).collect_vec(),
+        )])
+    }
+
+    impl HummockUploader {
+        fn local_seal_epoch_for_test(&mut self, instance_id: LocalInstanceId, epoch: HummockEpoch) {
+            self.local_seal_epoch(
+                instance_id,
+                epoch.next_epoch(),
+                SealCurrentEpochOptions::for_test(),
+            );
+        }
+    }
+
     #[tokio::test]
     pub async fn test_uploading_task_future() {
         let uploader_context = test_uploader_context(dummy_success_upload_future);
 
         let imm = gen_imm(INITIAL_EPOCH).await;
         let imm_size = imm.size();
-        let imm_id = imm.batch_id();
-        let task = UploadingTask::new(vec![imm], &uploader_context);
+        let imm_ids = get_imm_ids(vec![&imm]);
+        let task = UploadingTask::from_vec(vec![imm], &uploader_context);
         assert_eq!(imm_size, task.task_info.task_size);
-        assert_eq!(vec![imm_id], task.task_info.imm_ids);
+        assert_eq!(imm_ids, task.task_info.imm_ids);
         assert_eq!(vec![INITIAL_EPOCH], task.task_info.epochs);
         let output = task.await.unwrap();
-        assert_eq!(output.sstable_infos(), &dummy_success_upload_output());
+        assert_eq!(
+            output.sstable_infos(),
+            &dummy_success_upload_output().new_value_ssts
+        );
         assert_eq!(imm_size, output.imm_size());
-        assert_eq!(&vec![imm_id], output.imm_ids());
+        assert_eq!(&imm_ids, output.imm_ids());
         assert_eq!(&vec![INITIAL_EPOCH], output.epochs());
 
         let uploader_context = test_uploader_context(dummy_fail_upload_future);
-        let task = UploadingTask::new(vec![gen_imm(INITIAL_EPOCH).await], &uploader_context);
+        let imm = gen_imm(INITIAL_EPOCH).await;
+        let task = UploadingTask::from_vec(vec![imm], &uploader_context);
         let _ = task.await.unwrap_err();
     }
 
     #[tokio::test]
     pub async fn test_uploading_task_poll_result() {
         let uploader_context = test_uploader_context(dummy_success_upload_future);
-        let mut task = UploadingTask::new(vec![gen_imm(INITIAL_EPOCH).await], &uploader_context);
+        let mut task =
+            UploadingTask::from_vec(vec![gen_imm(INITIAL_EPOCH).await], &uploader_context);
         let output = poll_fn(|cx| task.poll_result(cx)).await.unwrap();
-        assert_eq!(output.sstable_infos(), &dummy_success_upload_output());
+        assert_eq!(
+            output.sstable_infos(),
+            &dummy_success_upload_output().new_value_ssts
+        );
 
         let uploader_context = test_uploader_context(dummy_fail_upload_future);
-        let mut task = UploadingTask::new(vec![gen_imm(INITIAL_EPOCH).await], &uploader_context);
+        let mut task =
+            UploadingTask::from_vec(vec![gen_imm(INITIAL_EPOCH).await], &uploader_context);
         let _ = poll_fn(|cx| task.poll_result(cx)).await.unwrap_err();
     }
 
@@ -1460,10 +1635,14 @@ mod tests {
                 ret
             }
         });
-        let mut task = UploadingTask::new(vec![gen_imm(INITIAL_EPOCH).await], &uploader_context);
+        let mut task =
+            UploadingTask::from_vec(vec![gen_imm(INITIAL_EPOCH).await], &uploader_context);
         let output = poll_fn(|cx| task.poll_ok_with_retry(cx)).await;
         assert_eq!(fail_num + 1, run_count_clone.load(SeqCst));
-        assert_eq!(output.sstable_infos(), &dummy_success_upload_output());
+        assert_eq!(
+            output.sstable_infos(),
+            &dummy_success_upload_output().new_value_ssts
+        );
     }
 
     #[tokio::test]
@@ -1471,165 +1650,76 @@ mod tests {
         let mut uploader = test_uploader(dummy_success_upload_future);
         let epoch1 = INITIAL_EPOCH.next_epoch();
         let imm = gen_imm(epoch1).await;
-        uploader.add_imm(imm.clone());
-        assert_eq!(1, uploader.unsealed_data.len());
-        assert_eq!(
-            epoch1 as HummockEpoch,
-            *uploader.unsealed_data.first_key_value().unwrap().0
-        );
-        assert_eq!(
-            1,
-            uploader
-                .unsealed_data
-                .first_key_value()
-                .unwrap()
-                .1
-                .imms
-                .len()
-        );
-        uploader.seal_epoch(epoch1);
-        assert_eq!(epoch1, uploader.max_sealed_epoch);
-        assert!(uploader.unsealed_data.is_empty());
-        assert_eq!(1, uploader.sealed_data.imm_count());
+        uploader.init_instance(TEST_LOCAL_INSTANCE_ID, TEST_TABLE_ID, epoch1);
+        uploader.add_imm(TEST_LOCAL_INSTANCE_ID, imm.clone());
+        uploader.local_seal_epoch_for_test(TEST_LOCAL_INSTANCE_ID, epoch1);
 
-        uploader.start_sync_epoch(epoch1);
+        uploader.start_sync_epoch_for_test(epoch1);
         assert_eq!(epoch1 as HummockEpoch, uploader.max_syncing_epoch);
-        assert_eq!(0, uploader.sealed_data.imm_count());
-        assert!(uploader.sealed_data.spilled_data.is_empty());
-        assert_eq!(1, uploader.syncing_data.len());
-        let syncing_data = uploader.syncing_data.front().unwrap();
-        assert_eq!(epoch1 as HummockEpoch, syncing_data.sync_epoch());
+        assert_eq!(1, uploader.data().syncing_data.len());
+        let syncing_data = uploader.data().syncing_data.front().unwrap();
+        assert_eq!(epoch1 as HummockEpoch, syncing_data.sync_epoch);
         assert!(syncing_data.uploaded.is_empty());
         assert!(syncing_data.uploading_tasks.is_some());
 
         match uploader.next_event().await {
-            UploaderEvent::SyncFinish(finished_epoch, ssts) => {
+            UploaderEvent::SyncFinish(finished_epoch, data) => {
                 assert_eq!(epoch1, finished_epoch);
-                assert_eq!(1, ssts.len());
-                let staging_sst = ssts.first().unwrap();
+                let SyncedData {
+                    newly_upload_ssts,
+                    uploaded_ssts,
+                    table_watermarks,
+                } = data;
+                assert_eq!(1, newly_upload_ssts.len());
+                let staging_sst = newly_upload_ssts.first().unwrap();
                 assert_eq!(&vec![epoch1], staging_sst.epochs());
-                assert_eq!(&vec![imm.batch_id()], staging_sst.imm_ids());
-                assert_eq!(&dummy_success_upload_output(), staging_sst.sstable_infos());
+                assert_eq!(
+                    &HashMap::from_iter([(TEST_LOCAL_INSTANCE_ID, vec![imm.batch_id()])]),
+                    staging_sst.imm_ids()
+                );
+                assert_eq!(
+                    &dummy_success_upload_output().new_value_ssts,
+                    staging_sst.sstable_infos()
+                );
+                assert!(uploaded_ssts.is_empty());
+                assert!(table_watermarks.is_empty());
             }
             _ => unreachable!(),
         };
         assert_eq!(epoch1, uploader.max_synced_epoch());
-        let synced_data = uploader.get_synced_data(epoch1).unwrap();
-        let ssts = &synced_data.as_ref().unwrap().staging_ssts;
-        assert_eq!(1, ssts.len());
-        let staging_sst = ssts.first().unwrap();
-        assert_eq!(&vec![epoch1], staging_sst.epochs());
-        assert_eq!(&vec![imm.batch_id()], staging_sst.imm_ids());
-        assert_eq!(&dummy_success_upload_output(), staging_sst.sstable_infos());
 
         let new_pinned_version = uploader
             .context
             .pinned_version
             .new_pin_version(test_hummock_version(epoch1));
         uploader.update_pinned_version(new_pinned_version);
-        assert!(uploader.synced_data.is_empty());
         assert_eq!(epoch1, uploader.max_committed_epoch());
     }
 
     #[tokio::test]
-    async fn test_uploader_merge_imms_without_flush() {
+    async fn test_empty_uploader_sync() {
         let mut uploader = test_uploader(dummy_success_upload_future);
-        let mut all_imms = VecDeque::new();
-        // assume a chckpoint consists of 11 epochs
-        let ckpt_intervals = 11;
-        let imm_merge_threshold: usize = uploader.imm_merge_threshold();
-
-        // For each epoch, we gen imm for 2 shards and add them to uploader and seal the epoch
-        // afterward. check uploader's state after each epoch has been sealed
-        // When we get IMM_MERGE_THRESHOLD epochs, there should be merging task started for sealed
-        // data. Then we await the merging task and check the uploader's state again.
-        let mut merged_imms = VecDeque::new();
-
-        let mut epoch = INITIAL_EPOCH;
-        for i in 1..=ckpt_intervals {
-            epoch.inc_epoch();
-            let mut imm1 = gen_imm(epoch).await;
-            let mut imm2 = gen_imm(epoch).await;
-
-            imm1.instance_id = 1 as LocalInstanceId;
-            imm2.instance_id = 2 as LocalInstanceId;
-
-            uploader.add_imm(imm1.clone());
-            uploader.add_imm(imm2.clone());
-
-            // newer imm comes in front
-            all_imms.push_front(imm1);
-            all_imms.push_front(imm2);
-
-            uploader.seal_epoch(epoch);
-
-            assert_eq!(epoch, uploader.max_sealed_epoch);
-            // check sealed data has two imms
-            let imms_by_epoch = uploader.sealed_data.imms_by_epoch();
-            if let Some((e, imms)) = imms_by_epoch.last_key_value()
-                && *e == epoch
-            {
-                assert_eq!(2, imms.len());
-            }
-
-            let epoch_cnt = i;
-
-            if epoch_cnt < imm_merge_threshold {
-                assert!(uploader.sealed_data.merging_tasks.is_empty());
-                assert!(uploader.sealed_data.spilled_data.is_empty());
-                assert_eq!(epoch_cnt, uploader.sealed_data.epochs.len());
-            } else {
-                assert_eq!(epoch_cnt, uploader.sealed_data.epochs.len());
-
-                let unmerged_imm_cnt: usize = epoch_cnt - imm_merge_threshold * merged_imms.len();
-
-                if unmerged_imm_cnt < imm_merge_threshold {
-                    continue;
-                }
-
-                let imms_by_shard = &mut uploader.sealed_data.imms_by_table_shard;
-                // check shard 1
-                if let Some(imms) = imms_by_shard.get(&(TEST_TABLE_ID, 1 as LocalInstanceId)) {
-                    assert_eq!(imm_merge_threshold, imms.len());
-                }
-
-                // check shard 2
-                if let Some(imms) = imms_by_shard.get(&(TEST_TABLE_ID, 2 as LocalInstanceId)) {
-                    assert_eq!(imm_merge_threshold, imms.len());
-                }
+        let epoch1 = INITIAL_EPOCH.next_epoch();
 
-                // we have enough sealed imms, start merging task
-                println!("start merging task for epoch {}", epoch);
-                uploader.start_merge_imms(epoch);
-                assert!(!uploader.sealed_data.merging_tasks.is_empty());
-                assert!(uploader.sealed_data.spilled_data.is_empty());
-
-                // check after generate merging task
-                if let Some(imms) = uploader
-                    .sealed_data
-                    .imms_by_table_shard
-                    .get(&(TEST_TABLE_ID, 1 as LocalInstanceId))
-                {
-                    assert_eq!(0, imms.len());
-                }
-                if let Some(imms) = uploader
-                    .sealed_data
-                    .imms_by_table_shard
-                    .get(&(TEST_TABLE_ID, 2 as LocalInstanceId))
-                {
-                    assert_eq!(0, imms.len());
-                }
+        uploader.start_sync_epoch_for_test(epoch1);
+        assert_eq!(epoch1, uploader.max_syncing_epoch);
 
-                // poll the merging task and check the result
-                match uploader.next_event().await {
-                    UploaderEvent::ImmMerged(output) => {
-                        println!("merging task success for epoch {}", epoch);
-                        merged_imms.push_front(output.merged_imm);
-                    }
-                    _ => unreachable!(),
-                };
+        match uploader.next_event().await {
+            UploaderEvent::SyncFinish(finished_epoch, data) => {
+                assert_eq!(epoch1, finished_epoch);
+                assert!(data.uploaded_ssts.is_empty());
+                assert!(data.newly_upload_ssts.is_empty());
             }
-        }
+            _ => unreachable!(),
+        };
+        assert_eq!(epoch1, uploader.max_synced_epoch());
+        let new_pinned_version = uploader
+            .context
+            .pinned_version
+            .new_pin_version(test_hummock_version(epoch1));
+        uploader.update_pinned_version(new_pinned_version);
+        assert!(uploader.data().syncing_data.is_empty());
+        assert_eq!(epoch1, uploader.max_committed_epoch());
     }
 
     #[tokio::test]
@@ -1639,17 +1729,18 @@ mod tests {
         let epoch2 = epoch1.next_epoch();
         let imm = gen_imm(epoch2).await;
         // epoch1 is empty while epoch2 is not. Going to seal empty epoch1.
-        uploader.add_imm(imm);
-        uploader.seal_epoch(epoch1);
-        assert_eq!(epoch1, uploader.max_sealed_epoch);
+        uploader.init_instance(TEST_LOCAL_INSTANCE_ID, TEST_TABLE_ID, epoch1);
+        uploader.local_seal_epoch_for_test(TEST_LOCAL_INSTANCE_ID, epoch1);
+        uploader.add_imm(TEST_LOCAL_INSTANCE_ID, imm);
 
-        uploader.start_sync_epoch(epoch1);
+        uploader.start_sync_epoch_for_test(epoch1);
         assert_eq!(epoch1, uploader.max_syncing_epoch);
 
         match uploader.next_event().await {
-            UploaderEvent::SyncFinish(finished_epoch, ssts) => {
+            UploaderEvent::SyncFinish(finished_epoch, data) => {
                 assert_eq!(epoch1, finished_epoch);
-                assert!(ssts.is_empty());
+                assert!(data.uploaded_ssts.is_empty());
+                assert!(data.newly_upload_ssts.is_empty());
             }
             _ => unreachable!(),
         };
@@ -1659,104 +1750,18 @@ mod tests {
             .pinned_version
             .new_pin_version(test_hummock_version(epoch1));
         uploader.update_pinned_version(new_pinned_version);
-        assert!(uploader.synced_data.is_empty());
+        assert!(uploader.data().syncing_data.is_empty());
         assert_eq!(epoch1, uploader.max_committed_epoch());
     }
 
-    #[tokio::test]
-    async fn test_drop_success_merging_task() {
-        let table_id = TableId { table_id: 1004 };
-        let shared_buffer_items1: Vec<(Vec<u8>, HummockValue<Bytes>)> = vec![
-            (
-                iterator_test_table_key_of(1),
-                HummockValue::put(Bytes::from("value1")),
-            ),
-            (
-                iterator_test_table_key_of(2),
-                HummockValue::put(Bytes::from("value2")),
-            ),
-            (
-                iterator_test_table_key_of(3),
-                HummockValue::put(Bytes::from("value3")),
-            ),
-        ];
-        let epoch = test_epoch(1);
-        let imm1 = SharedBufferBatch::for_test(
-            transform_shared_buffer(shared_buffer_items1.clone()),
-            epoch,
-            table_id,
-        );
-        let shared_buffer_items2: Vec<(Vec<u8>, HummockValue<Bytes>)> = vec![
-            (
-                iterator_test_table_key_of(1),
-                HummockValue::put(Bytes::from("value12")),
-            ),
-            (
-                iterator_test_table_key_of(2),
-                HummockValue::put(Bytes::from("value22")),
-            ),
-            (
-                iterator_test_table_key_of(3),
-                HummockValue::put(Bytes::from("value32")),
-            ),
-        ];
-        let epoch = test_epoch(2);
-        let imm2 = SharedBufferBatch::for_test(
-            transform_shared_buffer(shared_buffer_items2.clone()),
-            epoch,
-            table_id,
-        );
-
-        let shared_buffer_items3: Vec<(Vec<u8>, HummockValue<Bytes>)> = vec![
-            (
-                iterator_test_table_key_of(1),
-                HummockValue::put(Bytes::from("value13")),
-            ),
-            (
-                iterator_test_table_key_of(2),
-                HummockValue::put(Bytes::from("value23")),
-            ),
-            (
-                iterator_test_table_key_of(3),
-                HummockValue::put(Bytes::from("value33")),
-            ),
-        ];
-        let epoch = test_epoch(3);
-        let imm3 = SharedBufferBatch::for_test(
-            transform_shared_buffer(shared_buffer_items3.clone()),
-            epoch,
-            table_id,
-        );
-
-        // newer data comes first
-        let imms = vec![imm3, imm2, imm1];
-        let context = test_uploader_context(dummy_success_upload_future);
-        let mut task = MergingImmTask::new(table_id, 0, imms, None, &context);
-        let sleep = tokio::time::sleep(Duration::from_millis(500));
-        tokio::select! {
-            _ = sleep => {
-                println!("sleep timeout")
-            }
-            imm = &mut task => {
-                println!("merging task success");
-                assert_eq!(table_id, imm.table_id);
-                assert_eq!(9, imm.value_count());
-            }
-        }
-        task.join_handle.abort();
-        println!("merging task abort success");
-    }
-
     #[tokio::test]
     async fn test_uploader_poll_empty() {
         let mut uploader = test_uploader(dummy_success_upload_future);
-        assert!(poll_fn(|cx| uploader.poll_syncing_task(cx)).await.is_none());
-        assert!(poll_fn(|cx| uploader.poll_sealed_spill_task(cx))
-            .await
-            .is_none());
-        assert!(poll_fn(|cx| uploader.poll_unsealed_spill_task(cx))
+        let data = must_match!(&mut uploader.state, UploaderState::Working(data) => data);
+        assert!(poll_fn(|cx| data.poll_syncing_task(cx, &uploader.context))
             .await
             .is_none());
+        assert!(poll_fn(|cx| data.poll_spill_task(cx)).await.is_none());
     }
 
     #[tokio::test]
@@ -1777,49 +1782,45 @@ mod tests {
         uploader.update_pinned_version(version1);
         assert_eq!(epoch1, uploader.max_synced_epoch);
         assert_eq!(epoch1, uploader.max_syncing_epoch);
-        assert_eq!(epoch1, uploader.max_sealed_epoch);
 
-        uploader.add_imm(gen_imm(epoch6).await);
+        uploader.init_instance(TEST_LOCAL_INSTANCE_ID, TEST_TABLE_ID, epoch6);
+        uploader.add_imm(TEST_LOCAL_INSTANCE_ID, gen_imm(epoch6).await);
         uploader.update_pinned_version(version2);
         assert_eq!(epoch2, uploader.max_synced_epoch);
         assert_eq!(epoch2, uploader.max_syncing_epoch);
-        assert_eq!(epoch2, uploader.max_sealed_epoch);
 
-        uploader.seal_epoch(epoch6);
-        assert_eq!(epoch6, uploader.max_sealed_epoch);
+        uploader.local_seal_epoch_for_test(TEST_LOCAL_INSTANCE_ID, epoch6);
         uploader.update_pinned_version(version3);
         assert_eq!(epoch3, uploader.max_synced_epoch);
         assert_eq!(epoch3, uploader.max_syncing_epoch);
-        assert_eq!(epoch6, uploader.max_sealed_epoch);
 
-        uploader.start_sync_epoch(epoch6);
+        uploader.start_sync_epoch_for_test(epoch6);
         assert_eq!(epoch6, uploader.max_syncing_epoch);
         uploader.update_pinned_version(version4);
         assert_eq!(epoch4, uploader.max_synced_epoch);
         assert_eq!(epoch6, uploader.max_syncing_epoch);
-        assert_eq!(epoch6, uploader.max_sealed_epoch);
 
         match uploader.next_event().await {
             UploaderEvent::SyncFinish(epoch, _) => {
                 assert_eq!(epoch6, epoch);
             }
             UploaderEvent::DataSpilled(_) => unreachable!(),
-            UploaderEvent::ImmMerged(_) => unreachable!(),
         }
         uploader.update_pinned_version(version5);
         assert_eq!(epoch6, uploader.max_synced_epoch);
         assert_eq!(epoch6, uploader.max_syncing_epoch);
-        assert_eq!(epoch6, uploader.max_sealed_epoch);
     }
 
-    fn prepare_uploader_order_test() -> (
+    fn prepare_uploader_order_test(
+        config: &StorageOpts,
+        skip_schedule: bool,
+    ) -> (
         BufferTracker,
         HummockUploader,
-        impl Fn(Vec<ImmId>) -> (BoxFuture<'static, ()>, oneshot::Sender<()>),
+        impl Fn(HashMap<LocalInstanceId, Vec<ImmId>>) -> (BoxFuture<'static, ()>, oneshot::Sender<()>),
     ) {
-        // flush threshold is 0. Flush anyway
-        let buffer_tracker =
-            BufferTracker::new(usize::MAX, 0, GenericGauge::new("test", "test").unwrap());
+        let gauge = GenericGauge::new("test", "test").unwrap();
+        let buffer_tracker = BufferTracker::from_storage_opts(config, gauge);
         // (the started task send the imm ids of payload, the started task wait for finish notify)
         #[allow(clippy::type_complexity)]
         let task_notifier_holder: Arc<
@@ -1828,7 +1829,7 @@ mod tests {
 
         let new_task_notifier = {
             let task_notifier_holder = task_notifier_holder.clone();
-            move |imm_ids: Vec<ImmId>| {
+            move |imm_ids: HashMap<LocalInstanceId, Vec<ImmId>>| {
                 let (start_tx, start_rx) = oneshot::channel();
                 let (finish_tx, finish_rx) = oneshot::channel();
                 task_notifier_holder
@@ -1844,29 +1845,31 @@ mod tests {
         };
 
         let config = StorageOpts::default();
-        let compaction_executor = Arc::new(CompactionExecutor::new(None));
         let uploader = HummockUploader::new(
             Arc::new(HummockStateStoreMetrics::unused()),
             initial_pinned_version(),
             Arc::new({
-                move |_: UploadTaskPayload, task_info: UploadTaskInfo| {
+                move |_, task_info: UploadTaskInfo| {
                     let task_notifier_holder = task_notifier_holder.clone();
-                    let (start_tx, finish_rx) = task_notifier_holder.lock().pop_back().unwrap();
+                    let task_item = task_notifier_holder.lock().pop_back();
                     let start_epoch = *task_info.epochs.last().unwrap();
                     let end_epoch = *task_info.epochs.first().unwrap();
                     assert!(end_epoch >= start_epoch);
                     spawn(async move {
                         let ssts = gen_sstable_info(start_epoch, end_epoch);
-                        start_tx.send(task_info).unwrap();
-                        finish_rx.await.unwrap();
+                        if !skip_schedule {
+                            let (start_tx, finish_rx) = task_item.unwrap();
+                            start_tx.send(task_info).unwrap();
+                            finish_rx.await.unwrap();
+                        }
                         Ok(UploadTaskOutput {
-                            ssts,
+                            new_value_ssts: ssts,
+                            old_value_ssts: vec![],
                             wait_poll_timer: None,
                         })
                     })
                 }
             }),
-            default_spawn_merging_task(compaction_executor, None),
             buffer_tracker.clone(),
             &config,
         );
@@ -1883,28 +1886,43 @@ mod tests {
                 .is_pending()
         )
     }
-
     #[tokio::test]
     async fn test_uploader_finish_in_order() {
-        let (buffer_tracker, mut uploader, new_task_notifier) = prepare_uploader_order_test();
+        let config = StorageOpts {
+            shared_buffer_capacity_mb: 1024 * 1024,
+            shared_buffer_flush_ratio: 0.0,
+            ..Default::default()
+        };
+        let (buffer_tracker, mut uploader, new_task_notifier) =
+            prepare_uploader_order_test(&config, false);
 
         let epoch1 = INITIAL_EPOCH.next_epoch();
         let epoch2 = epoch1.next_epoch();
         let memory_limiter = buffer_tracker.get_memory_limiter().clone();
         let memory_limiter = Some(memory_limiter.deref());
 
+        let instance_id1 = 1;
+        let instance_id2 = 2;
+
+        uploader.init_instance(instance_id1, TEST_TABLE_ID, epoch1);
+        uploader.init_instance(instance_id2, TEST_TABLE_ID, epoch2);
+
         // imm2 contains data in newer epoch, but added first
         let imm2 = gen_imm_with_limiter(epoch2, memory_limiter).await;
-        uploader.add_imm(imm2.clone());
+        uploader.add_imm(instance_id2, imm2.clone());
         let imm1_1 = gen_imm_with_limiter(epoch1, memory_limiter).await;
-        uploader.add_imm(imm1_1.clone());
+        uploader.add_imm(instance_id1, imm1_1.clone());
         let imm1_2 = gen_imm_with_limiter(epoch1, memory_limiter).await;
-        uploader.add_imm(imm1_2.clone());
+        uploader.add_imm(instance_id1, imm1_2.clone());
 
         // imm1 will be spilled first
+        let epoch1_spill_payload12 =
+            HashMap::from_iter([(instance_id1, vec![imm1_2.clone(), imm1_1.clone()])]);
+        let epoch2_spill_payload = HashMap::from_iter([(instance_id2, vec![imm2.clone()])]);
         let (await_start1, finish_tx1) =
-            new_task_notifier(vec![imm1_2.batch_id(), imm1_1.batch_id()]);
-        let (await_start2, finish_tx2) = new_task_notifier(vec![imm2.batch_id()]);
+            new_task_notifier(get_payload_imm_ids(&epoch1_spill_payload12));
+        let (await_start2, finish_tx2) =
+            new_task_notifier(get_payload_imm_ids(&epoch2_spill_payload));
         uploader.may_flush();
         await_start1.await;
         await_start2.await;
@@ -1916,51 +1934,60 @@ mod tests {
 
         finish_tx1.send(()).unwrap();
         if let UploaderEvent::DataSpilled(sst) = uploader.next_event().await {
-            assert_eq!(&vec![imm1_2.batch_id(), imm1_1.batch_id()], sst.imm_ids());
+            assert_eq!(&get_payload_imm_ids(&epoch1_spill_payload12), sst.imm_ids());
             assert_eq!(&vec![epoch1], sst.epochs());
         } else {
             unreachable!("")
         }
 
         if let UploaderEvent::DataSpilled(sst) = uploader.next_event().await {
-            assert_eq!(&vec![imm2.batch_id()], sst.imm_ids());
+            assert_eq!(&get_payload_imm_ids(&epoch2_spill_payload), sst.imm_ids());
             assert_eq!(&vec![epoch2], sst.epochs());
         } else {
             unreachable!("")
         }
 
         let imm1_3 = gen_imm_with_limiter(epoch1, memory_limiter).await;
-        uploader.add_imm(imm1_3.clone());
-        let (await_start1_3, finish_tx1_3) = new_task_notifier(vec![imm1_3.batch_id()]);
+        uploader.add_imm(instance_id1, imm1_3.clone());
+        let epoch1_spill_payload3 = HashMap::from_iter([(instance_id1, vec![imm1_3.clone()])]);
+        let (await_start1_3, finish_tx1_3) =
+            new_task_notifier(get_payload_imm_ids(&epoch1_spill_payload3));
         uploader.may_flush();
         await_start1_3.await;
         let imm1_4 = gen_imm_with_limiter(epoch1, memory_limiter).await;
-        uploader.add_imm(imm1_4.clone());
-        let (await_start1_4, finish_tx1_4) = new_task_notifier(vec![imm1_4.batch_id()]);
-        uploader.seal_epoch(epoch1);
-        uploader.start_sync_epoch(epoch1);
+        uploader.add_imm(instance_id1, imm1_4.clone());
+        let epoch1_sync_payload = HashMap::from_iter([(instance_id1, vec![imm1_4.clone()])]);
+        let (await_start1_4, finish_tx1_4) =
+            new_task_notifier(get_payload_imm_ids(&epoch1_sync_payload));
+        uploader.local_seal_epoch_for_test(instance_id1, epoch1);
+        uploader.start_sync_epoch_for_test(epoch1);
         await_start1_4.await;
+        let epoch3 = epoch2.next_epoch();
 
-        uploader.seal_epoch(epoch2);
+        uploader.local_seal_epoch_for_test(instance_id1, epoch2);
+        uploader.local_seal_epoch_for_test(instance_id2, epoch2);
 
         // current uploader state:
         // unsealed: empty
         // sealed: epoch2: uploaded sst([imm2])
         // syncing: epoch1: uploading: [imm1_4], [imm1_3], uploaded: sst([imm1_2, imm1_1])
 
-        let epoch3 = epoch2.next_epoch();
         let imm3_1 = gen_imm_with_limiter(epoch3, memory_limiter).await;
-        uploader.add_imm(imm3_1.clone());
-        let (await_start3_1, finish_tx3_1) = new_task_notifier(vec![imm3_1.batch_id()]);
+        let epoch3_spill_payload1 = HashMap::from_iter([(instance_id1, vec![imm3_1.clone()])]);
+        uploader.add_imm(instance_id1, imm3_1.clone());
+        let (await_start3_1, finish_tx3_1) =
+            new_task_notifier(get_payload_imm_ids(&epoch3_spill_payload1));
         uploader.may_flush();
         await_start3_1.await;
         let imm3_2 = gen_imm_with_limiter(epoch3, memory_limiter).await;
-        uploader.add_imm(imm3_2.clone());
-        let (await_start3_2, finish_tx3_2) = new_task_notifier(vec![imm3_2.batch_id()]);
+        let epoch3_spill_payload2 = HashMap::from_iter([(instance_id2, vec![imm3_2.clone()])]);
+        uploader.add_imm(instance_id2, imm3_2.clone());
+        let (await_start3_2, finish_tx3_2) =
+            new_task_notifier(get_payload_imm_ids(&epoch3_spill_payload2));
         uploader.may_flush();
         await_start3_2.await;
         let imm3_3 = gen_imm_with_limiter(epoch3, memory_limiter).await;
-        uploader.add_imm(imm3_3.clone());
+        uploader.add_imm(instance_id1, imm3_3.clone());
 
         // current uploader state:
         // unsealed: epoch3: imm: imm3_3, uploading: [imm3_2], [imm3_1]
@@ -1968,8 +1995,9 @@ mod tests {
         // syncing: epoch1: uploading: [imm1_4], [imm1_3], uploaded: sst([imm1_2, imm1_1])
 
         let epoch4 = epoch3.next_epoch();
+        uploader.local_seal_epoch_for_test(instance_id1, epoch3);
         let imm4 = gen_imm_with_limiter(epoch4, memory_limiter).await;
-        uploader.add_imm(imm4.clone());
+        uploader.add_imm(instance_id1, imm4.clone());
         assert_uploader_pending(&mut uploader).await;
 
         // current uploader state:
@@ -1984,28 +2012,25 @@ mod tests {
         assert_uploader_pending(&mut uploader).await;
         finish_tx1_3.send(()).unwrap();
 
-        if let UploaderEvent::SyncFinish(epoch, newly_upload_sst) = uploader.next_event().await {
+        if let UploaderEvent::SyncFinish(epoch, data) = uploader.next_event().await {
             assert_eq!(epoch1, epoch);
-            assert_eq!(2, newly_upload_sst.len());
-            assert_eq!(&vec![imm1_4.batch_id()], newly_upload_sst[0].imm_ids());
-            assert_eq!(&vec![imm1_3.batch_id()], newly_upload_sst[1].imm_ids());
+            assert_eq!(2, data.newly_upload_ssts.len());
+            assert_eq!(1, data.uploaded_ssts.len());
+            assert_eq!(
+                &get_payload_imm_ids(&epoch1_sync_payload),
+                data.newly_upload_ssts[0].imm_ids()
+            );
+            assert_eq!(
+                &get_payload_imm_ids(&epoch1_spill_payload3),
+                data.newly_upload_ssts[1].imm_ids()
+            );
+            assert_eq!(
+                &get_payload_imm_ids(&epoch1_spill_payload12),
+                data.uploaded_ssts[0].imm_ids()
+            );
         } else {
             unreachable!("should be sync finish");
         }
-        assert_eq!(epoch1, uploader.max_synced_epoch);
-        let synced_data1 = &uploader
-            .get_synced_data(epoch1)
-            .unwrap()
-            .as_ref()
-            .unwrap()
-            .staging_ssts;
-        assert_eq!(3, synced_data1.len());
-        assert_eq!(&vec![imm1_4.batch_id()], synced_data1[0].imm_ids());
-        assert_eq!(&vec![imm1_3.batch_id()], synced_data1[1].imm_ids());
-        assert_eq!(
-            &vec![imm1_2.batch_id(), imm1_1.batch_id()],
-            synced_data1[2].imm_ids()
-        );
 
         // current uploader state:
         // unsealed: epoch3: imm: imm3_3, uploading: [imm3_2], [imm3_1]
@@ -2014,22 +2039,19 @@ mod tests {
         // syncing: empty
         // synced: epoch1: sst([imm1_4]), sst([imm1_3]), sst([imm1_2, imm1_1])
 
-        uploader.start_sync_epoch(epoch2);
-        if let UploaderEvent::SyncFinish(epoch, newly_upload_sst) = uploader.next_event().await {
+        uploader.start_sync_epoch_for_test(epoch2);
+        if let UploaderEvent::SyncFinish(epoch, data) = uploader.next_event().await {
             assert_eq!(epoch2, epoch);
-            assert!(newly_upload_sst.is_empty());
+            assert!(data.newly_upload_ssts.is_empty());
+            assert_eq!(data.uploaded_ssts.len(), 1);
+            assert_eq!(
+                &get_payload_imm_ids(&epoch2_spill_payload),
+                data.uploaded_ssts[0].imm_ids()
+            );
         } else {
             unreachable!("should be sync finish");
         }
         assert_eq!(epoch2, uploader.max_synced_epoch);
-        let synced_data2 = &uploader
-            .get_synced_data(epoch2)
-            .unwrap()
-            .as_ref()
-            .unwrap()
-            .staging_ssts;
-        assert_eq!(1, synced_data2.len());
-        assert_eq!(&vec![imm2.batch_id()], synced_data2[0].imm_ids());
 
         // current uploader state:
         // unsealed: epoch3: imm: imm3_3, uploading: [imm3_2], [imm3_1]
@@ -2039,9 +2061,9 @@ mod tests {
         // synced: epoch1: sst([imm1_4]), sst([imm1_3]), sst([imm1_2, imm1_1])
         //         epoch2: sst([imm2])
 
-        uploader.seal_epoch(epoch3);
+        uploader.local_seal_epoch_for_test(instance_id2, epoch3);
         if let UploaderEvent::DataSpilled(sst) = uploader.next_event().await {
-            assert_eq!(&vec![imm3_1.batch_id()], sst.imm_ids());
+            assert_eq!(&get_payload_imm_ids(&epoch3_spill_payload1), sst.imm_ids());
         } else {
             unreachable!("should be data spilled");
         }
@@ -2053,10 +2075,12 @@ mod tests {
         // synced: epoch1: sst([imm1_4]), sst([imm1_3]), sst([imm1_2, imm1_1])
         //         epoch2: sst([imm2])
 
-        uploader.seal_epoch(epoch4);
+        uploader.local_seal_epoch_for_test(instance_id1, epoch4);
+        uploader.local_seal_epoch_for_test(instance_id2, epoch4);
+        let epoch4_sync_payload = HashMap::from_iter([(instance_id1, vec![imm4, imm3_3])]);
         let (await_start4_with_3_3, finish_tx4_with_3_3) =
-            new_task_notifier(vec![imm4.batch_id(), imm3_3.batch_id()]);
-        uploader.start_sync_epoch(epoch4);
+            new_task_notifier(get_payload_imm_ids(&epoch4_sync_payload));
+        uploader.start_sync_epoch_for_test(epoch4);
         await_start4_with_3_3.await;
 
         // current uploader state:
@@ -2071,32 +2095,26 @@ mod tests {
         assert_uploader_pending(&mut uploader).await;
         finish_tx4_with_3_3.send(()).unwrap();
 
-        if let UploaderEvent::SyncFinish(epoch, newly_upload_sst) = uploader.next_event().await {
+        if let UploaderEvent::SyncFinish(epoch, data) = uploader.next_event().await {
             assert_eq!(epoch4, epoch);
-            assert_eq!(2, newly_upload_sst.len());
+            assert_eq!(2, data.newly_upload_ssts.len());
             assert_eq!(
-                &vec![imm4.batch_id(), imm3_3.batch_id()],
-                newly_upload_sst[0].imm_ids()
+                &get_payload_imm_ids(&epoch4_sync_payload),
+                data.newly_upload_ssts[0].imm_ids()
             );
-            assert_eq!(&vec![imm3_2.batch_id()], newly_upload_sst[1].imm_ids());
+            assert_eq!(
+                &get_payload_imm_ids(&epoch3_spill_payload2),
+                data.newly_upload_ssts[1].imm_ids()
+            );
+            assert_eq!(1, data.uploaded_ssts.len());
+            assert_eq!(
+                &get_payload_imm_ids(&epoch3_spill_payload1),
+                data.uploaded_ssts[0].imm_ids(),
+            )
         } else {
             unreachable!("should be sync finish");
         }
         assert_eq!(epoch4, uploader.max_synced_epoch);
-        let synced_data4 = &uploader
-            .get_synced_data(epoch4)
-            .unwrap()
-            .as_ref()
-            .unwrap()
-            .staging_ssts;
-        assert_eq!(3, synced_data4.len());
-        assert_eq!(&vec![epoch4, epoch3], synced_data4[0].epochs());
-        assert_eq!(
-            &vec![imm4.batch_id(), imm3_3.batch_id()],
-            synced_data4[0].imm_ids()
-        );
-        assert_eq!(&vec![imm3_2.batch_id()], synced_data4[1].imm_ids());
-        assert_eq!(&vec![imm3_1.batch_id()], synced_data4[2].imm_ids());
 
         // current uploader state:
         // unsealed: empty
@@ -2106,4 +2124,47 @@ mod tests {
         //         epoch2: sst([imm2])
         //         epoch4: sst([imm4, imm3_3]), sst([imm3_2]), sst([imm3_1])
     }
+
+    #[tokio::test]
+    async fn test_uploader_frequently_flush() {
+        let config = StorageOpts {
+            shared_buffer_capacity_mb: 10,
+            shared_buffer_flush_ratio: 0.8,
+            // This test will fail when we set it to 0
+            shared_buffer_min_batch_flush_size_mb: 1,
+            ..Default::default()
+        };
+        let (buffer_tracker, mut uploader, _new_task_notifier) =
+            prepare_uploader_order_test(&config, true);
+
+        let epoch1 = INITIAL_EPOCH.next_epoch();
+        let epoch2 = epoch1.next_epoch();
+        let instance_id1 = 1;
+        let instance_id2 = 2;
+        let flush_threshold = buffer_tracker.flush_threshold();
+        let memory_limiter = buffer_tracker.get_memory_limiter().clone();
+
+        uploader.init_instance(instance_id1, TEST_TABLE_ID, epoch1);
+        uploader.init_instance(instance_id2, TEST_TABLE_ID, epoch2);
+
+        // imm2 contains data in newer epoch, but added first
+        let mut total_memory = 0;
+        while total_memory < flush_threshold {
+            let imm = gen_imm_with_limiter(epoch2, Some(memory_limiter.as_ref())).await;
+            total_memory += imm.size();
+            if total_memory > flush_threshold {
+                break;
+            }
+            uploader.add_imm(instance_id2, imm);
+        }
+        let imm = gen_imm_with_limiter(epoch1, Some(memory_limiter.as_ref())).await;
+        uploader.add_imm(instance_id1, imm);
+        assert!(uploader.may_flush());
+
+        for _ in 0..10 {
+            let imm = gen_imm_with_limiter(epoch1, Some(memory_limiter.as_ref())).await;
+            uploader.add_imm(instance_id1, imm);
+            assert!(!uploader.may_flush());
+        }
+    }
 }
diff --git a/src/storage/src/hummock/file_cache/store.rs b/src/storage/src/hummock/file_cache/store.rs
deleted file mode 100644
index dcf121966886c..0000000000000
--- a/src/storage/src/hummock/file_cache/store.rs
+++ /dev/null
@@ -1,803 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::fmt::Debug;
-use std::hash::Hash;
-use std::path::PathBuf;
-use std::sync::Arc;
-
-use bytes::{Buf, BufMut, Bytes};
-use foyer::common::code::{CodingResult, Cursor, Key, Value};
-use foyer::intrusive::eviction::lfu::LfuConfig;
-use foyer::storage::admission::rated_ticket::RatedTicketAdmissionPolicy;
-use foyer::storage::admission::AdmissionPolicy;
-use foyer::storage::compress::Compression;
-use foyer::storage::device::fs::FsDeviceConfig;
-pub use foyer::storage::metrics::set_metrics_registry as set_foyer_metrics_registry;
-use foyer::storage::reinsertion::ReinsertionPolicy;
-use foyer::storage::runtime::{
-    RuntimeConfig, RuntimeLazyStore, RuntimeLazyStoreConfig, RuntimeLazyStoreWriter,
-};
-use foyer::storage::storage::{Storage, StorageWriter};
-use foyer::storage::store::{LfuFsStoreConfig, NoneStore, NoneStoreWriter};
-use risingwave_hummock_sdk::HummockSstableObjectId;
-
-use crate::hummock::{Block, HummockResult, Sstable, SstableMeta};
-
-pub mod preclude {
-    pub use foyer::storage::storage::{
-        AsyncStorageExt, ForceStorageExt, Storage, StorageExt, StorageWriter,
-    };
-}
-
-pub type Result<T> = core::result::Result<T, FileCacheError>;
-
-pub type FileCacheEvictionConfig = foyer::intrusive::eviction::lfu::LfuConfig;
-pub type DeviceConfig = foyer::storage::device::fs::FsDeviceConfig;
-
-pub type FileCacheResult<T> = foyer::storage::error::Result<T>;
-pub type FileCacheError = foyer::storage::error::Error;
-pub type FileCacheCompression = foyer::storage::compress::Compression;
-
-fn copy(src: impl AsRef<[u8]>, mut dst: impl BufMut + AsRef<[u8]>) -> usize {
-    let n = std::cmp::min(src.as_ref().len(), dst.as_ref().len());
-    dst.put_slice(&src.as_ref()[..n]);
-    n
-}
-
-#[derive(Debug)]
-pub struct FileCacheConfig<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    pub name: String,
-    pub dir: PathBuf,
-    pub capacity: usize,
-    pub file_capacity: usize,
-    pub device_align: usize,
-    pub device_io_size: usize,
-    pub flushers: usize,
-    pub reclaimers: usize,
-    pub recover_concurrency: usize,
-    pub lfu_window_to_cache_size_ratio: usize,
-    pub lfu_tiny_lru_capacity_ratio: f64,
-    pub insert_rate_limit: usize,
-    pub catalog_bits: usize,
-    pub admissions: Vec<Arc<dyn AdmissionPolicy<Key = K, Value = V>>>,
-    pub reinsertions: Vec<Arc<dyn ReinsertionPolicy<Key = K, Value = V>>>,
-    pub compression: Compression,
-}
-
-impl<K, V> Clone for FileCacheConfig<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn clone(&self) -> Self {
-        Self {
-            name: self.name.clone(),
-            dir: self.dir.clone(),
-            capacity: self.capacity,
-            file_capacity: self.file_capacity,
-            device_align: self.device_align,
-            device_io_size: self.device_io_size,
-            flushers: self.flushers,
-            reclaimers: self.reclaimers,
-            recover_concurrency: self.recover_concurrency,
-            lfu_window_to_cache_size_ratio: self.lfu_window_to_cache_size_ratio,
-            lfu_tiny_lru_capacity_ratio: self.lfu_tiny_lru_capacity_ratio,
-            insert_rate_limit: self.insert_rate_limit,
-            catalog_bits: self.catalog_bits,
-            admissions: self.admissions.clone(),
-            reinsertions: self.reinsertions.clone(),
-            compression: self.compression,
-        }
-    }
-}
-
-#[derive(Debug)]
-pub enum FileCacheWriter<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    Foyer {
-        writer: RuntimeLazyStoreWriter<K, V>,
-    },
-    None {
-        writer: NoneStoreWriter<K, V>,
-    },
-}
-
-impl<K, V> From<RuntimeLazyStoreWriter<K, V>> for FileCacheWriter<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn from(writer: RuntimeLazyStoreWriter<K, V>) -> Self {
-        Self::Foyer { writer }
-    }
-}
-
-impl<K, V> From<NoneStoreWriter<K, V>> for FileCacheWriter<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn from(writer: NoneStoreWriter<K, V>) -> Self {
-        Self::None { writer }
-    }
-}
-
-impl<K, V> StorageWriter for FileCacheWriter<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    type Key = K;
-    type Value = V;
-
-    fn key(&self) -> &Self::Key {
-        match self {
-            FileCacheWriter::Foyer { writer } => writer.key(),
-            FileCacheWriter::None { writer } => writer.key(),
-        }
-    }
-
-    fn weight(&self) -> usize {
-        match self {
-            FileCacheWriter::Foyer { writer } => writer.weight(),
-            FileCacheWriter::None { writer } => writer.weight(),
-        }
-    }
-
-    fn judge(&mut self) -> bool {
-        match self {
-            FileCacheWriter::Foyer { writer } => writer.judge(),
-            FileCacheWriter::None { writer } => writer.judge(),
-        }
-    }
-
-    fn force(&mut self) {
-        match self {
-            FileCacheWriter::Foyer { writer } => writer.force(),
-            FileCacheWriter::None { writer } => writer.force(),
-        }
-    }
-
-    async fn finish(self, value: Self::Value) -> FileCacheResult<bool> {
-        match self {
-            FileCacheWriter::Foyer { writer } => writer.finish(value).await,
-            FileCacheWriter::None { writer } => writer.finish(value).await,
-        }
-    }
-
-    fn compression(&self) -> Compression {
-        match self {
-            FileCacheWriter::Foyer { writer } => writer.compression(),
-            FileCacheWriter::None { writer } => writer.compression(),
-        }
-    }
-
-    fn set_compression(&mut self, compression: Compression) {
-        match self {
-            FileCacheWriter::Foyer { writer } => writer.set_compression(compression),
-            FileCacheWriter::None { writer } => writer.set_compression(compression),
-        }
-    }
-}
-
-#[derive(Debug)]
-pub enum FileCache<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    Foyer { store: RuntimeLazyStore<K, V> },
-    None { store: NoneStore<K, V> },
-}
-
-impl<K, V> Clone for FileCache<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    fn clone(&self) -> Self {
-        match self {
-            Self::Foyer { store } => Self::Foyer {
-                store: store.clone(),
-            },
-            Self::None { store } => Self::None {
-                store: store.clone(),
-            },
-        }
-    }
-}
-
-impl<K, V> FileCache<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    pub fn none() -> Self {
-        Self::None {
-            store: NoneStore::default(),
-        }
-    }
-
-    pub fn is_none(&self) -> bool {
-        matches!(self, FileCache::None { .. })
-    }
-}
-
-impl<K, V> Storage for FileCache<K, V>
-where
-    K: Key,
-    V: Value,
-{
-    type Config = FileCacheConfig<K, V>;
-    type Key = K;
-    type Value = V;
-    type Writer = FileCacheWriter<K, V>;
-
-    async fn open(config: Self::Config) -> FileCacheResult<Self> {
-        let mut admissions = config.admissions;
-        if config.insert_rate_limit > 0 {
-            admissions.push(Arc::new(RatedTicketAdmissionPolicy::new(
-                config.insert_rate_limit,
-            )));
-        }
-
-        let c = RuntimeLazyStoreConfig {
-            store: LfuFsStoreConfig {
-                name: config.name.clone(),
-                eviction_config: LfuConfig {
-                    window_to_cache_size_ratio: config.lfu_window_to_cache_size_ratio,
-                    tiny_lru_capacity_ratio: config.lfu_tiny_lru_capacity_ratio,
-                },
-                device_config: FsDeviceConfig {
-                    dir: config.dir,
-                    capacity: config.capacity,
-                    file_capacity: config.file_capacity,
-                    align: config.device_align,
-                    io_size: config.device_io_size,
-                },
-                catalog_bits: config.catalog_bits,
-                admissions,
-                reinsertions: config.reinsertions,
-                flushers: config.flushers,
-                reclaimers: config.reclaimers,
-                clean_region_threshold: config.reclaimers + config.reclaimers / 2,
-                recover_concurrency: config.recover_concurrency,
-                compression: config.compression,
-            }
-            .into(),
-            runtime: RuntimeConfig {
-                worker_threads: None,
-                thread_name: Some(config.name),
-            },
-        };
-        let store = RuntimeLazyStore::open(c).await?;
-        Ok(Self::Foyer { store })
-    }
-
-    fn is_ready(&self) -> bool {
-        match self {
-            FileCache::Foyer { store } => store.is_ready(),
-            FileCache::None { store } => store.is_ready(),
-        }
-    }
-
-    async fn close(&self) -> FileCacheResult<()> {
-        match self {
-            FileCache::Foyer { store } => store.close().await,
-            FileCache::None { store } => store.close().await,
-        }
-    }
-
-    fn writer(&self, key: Self::Key, weight: usize) -> Self::Writer {
-        match self {
-            FileCache::Foyer { store } => store.writer(key, weight).into(),
-            FileCache::None { store } => store.writer(key, weight).into(),
-        }
-    }
-
-    fn exists(&self, key: &Self::Key) -> FileCacheResult<bool> {
-        match self {
-            FileCache::Foyer { store } => store.exists(key),
-            FileCache::None { store } => store.exists(key),
-        }
-    }
-
-    async fn lookup(&self, key: &Self::Key) -> FileCacheResult<Option<Self::Value>> {
-        match self {
-            FileCache::Foyer { store } => store.lookup(key).await,
-            FileCache::None { store } => store.lookup(key).await,
-        }
-    }
-
-    fn remove(&self, key: &Self::Key) -> FileCacheResult<bool> {
-        match self {
-            FileCache::Foyer { store } => store.remove(key),
-            FileCache::None { store } => store.remove(key),
-        }
-    }
-
-    fn clear(&self) -> FileCacheResult<()> {
-        match self {
-            FileCache::Foyer { store } => store.clear(),
-            FileCache::None { store } => store.clear(),
-        }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Ord, Hash)]
-pub struct SstableBlockIndex {
-    pub sst_id: HummockSstableObjectId,
-    pub block_idx: u64,
-}
-
-impl Key for SstableBlockIndex {
-    type Cursor = SstableBlockIndexCursor;
-
-    fn serialized_len(&self) -> usize {
-        8 + 8 // sst_id (8B) + block_idx (8B)
-    }
-
-    fn read(mut buf: &[u8]) -> CodingResult<Self> {
-        let sst_id = buf.get_u64();
-        let block_idx = buf.get_u64();
-        Ok(Self { sst_id, block_idx })
-    }
-
-    fn into_cursor(self) -> Self::Cursor {
-        SstableBlockIndexCursor::new(self)
-    }
-}
-
-#[derive(Debug)]
-pub struct SstableBlockIndexCursor {
-    inner: SstableBlockIndex,
-    pos: u8,
-}
-
-impl SstableBlockIndexCursor {
-    pub fn new(inner: SstableBlockIndex) -> Self {
-        Self { inner, pos: 0 }
-    }
-}
-
-impl std::io::Read for SstableBlockIndexCursor {
-    fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result<usize> {
-        let pos = self.pos;
-        if self.pos < 8 {
-            self.pos += copy(
-                &self.inner.sst_id.to_be_bytes()[self.pos as usize..],
-                &mut buf,
-            ) as u8;
-        }
-        if self.pos < 16 {
-            self.pos += copy(
-                &self.inner.block_idx.to_be_bytes()[self.pos as usize - 8..],
-                &mut buf,
-            ) as u8;
-        }
-        let n = (self.pos - pos) as usize;
-        Ok(n)
-    }
-}
-
-impl Cursor for SstableBlockIndexCursor {
-    type T = SstableBlockIndex;
-
-    fn into_inner(self) -> Self::T {
-        self.inner
-    }
-}
-
-/// [`CachedBlock`] uses different coding for writing to use/bypass compression.
-///
-/// But when reading, it will always be `Loaded`.
-#[derive(Debug, Clone)]
-pub enum CachedBlock {
-    Loaded {
-        block: Box<Block>,
-    },
-    Fetched {
-        bytes: Bytes,
-        uncompressed_capacity: usize,
-    },
-}
-
-impl CachedBlock {
-    pub fn should_compress(&self) -> bool {
-        match self {
-            CachedBlock::Loaded { .. } => true,
-            // TODO(MrCroxx): based on block original compression algorithm?
-            CachedBlock::Fetched { .. } => false,
-        }
-    }
-
-    pub fn try_into_block(self) -> HummockResult<Box<Block>> {
-        let block = match self {
-            CachedBlock::Loaded { block } => block,
-            // for the block was not loaded yet (refill + inflight), we need to decode it.
-            // TODO(MrCroxx): avoid decode twice?
-            CachedBlock::Fetched {
-                bytes,
-                uncompressed_capacity,
-            } => {
-                let block = Block::decode(bytes, uncompressed_capacity)?;
-                Box::new(block)
-            }
-        };
-        Ok(block)
-    }
-}
-
-impl Value for CachedBlock {
-    type Cursor = CachedBlockCursor;
-
-    fn serialized_len(&self) -> usize {
-        1 /* type */ + match self {
-            CachedBlock::Loaded { block } => block.raw().len(),
-            CachedBlock::Fetched { bytes, uncompressed_capacity: _ } => 8 + bytes.len(),
-        }
-    }
-
-    fn read(mut buf: &[u8]) -> CodingResult<Self> {
-        let v = buf.get_u8();
-        let res = match v {
-            0 => {
-                let data = Bytes::copy_from_slice(buf);
-                let block = Block::decode_from_raw(data);
-                let block = Box::new(block);
-                Self::Loaded { block }
-            }
-            1 => {
-                let uncompressed_capacity = buf.get_u64() as usize;
-                let data = Bytes::copy_from_slice(buf);
-                let block = Block::decode(data, uncompressed_capacity)?;
-                let block = Box::new(block);
-                Self::Loaded { block }
-            }
-            _ => unreachable!(),
-        };
-        Ok(res)
-    }
-
-    fn into_cursor(self) -> Self::Cursor {
-        CachedBlockCursor::new(self)
-    }
-}
-
-#[derive(Debug)]
-pub struct CachedBlockCursor {
-    inner: CachedBlock,
-    pos: usize,
-}
-
-impl CachedBlockCursor {
-    pub fn new(inner: CachedBlock) -> Self {
-        Self { inner, pos: 0 }
-    }
-}
-
-impl std::io::Read for CachedBlockCursor {
-    fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result<usize> {
-        let pos = self.pos;
-        match &self.inner {
-            CachedBlock::Loaded { block } => {
-                if self.pos < 1 {
-                    self.pos += copy([0], &mut buf);
-                }
-                self.pos += copy(&block.raw()[self.pos - 1..], &mut buf);
-            }
-            CachedBlock::Fetched {
-                bytes,
-                uncompressed_capacity,
-            } => {
-                if self.pos < 1 {
-                    self.pos += copy([1], &mut buf);
-                }
-                if self.pos < 9 {
-                    self.pos += copy(
-                        &uncompressed_capacity.to_be_bytes()[self.pos - 1..],
-                        &mut buf,
-                    );
-                }
-                self.pos += copy(&bytes[self.pos - 9..], &mut buf);
-            }
-        }
-        let n = self.pos - pos;
-        Ok(n)
-    }
-}
-
-impl Cursor for CachedBlockCursor {
-    type T = CachedBlock;
-
-    fn into_inner(self) -> Self::T {
-        self.inner
-    }
-}
-
-impl Value for Box<Block> {
-    type Cursor = BoxBlockCursor;
-
-    fn serialized_len(&self) -> usize {
-        self.raw().len()
-    }
-
-    fn read(buf: &[u8]) -> CodingResult<Self> {
-        let data = Bytes::copy_from_slice(buf);
-        let block = Block::decode_from_raw(data);
-        let block = Box::new(block);
-        Ok(block)
-    }
-
-    fn into_cursor(self) -> Self::Cursor {
-        BoxBlockCursor::new(self)
-    }
-}
-
-#[derive(Debug)]
-pub struct BoxBlockCursor {
-    inner: Box<Block>,
-    pos: usize,
-}
-
-impl BoxBlockCursor {
-    pub fn new(inner: Box<Block>) -> Self {
-        Self { inner, pos: 0 }
-    }
-}
-
-impl std::io::Read for BoxBlockCursor {
-    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        let pos = self.pos;
-        self.pos += copy(&self.inner.raw()[self.pos..], buf);
-        let n = self.pos - pos;
-        Ok(n)
-    }
-}
-
-impl Cursor for BoxBlockCursor {
-    type T = Box<Block>;
-
-    fn into_inner(self) -> Self::T {
-        self.inner
-    }
-}
-
-#[derive(Debug)]
-pub struct CachedSstable(Arc<Sstable>);
-
-impl Clone for CachedSstable {
-    fn clone(&self) -> Self {
-        Self(Arc::clone(&self.0))
-    }
-}
-
-impl From<Box<Sstable>> for CachedSstable {
-    fn from(value: Box<Sstable>) -> Self {
-        Self(Arc::new(*value))
-    }
-}
-
-impl From<CachedSstable> for Box<Sstable> {
-    fn from(value: CachedSstable) -> Self {
-        Box::new(Arc::unwrap_or_clone(value.0))
-    }
-}
-
-impl CachedSstable {
-    pub fn into_inner(self) -> Arc<Sstable> {
-        self.0
-    }
-}
-
-impl Value for CachedSstable {
-    type Cursor = CachedSstableCursor;
-
-    fn weight(&self) -> usize {
-        self.0.estimate_size()
-    }
-
-    fn serialized_len(&self) -> usize {
-        8 + self.0.meta.encoded_size() // id (8B) + meta size
-    }
-
-    fn read(mut buf: &[u8]) -> CodingResult<Self> {
-        let id = buf.get_u64();
-        let meta = SstableMeta::decode(buf).unwrap();
-        let sstable = Arc::new(Sstable::new(id, meta));
-        Ok(Self(sstable))
-    }
-
-    fn into_cursor(self) -> Self::Cursor {
-        CachedSstableCursor::new(self)
-    }
-}
-
-#[derive(Debug)]
-pub struct CachedSstableCursor {
-    inner: CachedSstable,
-    pos: usize,
-    /// store pre-encoded bytes here, for it's hard to encode JIT
-    bytes: Vec<u8>,
-}
-
-impl CachedSstableCursor {
-    pub fn new(inner: CachedSstable) -> Self {
-        let mut bytes = vec![];
-        bytes.put_u64(inner.0.id);
-        inner.0.meta.encode_to(&mut bytes);
-        Self {
-            inner,
-            bytes,
-            pos: 0,
-        }
-    }
-}
-
-impl std::io::Read for CachedSstableCursor {
-    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        let pos = self.pos;
-        self.pos += copy(&self.bytes[self.pos..], buf);
-        let n = self.pos - pos;
-        Ok(n)
-    }
-}
-
-impl Cursor for CachedSstableCursor {
-    type T = CachedSstable;
-
-    fn into_inner(self) -> Self::T {
-        self.inner
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use risingwave_common::catalog::TableId;
-    use risingwave_common::util::epoch::test_epoch;
-    use risingwave_hummock_sdk::key::FullKey;
-
-    use super::*;
-    use crate::hummock::{BlockBuilder, BlockBuilderOptions, BlockMeta, CompressionAlgorithm};
-
-    pub fn construct_full_key_struct_for_test(
-        table_id: u32,
-        table_key: &[u8],
-        epoch: u64,
-    ) -> FullKey<&[u8]> {
-        FullKey::for_test(TableId::new(table_id), table_key, epoch)
-    }
-
-    fn block_for_test() -> Box<Block> {
-        let options = BlockBuilderOptions {
-            compression_algorithm: CompressionAlgorithm::Lz4,
-            ..Default::default()
-        };
-
-        let mut builder = BlockBuilder::new(options);
-        builder.add_for_test(
-            construct_full_key_struct_for_test(0, b"k1", test_epoch(1)),
-            b"v01",
-        );
-        builder.add_for_test(
-            construct_full_key_struct_for_test(0, b"k2", test_epoch(2)),
-            b"v02",
-        );
-        builder.add_for_test(
-            construct_full_key_struct_for_test(0, b"k3", test_epoch(3)),
-            b"v03",
-        );
-        builder.add_for_test(
-            construct_full_key_struct_for_test(0, b"k4", test_epoch(4)),
-            b"v04",
-        );
-
-        let uncompress = builder.uncompressed_block_size();
-        Box::new(Block::decode(builder.build().to_vec().into(), uncompress).unwrap())
-    }
-
-    fn sstable_for_test() -> Sstable {
-        Sstable::new(
-            114514,
-            SstableMeta {
-                block_metas: vec![
-                    BlockMeta {
-                        smallest_key: b"0-smallest-key".to_vec(),
-                        len: 100,
-                        ..Default::default()
-                    },
-                    BlockMeta {
-                        smallest_key: b"5-some-key".to_vec(),
-                        offset: 100,
-                        len: 100,
-                        ..Default::default()
-                    },
-                ],
-                bloom_filter: b"0123456789012345".to_vec(),
-                estimated_size: 123,
-                key_count: 123,
-                smallest_key: b"0-smallest-key".to_vec(),
-                largest_key: b"9-largest-key".to_vec(),
-                meta_offset: 123,
-                monotonic_tombstone_events: vec![],
-                version: 2,
-            },
-        )
-    }
-
-    #[test]
-    fn test_cursor() {
-        {
-            let block = block_for_test();
-            let mut cursor = block.into_cursor();
-            let mut buf = vec![];
-            std::io::copy(&mut cursor, &mut buf).unwrap();
-            let target = cursor.into_inner();
-            let block = Box::<Block>::read(&buf[..]).unwrap();
-            assert_eq!(target.raw(), block.raw());
-        }
-
-        {
-            let sstable: CachedSstable = Box::new(sstable_for_test()).into();
-            let mut cursor = sstable.into_cursor();
-            let mut buf = vec![];
-            std::io::copy(&mut cursor, &mut buf).unwrap();
-            let target = cursor.into_inner();
-            let sstable = CachedSstable::read(&buf[..]).unwrap();
-            assert_eq!(target.0.id, sstable.0.id);
-            assert_eq!(target.0.meta, sstable.0.meta);
-        }
-
-        {
-            let cached = CachedBlock::Loaded {
-                block: block_for_test(),
-            };
-            let mut cursor = cached.into_cursor();
-            let mut buf = vec![];
-            std::io::copy(&mut cursor, &mut buf).unwrap();
-            let target = cursor.into_inner();
-            let cached = CachedBlock::read(&buf[..]).unwrap();
-            let target = match target {
-                CachedBlock::Loaded { block } => block,
-                CachedBlock::Fetched { .. } => panic!(),
-            };
-            let block = match cached {
-                CachedBlock::Loaded { block } => block,
-                CachedBlock::Fetched { .. } => panic!(),
-            };
-            assert_eq!(target.raw(), block.raw());
-        }
-
-        {
-            let index = SstableBlockIndex {
-                sst_id: 114,
-                block_idx: 514,
-            };
-            let mut cursor = index.into_cursor();
-            let mut buf = vec![];
-            std::io::copy(&mut cursor, &mut buf).unwrap();
-            let target = cursor.into_inner();
-            let index = SstableBlockIndex::read(&buf[..]).unwrap();
-            assert_eq!(target, index);
-        }
-    }
-}
diff --git a/src/storage/src/hummock/iterator/backward_concat.rs b/src/storage/src/hummock/iterator/backward_concat.rs
index 656d83e25b5eb..71b235aaa1c47 100644
--- a/src/storage/src/hummock/iterator/backward_concat.rs
+++ b/src/storage/src/hummock/iterator/backward_concat.rs
@@ -33,7 +33,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_backward_concat_iterator() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table0 = gen_iterator_test_sstable_info(
             0,
             default_builder_opt_for_test(),
@@ -92,7 +92,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_backward_concat_seek_exists() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table1 = gen_iterator_test_sstable_info(
             0,
             default_builder_opt_for_test(),
@@ -169,7 +169,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_backward_concat_seek_not_exists() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table0 = gen_iterator_test_sstable_info(
             0,
             default_builder_opt_for_test(),
diff --git a/src/storage/src/hummock/iterator/backward_merge.rs b/src/storage/src/hummock/iterator/backward_merge.rs
index f8464a777e078..02e47fe7ec641 100644
--- a/src/storage/src/hummock/iterator/backward_merge.rs
+++ b/src/storage/src/hummock/iterator/backward_merge.rs
@@ -23,7 +23,7 @@ mod test {
 
     #[tokio::test]
     async fn test_backward_merge_basic() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table0 = gen_iterator_test_sstable_base(
             0,
             default_builder_opt_for_test(),
@@ -76,7 +76,7 @@ mod test {
 
     #[tokio::test]
     async fn test_backward_merge_seek() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table0 = gen_iterator_test_sstable_base(
             0,
             default_builder_opt_for_test(),
@@ -151,7 +151,7 @@ mod test {
 
     #[tokio::test]
     async fn test_backward_merge_invalidate_reset() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table0 = gen_iterator_test_sstable_base(
             0,
             default_builder_opt_for_test(),
diff --git a/src/storage/src/hummock/iterator/backward_user.rs b/src/storage/src/hummock/iterator/backward_user.rs
index 39e6f7eafa256..ab348571a4128 100644
--- a/src/storage/src/hummock/iterator/backward_user.rs
+++ b/src/storage/src/hummock/iterator/backward_user.rs
@@ -15,6 +15,7 @@
 use std::ops::Bound::*;
 
 use bytes::Bytes;
+use more_asserts::debug_assert_le;
 use risingwave_hummock_sdk::key::{FullKey, UserKey, UserKeyRange};
 use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch};
 
@@ -62,8 +63,7 @@ pub struct BackwardUserIterator<I: HummockIterator<Direction = Backward>> {
 
 impl<I: HummockIterator<Direction = Backward>> BackwardUserIterator<I> {
     /// Creates [`BackwardUserIterator`] with given `read_epoch`.
-    #[cfg(test)]
-    pub(crate) fn with_epoch(
+    pub fn new(
         iterator: I,
         key_range: UserKeyRange,
         read_epoch: u64,
@@ -196,15 +196,15 @@ impl<I: HummockIterator<Direction = Backward>> BackwardUserIterator<I> {
     /// `rewind` or `seek` methods are called.
     ///
     /// Note: before call the function you need to ensure that the iterator is valid.
-    pub fn key(&self) -> &FullKey<Bytes> {
+    pub fn key(&self) -> FullKey<&[u8]> {
         assert!(self.is_valid());
-        &self.last_key
+        self.last_key.to_ref()
     }
 
     /// The returned value is in the form of user value.
     ///
-    /// Note: before calling the function you need to ensure that the iterator is valid.
-    pub fn value(&self) -> &Bytes {
+    /// Note: before call the function you need to ensure that the iterator is valid.
+    pub fn value(&self) -> &[u8] {
         assert!(self.is_valid());
         &self.last_val
     }
@@ -213,28 +213,34 @@ impl<I: HummockIterator<Direction = Backward>> BackwardUserIterator<I> {
     pub async fn rewind(&mut self) -> HummockResult<()> {
         // Handle range scan
         match &self.key_range.1 {
-            Included(end_key) => {
+            Included(end_key) | Excluded(end_key) => {
                 let full_key = FullKey {
-                    user_key: end_key.clone(),
+                    user_key: end_key.as_ref(),
                     epoch_with_gap: EpochWithGap::new_min_epoch(),
                 };
-                self.iterator.seek(full_key.to_ref()).await?;
+                self.iterator.seek(full_key).await?;
             }
-            Excluded(_) => unimplemented!("excluded begin key is not supported"),
             Unbounded => self.iterator.rewind().await?,
         };
 
         // Handle multi-version
         self.reset();
         // Handle range scan when key < begin_key
-        self.next().await
+        self.next().await?;
+        if let Excluded(end_key) = &self.key_range.1
+            && self.is_valid()
+            && self.key().user_key == end_key.as_ref()
+        {
+            self.next().await?;
+        }
+        Ok(())
     }
 
     /// Resets the iterating position to the first position where the key >= provided key.
     pub async fn seek(&mut self, user_key: UserKey<&[u8]>) -> HummockResult<()> {
         // Handle range scan when key > end_key
-        let user_key = match &self.key_range.1 {
-            Included(end_key) => {
+        let seek_key = match &self.key_range.1 {
+            Included(end_key) | Excluded(end_key) => {
                 let end_key = end_key.as_ref();
                 if end_key < user_key {
                     end_key
@@ -242,11 +248,10 @@ impl<I: HummockIterator<Direction = Backward>> BackwardUserIterator<I> {
                     user_key
                 }
             }
-            Excluded(_) => unimplemented!("excluded begin key is not supported"),
             Unbounded => user_key,
         };
         let full_key = FullKey {
-            user_key,
+            user_key: seek_key,
             epoch_with_gap: EpochWithGap::new_min_epoch(),
         };
         self.iterator.seek(full_key).await?;
@@ -254,7 +259,15 @@ impl<I: HummockIterator<Direction = Backward>> BackwardUserIterator<I> {
         // Handle multi-version
         self.reset();
         // Handle range scan when key < begin_key
-        self.next().await
+        self.next().await?;
+        if let Excluded(end_key) = &self.key_range.1
+            && self.is_valid()
+            && self.key().user_key == end_key.as_ref()
+        {
+            debug_assert_le!(end_key.as_ref(), user_key);
+            self.next().await?;
+        }
+        Ok(())
     }
 
     /// Indicates whether the iterator can be used.
@@ -277,7 +290,7 @@ impl<I: HummockIterator<Direction = Backward>> BackwardUserIterator<I> {
 impl<I: HummockIterator<Direction = Backward>> BackwardUserIterator<I> {
     /// Creates [`BackwardUserIterator`] with maximum epoch.
     pub(crate) fn for_test(iterator: I, key_range: UserKeyRange) -> Self {
-        Self::with_epoch(iterator, key_range, HummockEpoch::MAX, 0, None)
+        Self::new(iterator, key_range, HummockEpoch::MAX, 0, None)
     }
 
     /// Creates [`BackwardUserIterator`] with maximum epoch.
@@ -286,11 +299,10 @@ impl<I: HummockIterator<Direction = Backward>> BackwardUserIterator<I> {
         key_range: UserKeyRange,
         min_epoch: HummockEpoch,
     ) -> Self {
-        Self::with_epoch(iterator, key_range, HummockEpoch::MAX, min_epoch, None)
+        Self::new(iterator, key_range, HummockEpoch::MAX, min_epoch, None)
     }
 }
 
-#[expect(unused_variables)]
 #[cfg(test)]
 mod tests {
     use std::cmp::Reverse;
@@ -313,12 +325,11 @@ mod tests {
     };
     use crate::hummock::iterator::MergeIterator;
     use crate::hummock::test_utils::gen_test_sstable;
-    use crate::hummock::value::HummockValue;
     use crate::hummock::{BackwardSstableIterator, SstableStoreRef, TableHolder};
 
     #[tokio::test]
     async fn test_backward_user_basic() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table0 = gen_iterator_test_sstable_base(
             0,
             default_builder_opt_for_test(),
@@ -357,7 +368,7 @@ mod tests {
         while ui.is_valid() {
             let key = ui.key();
             let val = ui.value();
-            assert_eq!(key, &iterator_test_bytes_key_of(i));
+            assert_eq!(key, iterator_test_bytes_key_of(i).to_ref());
             assert_eq!(val, iterator_test_value_of(i).as_slice());
             i -= 1;
             ui.next().await.unwrap();
@@ -370,7 +381,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_backward_user_seek() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table0 = gen_iterator_test_sstable_base(
             0,
             default_builder_opt_for_test(),
@@ -417,7 +428,7 @@ mod tests {
         let k = bui.key();
         let v = bui.value();
         assert_eq!(v, iterator_test_value_of(TEST_KEYS_COUNT + 4).as_slice());
-        assert_eq!(k, &iterator_test_bytes_key_of(TEST_KEYS_COUNT + 4));
+        assert_eq!(k, iterator_test_bytes_key_of(TEST_KEYS_COUNT + 4).to_ref());
         bui.seek(iterator_test_user_key_of(2 * TEST_KEYS_COUNT + 5).as_ref())
             .await
             .unwrap();
@@ -427,7 +438,10 @@ mod tests {
             v,
             iterator_test_value_of(2 * TEST_KEYS_COUNT + 5).as_slice()
         );
-        assert_eq!(k, &iterator_test_bytes_key_of(2 * TEST_KEYS_COUNT + 5));
+        assert_eq!(
+            k,
+            iterator_test_bytes_key_of(2 * TEST_KEYS_COUNT + 5).to_ref()
+        );
 
         // left edge case
         bui.seek(iterator_test_user_key_of(3 * TEST_KEYS_COUNT).as_ref())
@@ -436,12 +450,12 @@ mod tests {
         let k = bui.key();
         let v = bui.value();
         assert_eq!(v, iterator_test_value_of(3 * TEST_KEYS_COUNT).as_slice());
-        assert_eq!(k, &iterator_test_bytes_key_of(3 * TEST_KEYS_COUNT));
+        assert_eq!(k, iterator_test_bytes_key_of(3 * TEST_KEYS_COUNT).to_ref());
     }
 
     #[tokio::test]
     async fn test_backward_user_delete() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         // key=[idx, epoch], value
         let kv_pairs = vec![
             (1, 300, HummockValue::delete()),
@@ -469,8 +483,8 @@ mod tests {
         let k = bui.key();
         let v = bui.value();
 
-        assert_eq!(k, &iterator_test_bytes_key_of_epoch(1, 400));
-        assert_eq!(v, &Bytes::from(iterator_test_value_of(1)));
+        assert_eq!(k, iterator_test_bytes_key_of_epoch(1, 400).to_ref());
+        assert_eq!(v, iterator_test_value_of(1).as_slice());
 
         // only one valid kv pair
         bui.next().await.unwrap();
@@ -480,7 +494,7 @@ mod tests {
     // left..=end
     #[tokio::test]
     async fn test_backward_user_range_inclusive() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         // key=[idx, epoch], value
         let kv_pairs = vec![
             (0, 200, HummockValue::delete()),
@@ -512,11 +526,11 @@ mod tests {
 
         // ----- basic iterate -----
         bui.rewind().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(7, 300));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(7, 300).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(6, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -524,11 +538,11 @@ mod tests {
         bui.seek(iterator_test_user_key_of(8).as_ref())
             .await
             .unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(7, 300));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(7, 300).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(6, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -536,11 +550,11 @@ mod tests {
         bui.seek(iterator_test_user_key_of(7).as_ref())
             .await
             .unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(7, 300));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(7, 300).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(6, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -560,7 +574,7 @@ mod tests {
     // left..end
     #[tokio::test]
     async fn test_backward_user_range() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         // key=[idx, epoch], value
         let kv_pairs = vec![
             (0, 200, HummockValue::delete()),
@@ -579,7 +593,10 @@ mod tests {
         ];
         let sstable =
             gen_iterator_test_sstable_from_kv_pair(0, kv_pairs, sstable_store.clone()).await;
-        let backward_iters = vec![BackwardSstableIterator::new(sstable, sstable_store)];
+        let backward_iters = vec![BackwardSstableIterator::new(
+            sstable.clone(),
+            sstable_store.clone(),
+        )];
         let bmi = MergeIterator::new(backward_iters);
 
         let begin_key = Excluded(iterator_test_bytes_user_key_of(2));
@@ -589,11 +606,11 @@ mod tests {
 
         // ----- basic iterate -----
         bui.rewind().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(7, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(7, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(6, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -601,11 +618,11 @@ mod tests {
         bui.seek(iterator_test_user_key_of(8).as_ref())
             .await
             .unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(7, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(7, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(6, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -613,11 +630,11 @@ mod tests {
         bui.seek(iterator_test_user_key_of(7).as_ref())
             .await
             .unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(7, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(7, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(6, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -632,12 +649,32 @@ mod tests {
             .await
             .unwrap();
         assert!(!bui.is_valid());
+
+        let backward_iters = vec![BackwardSstableIterator::new(sstable, sstable_store)];
+        let bmi = MergeIterator::new(backward_iters);
+
+        let begin_key = Excluded(iterator_test_bytes_user_key_of(2));
+        let end_key = Excluded(iterator_test_bytes_user_key_of(7));
+
+        let mut bui = BackwardUserIterator::for_test(bmi, (begin_key, end_key));
+        bui.rewind().await.unwrap();
+        assert!(bui.is_valid());
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
+        // ----- end-range iterate -----
+        bui.seek(iterator_test_user_key_of(7).as_ref())
+            .await
+            .unwrap();
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
+        bui.seek(iterator_test_user_key_of(5).as_ref())
+            .await
+            .unwrap();
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
     }
 
     // ..=right
     #[tokio::test]
     async fn test_backward_user_range_to_inclusive() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         // key=[idx, epoch], value
         let kv_pairs = vec![
             (0, 200, HummockValue::delete()),
@@ -657,21 +694,25 @@ mod tests {
         ];
         let sstable =
             gen_iterator_test_sstable_from_kv_pair(0, kv_pairs, sstable_store.clone()).await;
-        let backward_iters = vec![BackwardSstableIterator::new(sstable, sstable_store)];
+        let backward_iters = vec![BackwardSstableIterator::new(
+            sstable.clone(),
+            sstable_store.clone(),
+        )];
         let bmi = MergeIterator::new(backward_iters);
         let end_key = Included(iterator_test_bytes_user_key_of(7));
 
-        let mut bui = BackwardUserIterator::for_test(bmi, (Unbounded, end_key));
+        let mut bui = BackwardUserIterator::for_test(
+            bmi,
+            (Included(iterator_test_bytes_user_key_of(2)), end_key),
+        );
 
         // ----- basic iterate -----
         bui.rewind().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(6, 100));
-        bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(2, 300));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(1, 200));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(2, 300).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -679,13 +720,11 @@ mod tests {
         bui.seek(iterator_test_user_key_of(7).as_ref())
             .await
             .unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(6, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(2, 300));
-        bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(1, 200));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(2, 300).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -693,13 +732,11 @@ mod tests {
         bui.seek(iterator_test_user_key_of(6).as_ref())
             .await
             .unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(6, 100));
-        bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(2, 300));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(1, 200));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(2, 300).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -708,12 +745,33 @@ mod tests {
             .await
             .unwrap();
         assert!(!bui.is_valid());
+
+        let end_key = Excluded(iterator_test_bytes_user_key_of(6));
+        let backward_iters = vec![BackwardSstableIterator::new(sstable, sstable_store)];
+        let bmi = MergeIterator::new(backward_iters);
+        let mut bui = BackwardUserIterator::for_test(
+            bmi,
+            (Excluded(iterator_test_bytes_user_key_of(2)), end_key),
+        );
+        // ----- basic iterate -----
+        bui.seek(iterator_test_user_key_of(6).as_ref())
+            .await
+            .unwrap();
+        assert!(bui.is_valid());
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
+        bui.next().await.unwrap();
+        assert!(!bui.is_valid());
+        bui.seek(iterator_test_user_key_of(7).as_ref())
+            .await
+            .unwrap();
+        assert!(bui.is_valid());
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
     }
 
     // left..
     #[tokio::test]
     async fn test_backward_user_range_from() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         // key=[idx, epoch], value
         let kv_pairs = vec![
             (0, 200, HummockValue::delete()),
@@ -741,13 +799,13 @@ mod tests {
 
         // ----- basic iterate -----
         bui.rewind().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(8, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(8, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(6, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(2, 300));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(2, 300).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -755,7 +813,7 @@ mod tests {
         bui.seek(iterator_test_user_key_of(2).as_ref())
             .await
             .unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(2, 300));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(2, 300).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -763,9 +821,9 @@ mod tests {
         bui.seek(iterator_test_user_key_of(5).as_ref())
             .await
             .unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(2, 300));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(2, 300).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -773,13 +831,13 @@ mod tests {
         bui.seek(iterator_test_user_key_of(8).as_ref())
             .await
             .unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(8, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(8, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(6, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(2, 300));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(2, 300).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
 
@@ -787,13 +845,13 @@ mod tests {
         bui.seek(iterator_test_user_key_of(9).as_ref())
             .await
             .unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(8, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(8, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(6, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(3, 100));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
         bui.next().await.unwrap();
-        assert_eq!(bui.key(), &iterator_test_bytes_key_of_epoch(2, 300));
+        assert_eq!(bui.key(), iterator_test_bytes_key_of_epoch(2, 300).to_ref());
         bui.next().await.unwrap();
         assert!(!bui.is_valid());
     }
@@ -851,12 +909,12 @@ mod tests {
             if *key > end_key || *key <= start_key {
                 continue;
             }
-            let (time, value) = value.first_key_value().unwrap();
+            let (_, value) = value.first_key_value().unwrap();
             if let HummockValue::Delete = value {
                 continue;
             }
             assert!(bui.is_valid(), "num_kvs:{}", num_kvs);
-            assert_eq!(&bui.key().user_key, key, "num_kvs:{}", num_kvs);
+            assert_eq!(bui.key().user_key, key.as_ref(), "num_kvs:{}", num_kvs);
             if let HummockValue::Put(bytes) = &value {
                 assert_eq!(bui.value(), bytes, "num_kvs:{}", num_kvs);
             }
@@ -909,7 +967,7 @@ mod tests {
                 prev_time = time.next_epoch();
             }
         }
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let sst = gen_test_sstable(
             default_builder_opt_for_test(),
             0,
@@ -931,14 +989,9 @@ mod tests {
 
     #[tokio::test]
     async fn test_backward_user_chaos_unbounded_unbounded() {
-        let (prev_key_number, sst, truth, sstable_store) = generate_chaos_test_data().await;
+        let (_prev_key_number, sst, truth, sstable_store) = generate_chaos_test_data().await;
         let repeat = 20;
         for _ in 0..repeat {
-            let mut rng = thread_rng();
-            let end_key: usize = rng.gen_range(2..=prev_key_number);
-            let end_key_bytes = key_from_num(end_key);
-            let begin_key: usize = rng.gen_range(1..=end_key);
-            let begin_key_bytes = key_from_num(begin_key);
             chaos_test_case(
                 sst.clone(),
                 Unbounded,
@@ -958,8 +1011,6 @@ mod tests {
             let mut rng = thread_rng();
             let end_key: usize = rng.gen_range(2..=prev_key_number);
             let end_key_bytes = key_from_num(end_key).into_bytes();
-            let begin_key: usize = rng.gen_range(1..=end_key);
-            let begin_key_bytes = key_from_num(begin_key);
             chaos_test_case(
                 sst.clone(),
                 Unbounded,
@@ -978,7 +1029,6 @@ mod tests {
         for _ in 0..repeat {
             let mut rng = thread_rng();
             let end_key: usize = rng.gen_range(2..=prev_key_number);
-            let end_key_bytes = key_from_num(end_key);
             let begin_key: usize = rng.gen_range(1..=end_key);
             let begin_key_bytes = key_from_num(begin_key).into_bytes();
             chaos_test_case(
@@ -999,7 +1049,6 @@ mod tests {
         for _ in 0..repeat {
             let mut rng = thread_rng();
             let end_key: usize = rng.gen_range(2..=prev_key_number);
-            let end_key_bytes = key_from_num(end_key);
             let begin_key: usize = rng.gen_range(1..=end_key);
             let begin_key_bytes = key_from_num(begin_key).into_bytes();
             chaos_test_case(
@@ -1057,7 +1106,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_min_epoch() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table0 = gen_iterator_test_sstable_with_incr_epoch(
             0,
             default_builder_opt_for_test(),
diff --git a/src/storage/src/hummock/iterator/change_log.rs b/src/storage/src/hummock/iterator/change_log.rs
index 8d6187b8cd474..7f4d17af6d740 100644
--- a/src/storage/src/hummock/iterator/change_log.rs
+++ b/src/storage/src/hummock/iterator/change_log.rs
@@ -56,10 +56,10 @@ struct ChangeLogIteratorInner<
     /// Indicate whether the current new value is delete.
     is_new_value_delete: bool,
 
-    /// Whether Indicate whether the current `old_value_iter` represents the old value in ChangeLogValue
+    /// Whether Indicate whether the current `old_value_iter` represents the old value in `ChangeLogValue`
     is_old_value_set: bool,
 
-    /// Whether the iterator is currently pointing at a valid key with ChangeLogValue
+    /// Whether the iterator is currently pointing at a valid key with `ChangeLogValue`
     is_current_pos_valid: bool,
 }
 
@@ -387,6 +387,123 @@ impl StateStoreIter<StateStoreReadLogItem> for ChangeLogIterator {
     }
 }
 
+#[cfg(any(test, feature = "test"))]
+pub mod test_utils {
+    use std::collections::HashMap;
+
+    use bytes::Bytes;
+    use rand::{thread_rng, Rng, RngCore};
+    use risingwave_common::util::epoch::{test_epoch, EpochPair, MAX_EPOCH};
+    use risingwave_hummock_sdk::key::TableKey;
+
+    use crate::hummock::iterator::test_utils::iterator_test_table_key_of;
+    use crate::mem_table::KeyOp;
+    use crate::store::{InitOptions, LocalStateStore, SealCurrentEpochOptions};
+
+    pub type TestLogDataType = Vec<(u64, Vec<(TableKey<Bytes>, KeyOp)>)>;
+
+    pub fn gen_test_data(
+        epoch_count: usize,
+        key_count: usize,
+        skip_ratio: f64,
+        delete_ratio: f64,
+    ) -> TestLogDataType {
+        let mut store: HashMap<TableKey<Bytes>, Bytes> = HashMap::new();
+        let mut rng = thread_rng();
+        let mut logs = Vec::new();
+        for epoch_idx in 1..=(epoch_count - 1) {
+            let mut epoch_logs = Vec::new();
+            let epoch = test_epoch(epoch_idx as _);
+            for key_idx in 0..key_count {
+                if rng.gen_bool(skip_ratio) {
+                    continue;
+                }
+                let key = TableKey(Bytes::from(iterator_test_table_key_of(key_idx)));
+                if rng.gen_bool(delete_ratio) {
+                    if let Some(prev_value) = store.remove(&key) {
+                        epoch_logs.push((key, KeyOp::Delete(prev_value)));
+                    }
+                } else {
+                    let value = Bytes::copy_from_slice(rng.next_u64().to_string().as_bytes());
+                    let prev_value = store.get(&key);
+                    if let Some(prev_value) = prev_value {
+                        epoch_logs.push((
+                            key.clone(),
+                            KeyOp::Update((prev_value.clone(), value.clone())),
+                        ));
+                    } else {
+                        epoch_logs.push((key.clone(), KeyOp::Insert(value.clone())));
+                    }
+                    store.insert(key, value);
+                }
+            }
+            logs.push((epoch, epoch_logs));
+        }
+        // at the end add an epoch with only delete
+        {
+            let mut epoch_logs = Vec::new();
+            let epoch = test_epoch(epoch_count as _);
+            for (key, value) in store {
+                epoch_logs.push((key, KeyOp::Delete(value)));
+            }
+            logs.push((epoch, epoch_logs));
+        }
+        logs
+    }
+
+    pub async fn apply_test_log_data(
+        log_data: TestLogDataType,
+        state_store: &mut impl LocalStateStore,
+        try_flush_ratio: f64,
+    ) {
+        let mut rng = thread_rng();
+        let first_epoch = log_data[0].0;
+        for (epoch, epoch_logs) in log_data {
+            if epoch == first_epoch {
+                state_store
+                    .init(InitOptions {
+                        epoch: EpochPair::new_test_epoch(epoch),
+                    })
+                    .await
+                    .unwrap();
+            } else {
+                state_store.flush().await.unwrap();
+                state_store.seal_current_epoch(
+                    epoch,
+                    SealCurrentEpochOptions {
+                        table_watermarks: None,
+                        switch_op_consistency_level: None,
+                    },
+                );
+            }
+            for (key, op) in epoch_logs {
+                match op {
+                    KeyOp::Insert(value) => {
+                        state_store.insert(key, value, None).unwrap();
+                    }
+                    KeyOp::Delete(old_value) => {
+                        state_store.delete(key, old_value).unwrap();
+                    }
+                    KeyOp::Update((old_value, value)) => {
+                        state_store.insert(key, value, Some(old_value)).unwrap();
+                    }
+                }
+                if rng.gen_bool(try_flush_ratio) {
+                    state_store.try_flush().await.unwrap();
+                }
+            }
+        }
+        state_store.flush().await.unwrap();
+        state_store.seal_current_epoch(
+            MAX_EPOCH,
+            SealCurrentEpochOptions {
+                table_watermarks: None,
+                switch_op_consistency_level: None,
+            },
+        );
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::collections::BTreeMap;
@@ -394,13 +511,15 @@ mod tests {
 
     use bytes::Bytes;
     use itertools::Itertools;
-    use rand::{thread_rng, Rng, RngCore};
     use risingwave_common::catalog::TableId;
     use risingwave_common::hash::table_distribution::TableDistribution;
-    use risingwave_common::util::epoch::{test_epoch, EpochPair};
+    use risingwave_common::util::epoch::test_epoch;
     use risingwave_hummock_sdk::key::{TableKey, UserKey};
     use risingwave_hummock_sdk::EpochWithGap;
 
+    use crate::hummock::iterator::change_log::test_utils::{
+        apply_test_log_data, gen_test_data, TestLogDataType,
+    };
     use crate::hummock::iterator::change_log::ChangeLogIteratorInner;
     use crate::hummock::iterator::test_utils::{
         iterator_test_table_key_of, iterator_test_value_of,
@@ -409,9 +528,8 @@ mod tests {
     use crate::mem_table::{KeyOp, MemTable, MemTableHummockIterator, MemTableStore};
     use crate::memory::MemoryStateStore;
     use crate::store::{
-        ChangeLogValue, InitOptions, LocalStateStore, NewLocalOptions, OpConsistencyLevel,
-        ReadLogOptions, ReadOptions, SealCurrentEpochOptions, StateStoreIter, StateStoreRead,
-        CHECK_BYTES_EQUAL,
+        ChangeLogValue, NewLocalOptions, OpConsistencyLevel, ReadLogOptions, StateStoreIter,
+        StateStoreRead, CHECK_BYTES_EQUAL,
     };
     use crate::StateStore;
 
@@ -533,131 +651,66 @@ mod tests {
         assert!(!iter.is_valid());
     }
 
-    async fn gen_test_data(
-        table_id: TableId,
-        epoch_count: usize,
-        key_count: usize,
-        delete_ratio: f64,
-    ) -> (Vec<(u64, MemTableStore, MemTableStore)>, MemoryStateStore) {
-        let state_store = MemoryStateStore::new();
-        let mut rng = thread_rng();
-        let mut local = state_store
-            .new_local(NewLocalOptions {
-                table_id,
-                op_consistency_level: OpConsistencyLevel::ConsistentOldValue(
-                    CHECK_BYTES_EQUAL.clone(),
-                ),
-                table_option: Default::default(),
-                is_replicated: false,
-                vnodes: TableDistribution::all_vnodes(),
-            })
-            .await;
+    fn gen_test_mem_table_store(
+        test_log_data: TestLogDataType,
+    ) -> Vec<(u64, MemTableStore, MemTableStore)> {
         let mut logs = Vec::new();
-        for epoch_idx in 1..=epoch_count {
-            let epoch = test_epoch(epoch_idx as _);
+        for (epoch, epoch_logs) in test_log_data {
             let mut new_values = MemTableStore::new();
             let mut old_values = MemTableStore::new();
-            if epoch_idx == 1 {
-                local
-                    .init(InitOptions {
-                        epoch: EpochPair::new_test_epoch(epoch),
-                    })
-                    .await
-                    .unwrap();
-            } else {
-                local.flush().await.unwrap();
-                local.seal_current_epoch(
-                    epoch,
-                    SealCurrentEpochOptions {
-                        table_watermarks: None,
-                        switch_op_consistency_level: None,
-                    },
-                );
-            }
-            for key_idx in 0..key_count {
-                let key = TableKey(Bytes::from(iterator_test_table_key_of(key_idx)));
-                if rng.gen_bool(delete_ratio) {
-                    if let Some(prev_value) = local
-                        .get(
-                            key.clone(),
-                            ReadOptions {
-                                prefix_hint: None,
-                                ignore_range_tombstone: false,
-                                prefetch_options: Default::default(),
-                                cache_policy: Default::default(),
-                                retention_seconds: None,
-                                table_id,
-                                read_version_from_backup: false,
-                            },
-                        )
-                        .await
-                        .unwrap()
-                    {
-                        new_values.insert(key.clone(), KeyOp::Delete(Bytes::new()));
-                        old_values.insert(key.clone(), KeyOp::Insert(prev_value.clone()));
-                        local.delete(key, prev_value).unwrap();
-                    }
-                } else {
-                    let value = Bytes::copy_from_slice(rng.next_u64().to_string().as_bytes());
-                    new_values.insert(key.clone(), KeyOp::Insert(value.clone()));
-                    let prev_value = local
-                        .get(
-                            key.clone(),
-                            ReadOptions {
-                                prefix_hint: None,
-                                ignore_range_tombstone: false,
-                                prefetch_options: Default::default(),
-                                cache_policy: Default::default(),
-                                retention_seconds: None,
-                                table_id,
-                                read_version_from_backup: false,
-                            },
-                        )
-                        .await
-                        .unwrap();
-                    if let Some(prev_value) = prev_value.clone() {
-                        old_values.insert(key.clone(), KeyOp::Insert(prev_value));
-                    }
-                    local.insert(key, value, prev_value).unwrap();
+            for (key, op) in epoch_logs {
+                new_values.insert(key.clone(), op.clone());
+                if let KeyOp::Delete(old_value) | KeyOp::Update((old_value, _)) = op {
+                    old_values.insert(key, KeyOp::Insert(old_value));
                 }
             }
             logs.push((epoch, new_values, old_values));
         }
-        local.flush().await.unwrap();
-        local.seal_current_epoch(
-            test_epoch((epoch_count + 1) as _),
-            SealCurrentEpochOptions {
-                table_watermarks: None,
-                switch_op_consistency_level: None,
-            },
-        );
-        (logs, state_store)
+        logs
     }
 
     #[tokio::test]
     async fn test_random_data() {
         let table_id = TableId::new(233);
         let epoch_count = 10;
-        let (logs, state_store) = gen_test_data(table_id, epoch_count, 10000, 0.2).await;
+        let state_store = MemoryStateStore::new();
+        let mut local = state_store
+            .new_local(NewLocalOptions {
+                table_id,
+                op_consistency_level: OpConsistencyLevel::ConsistentOldValue {
+                    check_old_value: CHECK_BYTES_EQUAL.clone(),
+                    is_log_store: true,
+                },
+                table_option: Default::default(),
+                is_replicated: false,
+                vnodes: TableDistribution::all_vnodes(),
+            })
+            .await;
+        let logs = gen_test_data(epoch_count, 10000, 0.05, 0.2);
         assert_eq!(logs.len(), epoch_count);
+        apply_test_log_data(logs.clone(), &mut local, 0.0).await;
+        let mem_table_logs = gen_test_mem_table_store(logs.clone());
+        assert_eq!(mem_table_logs.len(), epoch_count);
         for start_epoch_idx in 0..epoch_count {
-            for end_epoch_idx in start_epoch_idx + 1..epoch_count {
-                let new_value_iter =
-                    MergeIterator::new(logs.iter().map(|(epoch, new_value_memtable, _)| {
+            for end_epoch_idx in start_epoch_idx..epoch_count {
+                let new_value_iter = MergeIterator::new(mem_table_logs.iter().map(
+                    |(epoch, new_value_memtable, _)| {
                         MemTableHummockIterator::new(
                             new_value_memtable,
                             EpochWithGap::new_from_epoch(*epoch),
                             table_id,
                         )
-                    }));
-                let old_value_iter =
-                    MergeIterator::new(logs.iter().map(|(epoch, _, old_value_memtable)| {
+                    },
+                ));
+                let old_value_iter = MergeIterator::new(mem_table_logs.iter().map(
+                    |(epoch, _, old_value_memtable)| {
                         MemTableHummockIterator::new(
                             old_value_memtable,
                             EpochWithGap::new_from_epoch(*epoch),
                             table_id,
                         )
-                    }));
+                    },
+                ));
                 let epoch_range = (logs[start_epoch_idx].0, logs[end_epoch_idx].0);
                 let mut change_log_iter = ChangeLogIteratorInner::new(
                     epoch_range,
diff --git a/src/storage/src/hummock/iterator/forward_concat.rs b/src/storage/src/hummock/iterator/forward_concat.rs
index 425c181a77332..8837f0ae1ad5d 100644
--- a/src/storage/src/hummock/iterator/forward_concat.rs
+++ b/src/storage/src/hummock/iterator/forward_concat.rs
@@ -32,7 +32,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_concat_iterator() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table0 = gen_iterator_test_sstable_info(
             0,
             default_builder_opt_for_test(),
@@ -93,7 +93,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_concat_seek() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table0 = gen_iterator_test_sstable_info(
             0,
             default_builder_opt_for_test(),
@@ -168,7 +168,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_concat_seek_not_exists() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table0 = gen_iterator_test_sstable_info(
             0,
             default_builder_opt_for_test(),
diff --git a/src/storage/src/hummock/iterator/forward_merge.rs b/src/storage/src/hummock/iterator/forward_merge.rs
index 6e360d7f18c9c..60cb9a84f034b 100644
--- a/src/storage/src/hummock/iterator/forward_merge.rs
+++ b/src/storage/src/hummock/iterator/forward_merge.rs
@@ -110,7 +110,7 @@ mod test {
 
     #[tokio::test]
     async fn test_merge_invalidate_reset() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let read_options = Arc::new(SstableIteratorReadOptions::default());
         let table0 = gen_iterator_test_sstable_info(
             0,
diff --git a/src/storage/src/hummock/iterator/forward_user.rs b/src/storage/src/hummock/iterator/forward_user.rs
index 8a68c10d0b515..768aa01af686c 100644
--- a/src/storage/src/hummock/iterator/forward_user.rs
+++ b/src/storage/src/hummock/iterator/forward_user.rs
@@ -14,6 +14,7 @@
 
 use std::ops::Bound::*;
 
+use more_asserts::debug_assert_ge;
 use risingwave_common::must_match;
 use risingwave_common::util::epoch::MAX_SPILL_TIMES;
 use risingwave_hummock_sdk::key::{FullKey, FullKeyTracker, UserKey, UserKeyRange};
@@ -124,20 +125,26 @@ impl<I: HummockIterator<Direction = Forward>> UserIterator<I> {
 
         // Handle range scan
         match &self.key_range.0 {
-            Included(begin_key) => {
+            Included(begin_key) | Excluded(begin_key) => {
                 let full_key = FullKey {
-                    user_key: begin_key.clone(),
+                    user_key: begin_key.as_ref(),
                     epoch_with_gap: EpochWithGap::new(self.read_epoch, MAX_SPILL_TIMES),
                 };
-                self.iterator.seek(full_key.to_ref()).await?;
+                self.iterator.seek(full_key).await?;
             }
-            Excluded(_) => unimplemented!("excluded begin key is not supported"),
             Unbounded => {
                 self.iterator.rewind().await?;
             }
         };
 
-        self.try_advance_to_next_valid().await
+        self.try_advance_to_next_valid().await?;
+        if let Excluded(begin_key) = &self.key_range.0
+            && self.is_valid()
+            && self.key().user_key == begin_key.as_ref()
+        {
+            self.next().await?;
+        }
+        Ok(())
     }
 
     /// Resets the iterating position to the first position where the key >= provided key.
@@ -147,8 +154,8 @@ impl<I: HummockIterator<Direction = Forward>> UserIterator<I> {
         self.full_key_tracker = FullKeyTracker::new(FullKey::default());
 
         // Handle range scan when key < begin_key
-        let user_key = match &self.key_range.0 {
-            Included(begin_key) => {
+        let seek_key = match &self.key_range.0 {
+            Included(begin_key) | Excluded(begin_key) => {
                 let begin_key = begin_key.as_ref();
                 if begin_key > user_key {
                     begin_key
@@ -156,17 +163,24 @@ impl<I: HummockIterator<Direction = Forward>> UserIterator<I> {
                     user_key
                 }
             }
-            Excluded(_) => unimplemented!("excluded begin key is not supported"),
             Unbounded => user_key,
         };
 
         let full_key = FullKey {
-            user_key,
+            user_key: seek_key,
             epoch_with_gap: EpochWithGap::new(self.read_epoch, MAX_SPILL_TIMES),
         };
         self.iterator.seek(full_key).await?;
 
-        self.try_advance_to_next_valid().await
+        self.try_advance_to_next_valid().await?;
+        if let Excluded(begin_key) = &self.key_range.0
+            && self.is_valid()
+            && self.key().user_key == begin_key.as_ref()
+        {
+            debug_assert_ge!(begin_key.as_ref(), user_key);
+            self.next().await?;
+        }
+        Ok(())
     }
 
     /// Indicates whether the iterator can be used.
@@ -274,12 +288,11 @@ mod tests {
         SstableIterator, SstableIteratorReadOptions, SstableIteratorType,
     };
     use crate::hummock::sstable_store::SstableStoreRef;
-    use crate::hummock::value::HummockValue;
     use crate::hummock::TableHolder;
 
     #[tokio::test]
     async fn test_basic() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let read_options = Arc::new(SstableIteratorReadOptions::default());
         let table0 = gen_iterator_test_sstable_base(
             0,
@@ -333,7 +346,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_seek() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let table0 = gen_iterator_test_sstable_base(
             0,
             default_builder_opt_for_test(),
@@ -408,7 +421,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_delete() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
 
         // key=[idx, epoch], value
         let kv_pairs = vec![
@@ -475,7 +488,7 @@ mod tests {
     // left..=end
     #[tokio::test]
     async fn test_range_inclusive() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         // key=[idx, epoch], value
         let table = generate_test_data(sstable_store.clone()).await;
         let read_options = Arc::new(SstableIteratorReadOptions::default());
@@ -537,7 +550,7 @@ mod tests {
     // left..end
     #[tokio::test]
     async fn test_range() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         // key=[idx, epoch], value
         let kv_pairs = vec![
             (0, 200, HummockValue::delete()),
@@ -557,7 +570,11 @@ mod tests {
         let table =
             gen_iterator_test_sstable_from_kv_pair(0, kv_pairs, sstable_store.clone()).await;
         let read_options = Arc::new(SstableIteratorReadOptions::default());
-        let iters = vec![SstableIterator::create(table, sstable_store, read_options)];
+        let iters = vec![SstableIterator::create(
+            table.clone(),
+            sstable_store.clone(),
+            read_options.clone(),
+        )];
         let mi = MergeIterator::new(iters);
 
         let begin_key = Included(iterator_test_bytes_user_key_of(2));
@@ -610,12 +627,41 @@ mod tests {
             .await
             .unwrap();
         assert!(!ui.is_valid());
+
+        let iters = vec![SstableIterator::create(table, sstable_store, read_options)];
+        let mi = MergeIterator::new(iters);
+
+        let begin_key = Excluded(iterator_test_bytes_user_key_of(2));
+        let end_key = Excluded(iterator_test_bytes_user_key_of(7));
+
+        let mut ui = UserIterator::for_test(mi, (begin_key, end_key));
+        // ----- after-end-range iterate -----
+        ui.seek(iterator_test_bytes_user_key_of(1).as_ref())
+            .await
+            .unwrap();
+        assert!(ui.is_valid());
+        assert_eq!(ui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
+        ui.seek(iterator_test_bytes_user_key_of(2).as_ref())
+            .await
+            .unwrap();
+        assert!(ui.is_valid());
+        assert_eq!(ui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
+        ui.seek(iterator_test_bytes_user_key_of(3).as_ref())
+            .await
+            .unwrap();
+        assert!(ui.is_valid());
+        assert_eq!(ui.key(), iterator_test_bytes_key_of_epoch(3, 100).to_ref());
+        ui.seek(iterator_test_bytes_user_key_of(4).as_ref())
+            .await
+            .unwrap();
+        assert!(ui.is_valid());
+        assert_eq!(ui.key(), iterator_test_bytes_key_of_epoch(6, 100).to_ref());
     }
 
     // ..=right
     #[tokio::test]
     async fn test_range_to_inclusive() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         // key=[idx, epoch], value
 
         let table = generate_test_data(sstable_store.clone()).await;
@@ -680,7 +726,7 @@ mod tests {
     // left..
     #[tokio::test]
     async fn test_range_from() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         // key=[idx, epoch], value
         let table = generate_test_data(sstable_store.clone()).await;
         let read_options = Arc::new(SstableIteratorReadOptions::default());
@@ -747,7 +793,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_min_epoch() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let read_options = Arc::new(SstableIteratorReadOptions::default());
         let table0 = gen_iterator_test_sstable_with_incr_epoch(
             0,
diff --git a/src/storage/src/hummock/iterator/mod.rs b/src/storage/src/hummock/iterator/mod.rs
index 9cae83be3f895..8a14efc801062 100644
--- a/src/storage/src/hummock/iterator/mod.rs
+++ b/src/storage/src/hummock/iterator/mod.rs
@@ -15,10 +15,13 @@
 use std::future::Future;
 use std::marker::PhantomData;
 use std::ops::{Deref, DerefMut};
+use std::sync::Arc;
 
-use more_asserts::assert_gt;
+use more_asserts::{assert_gt, assert_lt};
 
-use super::{HummockResult, HummockValue};
+use super::{
+    HummockResult, HummockValue, SstableIteratorReadOptions, SstableIteratorType, SstableStoreRef,
+};
 
 mod forward_concat;
 pub use forward_concat::*;
@@ -41,19 +44,21 @@ use risingwave_hummock_sdk::EpochWithGap;
 
 use crate::hummock::iterator::HummockIteratorUnion::{First, Fourth, Second, Third};
 
-mod change_log;
-pub use change_log::*;
+pub mod change_log;
 mod concat_delete_range_iterator;
 mod delete_range_iterator;
 mod skip_watermark;
 #[cfg(any(test, feature = "test"))]
 pub mod test_utils;
+
 pub use delete_range_iterator::{
     DeleteRangeIterator, ForwardMergeRangeIterator, RangeIteratorTyped,
 };
 use risingwave_common::catalog::TableId;
+use risingwave_pb::hummock::SstableInfo;
 pub use skip_watermark::*;
 
+use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferBatch;
 use crate::monitor::StoreLocalStatistic;
 
 #[derive(Default)]
@@ -339,6 +344,7 @@ impl<'a, B: RustIteratorBuilder> Iterator for RustIteratorOfBuilder<'a, B> {
 
 pub trait RustIteratorBuilder: Send + Sync + 'static {
     type Iterable: Send + Sync;
+    type Direction: HummockIteratorDirection;
     type RewindIter<'a>: Iterator<Item = (TableKey<&'a [u8]>, HummockValue<&'a [u8]>)>
         + Send
         + Sync
@@ -373,10 +379,86 @@ impl<'a, B: RustIteratorBuilder> FromRustIterator<'a, B> {
             table_id,
         }
     }
+
+    async fn seek_inner<'b>(&'b mut self, key: FullKey<&'b [u8]>) -> HummockResult<()> {
+        match self.table_id.cmp(&key.user_key.table_id) {
+            std::cmp::Ordering::Less => {
+                self.iter = None;
+                return Ok(());
+            }
+            std::cmp::Ordering::Greater => {
+                return self.rewind().await;
+            }
+            _ => {}
+        }
+        let mut iter = B::seek(self.inner, key.user_key.table_key);
+        match iter.next() {
+            Some((first_key, first_value)) => {
+                if first_key.eq(&key.user_key.table_key) && self.epoch > key.epoch_with_gap {
+                    // The semantic of `seek_fn` will ensure that `first_key` >= table_key of `key`.
+                    // At the beginning we have checked that `self.table_id` >= table_id of `key`.
+                    match iter.next() {
+                        Some((next_key, next_value)) => {
+                            assert_gt!(next_key, first_key);
+                            self.iter =
+                                Some((RustIteratorOfBuilder::Seek(iter), next_key, next_value));
+                        }
+                        None => {
+                            self.iter = None;
+                        }
+                    }
+                } else {
+                    self.iter = Some((RustIteratorOfBuilder::Seek(iter), first_key, first_value));
+                }
+            }
+            None => {
+                self.iter = None;
+            }
+        }
+        Ok(())
+    }
+
+    async fn rev_seek_inner<'b>(&'b mut self, key: FullKey<&'b [u8]>) -> HummockResult<()> {
+        match self.table_id.cmp(&key.user_key.table_id) {
+            std::cmp::Ordering::Less => {
+                return self.rewind().await;
+            }
+            std::cmp::Ordering::Greater => {
+                self.iter = None;
+                return Ok(());
+            }
+            _ => {}
+        }
+        let mut iter = B::seek(self.inner, key.user_key.table_key);
+        match iter.next() {
+            Some((first_key, first_value)) => {
+                if first_key.eq(&key.user_key.table_key) && self.epoch < key.epoch_with_gap {
+                    // The semantic of `seek_fn` will ensure that `first_key` <= table_key of `key`.
+                    // At the beginning we have checked that `self.table_id` <= table_id of `key`.
+                    match iter.next() {
+                        Some((next_key, next_value)) => {
+                            assert_lt!(next_key, first_key);
+                            self.iter =
+                                Some((RustIteratorOfBuilder::Seek(iter), next_key, next_value));
+                        }
+                        None => {
+                            self.iter = None;
+                        }
+                    }
+                } else {
+                    self.iter = Some((RustIteratorOfBuilder::Seek(iter), first_key, first_value));
+                }
+            }
+            None => {
+                self.iter = None;
+            }
+        }
+        Ok(())
+    }
 }
 
 impl<'a, B: RustIteratorBuilder> HummockIterator for FromRustIterator<'a, B> {
-    type Direction = Forward;
+    type Direction = B::Direction;
 
     async fn next(&mut self) -> HummockResult<()> {
         let (iter, key, value) = self.iter.as_mut().expect("should be valid");
@@ -420,50 +502,10 @@ impl<'a, B: RustIteratorBuilder> HummockIterator for FromRustIterator<'a, B> {
     }
 
     async fn seek<'b>(&'b mut self, key: FullKey<&'b [u8]>) -> HummockResult<()> {
-        if self.table_id < key.user_key.table_id {
-            // returns None when the range of self.table_id must not include the given key
-            self.iter = None;
-            return Ok(());
-        }
-        if self.table_id > key.user_key.table_id {
-            return self.rewind().await;
-        }
-        let mut iter = B::seek(self.inner, key.user_key.table_key);
-        match iter.next() {
-            Some((first_key, first_value)) => {
-                let first_full_key = FullKey {
-                    epoch_with_gap: self.epoch,
-                    user_key: UserKey {
-                        table_id: self.table_id,
-                        table_key: first_key,
-                    },
-                };
-                if first_full_key < key {
-                    // The semantic of `seek_fn` will ensure that `first_key` >= table_key of `key`.
-                    // At the beginning we have checked that `self.table_id` >= table_id of `key`.
-                    // Therefore, when `first_full_key` < `key`, the only possibility is that
-                    // `first_key` == table_key of `key`, and `self.table_id` == table_id of `key`,
-                    // the `self.epoch` < epoch of `key`.
-                    assert_eq!(first_key, key.user_key.table_key);
-                    match iter.next() {
-                        Some((next_key, next_value)) => {
-                            assert_gt!(next_key, first_key);
-                            self.iter =
-                                Some((RustIteratorOfBuilder::Seek(iter), next_key, next_value));
-                        }
-                        None => {
-                            self.iter = None;
-                        }
-                    }
-                } else {
-                    self.iter = Some((RustIteratorOfBuilder::Seek(iter), first_key, first_value));
-                }
-            }
-            None => {
-                self.iter = None;
-            }
+        match Self::Direction::direction() {
+            DirectionEnum::Forward => self.seek_inner(key).await,
+            DirectionEnum::Backward => self.rev_seek_inner(key).await,
         }
-        Ok(())
     }
 
     fn collect_local_statistic(&self, _stats: &mut StoreLocalStatistic) {}
@@ -498,3 +540,17 @@ impl HummockIteratorDirection for Backward {
         DirectionEnum::Backward
     }
 }
+
+pub trait IteratorFactory {
+    type Direction: HummockIteratorDirection;
+    type SstableIteratorType: SstableIteratorType<Direction = Self::Direction>;
+    fn add_batch_iter(&mut self, batch: SharedBufferBatch);
+    fn add_staging_sst_iter(&mut self, sst: Self::SstableIteratorType);
+    fn add_overlapping_sst_iter(&mut self, iter: Self::SstableIteratorType);
+    fn add_concat_sst_iter(
+        &mut self,
+        tables: Vec<SstableInfo>,
+        sstable_store: SstableStoreRef,
+        read_options: Arc<SstableIteratorReadOptions>,
+    );
+}
diff --git a/src/storage/src/hummock/iterator/skip_watermark.rs b/src/storage/src/hummock/iterator/skip_watermark.rs
index b9c6609bd524b..7baba6b2977e7 100644
--- a/src/storage/src/hummock/iterator/skip_watermark.rs
+++ b/src/storage/src/hummock/iterator/skip_watermark.rs
@@ -20,6 +20,7 @@ use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
 use risingwave_hummock_sdk::key::FullKey;
+use risingwave_hummock_sdk::table_stats::{add_table_stats_map, TableStats, TableStatsMap};
 use risingwave_hummock_sdk::table_watermark::{ReadTableWatermark, WatermarkDirection};
 use risingwave_pb::hummock::PbTableWatermarks;
 
@@ -31,6 +32,12 @@ use crate::monitor::StoreLocalStatistic;
 pub struct SkipWatermarkIterator<I> {
     inner: I,
     state: SkipWatermarkState,
+    /// The stats of skipped key-value pairs for each table.
+    skipped_entry_table_stats: TableStatsMap,
+    /// The id of table currently undergoing processing.
+    last_table_id: Option<u32>,
+    /// The stats of table currently undergoing processing.
+    last_table_stats: TableStats,
 }
 
 impl<I: HummockIterator<Direction = Forward>> SkipWatermarkIterator<I> {
@@ -38,6 +45,9 @@ impl<I: HummockIterator<Direction = Forward>> SkipWatermarkIterator<I> {
         Self {
             inner,
             state: SkipWatermarkState::new(watermarks),
+            skipped_entry_table_stats: TableStatsMap::default(),
+            last_table_id: None,
+            last_table_stats: TableStats::default(),
         }
     }
 
@@ -48,6 +58,9 @@ impl<I: HummockIterator<Direction = Forward>> SkipWatermarkIterator<I> {
         Self {
             inner,
             state: SkipWatermarkState::from_safe_epoch_watermarks(safe_epoch_watermarks),
+            skipped_entry_table_stats: TableStatsMap::default(),
+            last_table_id: None,
+            last_table_stats: TableStats::default(),
         }
     }
 
@@ -55,6 +68,12 @@ impl<I: HummockIterator<Direction = Forward>> SkipWatermarkIterator<I> {
         self.state.reset_watermark();
     }
 
+    fn reset_skipped_entry_table_stats(&mut self) {
+        self.skipped_entry_table_stats = TableStatsMap::default();
+        self.last_table_id = None;
+        self.last_table_stats = TableStats::default();
+    }
+
     /// Advance the key until iterator invalid or the current key will not be filtered by the latest watermark.
     /// Calling this method should ensure that the first remaining watermark has been advanced to the current key.
     ///
@@ -66,10 +85,36 @@ impl<I: HummockIterator<Direction = Forward>> SkipWatermarkIterator<I> {
             if !self.state.should_delete(&self.inner.key()) {
                 break;
             }
+
+            if self.last_table_id.map_or(true, |last_table_id| {
+                last_table_id != self.inner.key().user_key.table_id.table_id
+            }) {
+                self.add_last_table_stats();
+                self.last_table_id = Some(self.inner.key().user_key.table_id.table_id);
+            }
+            self.last_table_stats.total_key_count -= 1;
+            self.last_table_stats.total_key_size -= self.inner.key().encoded_len() as i64;
+            self.last_table_stats.total_value_size -= self.inner.value().encoded_len() as i64;
+
             self.inner.next().await?;
         }
+        self.add_last_table_stats();
         Ok(())
     }
+
+    fn add_last_table_stats(&mut self) {
+        let Some(last_table_id) = self.last_table_id.take() else {
+            return;
+        };
+        let delta = std::mem::take(&mut self.last_table_stats);
+        let e = self
+            .skipped_entry_table_stats
+            .entry(last_table_id)
+            .or_default();
+        e.total_key_count += delta.total_key_count;
+        e.total_key_size += delta.total_key_size;
+        e.total_value_size += delta.total_value_size;
+    }
 }
 
 impl<I: HummockIterator<Direction = Forward>> HummockIterator for SkipWatermarkIterator<I> {
@@ -100,6 +145,7 @@ impl<I: HummockIterator<Direction = Forward>> HummockIterator for SkipWatermarkI
 
     async fn rewind(&mut self) -> HummockResult<()> {
         self.reset_watermark();
+        self.reset_skipped_entry_table_stats();
         self.inner.rewind().await?;
         self.advance_key_and_watermark().await?;
         Ok(())
@@ -107,12 +153,17 @@ impl<I: HummockIterator<Direction = Forward>> HummockIterator for SkipWatermarkI
 
     async fn seek<'a>(&'a mut self, key: FullKey<&'a [u8]>) -> HummockResult<()> {
         self.reset_watermark();
+        self.reset_skipped_entry_table_stats();
         self.inner.seek(key).await?;
         self.advance_key_and_watermark().await?;
         Ok(())
     }
 
     fn collect_local_statistic(&self, stats: &mut StoreLocalStatistic) {
+        add_table_stats_map(
+            &mut stats.skipped_by_watermark_table_stats,
+            &self.skipped_entry_table_stats,
+        );
         self.inner.collect_local_statistic(stats)
     }
 
@@ -303,8 +354,9 @@ mod tests {
     use risingwave_hummock_sdk::EpochWithGap;
 
     use crate::hummock::iterator::{HummockIterator, SkipWatermarkIterator};
-    use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferBatch;
-    use crate::hummock::value::HummockValue;
+    use crate::hummock::shared_buffer::shared_buffer_batch::{
+        SharedBufferBatch, SharedBufferValue,
+    };
 
     const EPOCH: u64 = test_epoch(1);
     const TABLE_ID: TableId = TableId::new(233);
@@ -349,7 +401,7 @@ mod tests {
     }
 
     fn build_batch(
-        pairs: impl Iterator<Item = (TableKey<Bytes>, HummockValue<Bytes>)>,
+        pairs: impl Iterator<Item = (TableKey<Bytes>, SharedBufferValue<Bytes>)>,
     ) -> Option<SharedBufferBatch> {
         let pairs: Vec<_> = pairs.collect();
         if pairs.is_empty() {
@@ -360,9 +412,9 @@ mod tests {
     }
 
     fn filter_with_watermarks(
-        iter: impl Iterator<Item = (TableKey<Bytes>, HummockValue<Bytes>)>,
+        iter: impl Iterator<Item = (TableKey<Bytes>, SharedBufferValue<Bytes>)>,
         table_watermarks: ReadTableWatermark,
-    ) -> impl Iterator<Item = (TableKey<Bytes>, HummockValue<Bytes>)> {
+    ) -> impl Iterator<Item = (TableKey<Bytes>, SharedBufferValue<Bytes>)> {
         iter.filter(move |(key, _)| {
             if let Some(watermark) = table_watermarks.vnode_watermarks.get(&key.vnode_part()) {
                 !table_watermarks
@@ -424,10 +476,10 @@ mod tests {
         assert_iter_eq(first, second, Some((*last_vnode, last_key_index + 1))).await;
     }
 
-    fn gen_key_value(vnode: usize, index: usize) -> (TableKey<Bytes>, HummockValue<Bytes>) {
+    fn gen_key_value(vnode: usize, index: usize) -> (TableKey<Bytes>, SharedBufferValue<Bytes>) {
         (
             gen_key_from_str(VirtualNode::from_index(vnode), &gen_inner_key(index)),
-            HummockValue::Put(Bytes::copy_from_slice(
+            SharedBufferValue::Insert(Bytes::copy_from_slice(
                 format!("{}-value-{}", vnode, index).as_bytes(),
             )),
         )
diff --git a/src/storage/src/hummock/iterator/test_utils.rs b/src/storage/src/hummock/iterator/test_utils.rs
index 0103280570f5e..d315b51ab05eb 100644
--- a/src/storage/src/hummock/iterator/test_utils.rs
+++ b/src/storage/src/hummock/iterator/test_utils.rs
@@ -12,13 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::iter::Iterator;
 use std::sync::Arc;
 
 use bytes::Bytes;
+use foyer::HybridCacheBuilder;
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
-use risingwave_common::config::{EvictionConfig, MetricLevel, ObjectStoreConfig};
+use risingwave_common::config::{MetricLevel, ObjectStoreConfig};
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::util::epoch::test_epoch;
 use risingwave_hummock_sdk::key::{prefix_slice_with_vnode, FullKey, TableKey, UserKey};
@@ -28,6 +28,7 @@ use risingwave_object_store::object::{
 };
 use risingwave_pb::hummock::SstableInfo;
 
+use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferValue;
 use crate::hummock::sstable::SstableIteratorReadOptions;
 use crate::hummock::sstable_store::SstableStore;
 pub use crate::hummock::test_utils::default_builder_opt_for_test;
@@ -35,8 +36,8 @@ use crate::hummock::test_utils::{
     gen_test_sstable, gen_test_sstable_info, gen_test_sstable_with_range_tombstone,
 };
 use crate::hummock::{
-    FileCache, HummockValue, SstableBuilderOptions, SstableIterator, SstableIteratorType,
-    SstableStoreConfig, SstableStoreRef, TableHolder,
+    HummockValue, SstableBuilderOptions, SstableIterator, SstableIteratorType, SstableStoreConfig,
+    SstableStoreRef, TableHolder,
 };
 use crate::monitor::{global_hummock_state_store_metrics, ObjectStoreMetrics};
 
@@ -54,33 +55,44 @@ macro_rules! assert_bytes_eq {
 
 pub const TEST_KEYS_COUNT: usize = 10;
 
-pub fn mock_sstable_store() -> SstableStoreRef {
+pub async fn mock_sstable_store() -> SstableStoreRef {
     mock_sstable_store_with_object_store(Arc::new(ObjectStoreImpl::InMem(
         InMemObjectStore::new().monitored(
             Arc::new(ObjectStoreMetrics::unused()),
-            ObjectStoreConfig::default(),
+            Arc::new(ObjectStoreConfig::default()),
         ),
     )))
+    .await
 }
 
-pub fn mock_sstable_store_with_object_store(store: ObjectStoreRef) -> SstableStoreRef {
+pub async fn mock_sstable_store_with_object_store(store: ObjectStoreRef) -> SstableStoreRef {
     let path = "test".to_string();
+    let meta_cache = HybridCacheBuilder::new()
+        .memory(64 << 20)
+        .with_shards(2)
+        .storage()
+        .build()
+        .await
+        .unwrap();
+    let block_cache = HybridCacheBuilder::new()
+        .memory(64 << 20)
+        .with_shards(2)
+        .storage()
+        .build()
+        .await
+        .unwrap();
     Arc::new(SstableStore::new(SstableStoreConfig {
         store,
         path,
-        block_cache_capacity: 64 << 20,
-        block_cache_shard_num: 2,
-        block_cache_eviction: EvictionConfig::for_test(),
-        meta_cache_capacity: 64 << 20,
-        meta_cache_shard_num: 2,
-        meta_cache_eviction: EvictionConfig::for_test(),
+
         prefetch_buffer_capacity: 64 << 20,
         max_prefetch_block_number: 16,
 
-        data_file_cache: FileCache::none(),
-        meta_file_cache: FileCache::none(),
         recent_filter: None,
         state_store_metrics: Arc::new(global_hummock_state_store_metrics(MetricLevel::Disabled)),
+
+        meta_cache,
+        block_cache,
     }))
 }
 
@@ -132,8 +144,8 @@ pub fn iterator_test_value_of(idx: usize) -> Vec<u8> {
 }
 
 pub fn transform_shared_buffer(
-    batches: Vec<(Vec<u8>, HummockValue<Bytes>)>,
-) -> Vec<(TableKey<Bytes>, HummockValue<Bytes>)> {
+    batches: Vec<(Vec<u8>, SharedBufferValue<Bytes>)>,
+) -> Vec<(TableKey<Bytes>, SharedBufferValue<Bytes>)> {
     batches
         .into_iter()
         .map(|(k, v)| (TableKey(k.into()), v))
@@ -226,7 +238,7 @@ pub async fn gen_merge_iterator_interleave_test_sstable_iters(
     key_count: usize,
     count: usize,
 ) -> Vec<SstableIterator> {
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     let mut result = vec![];
     for i in 0..count {
         let table = gen_iterator_test_sstable_base(
diff --git a/src/storage/src/hummock/local_version/pinned_version.rs b/src/storage/src/hummock/local_version/pinned_version.rs
index da9569e6bb83c..5e89a135d825c 100644
--- a/src/storage/src/hummock/local_version/pinned_version.rs
+++ b/src/storage/src/hummock/local_version/pinned_version.rs
@@ -85,7 +85,7 @@ impl PinnedVersion {
         pinned_version_manager_tx: UnboundedSender<PinVersionAction>,
     ) -> Self {
         let version_id = version.id;
-        let compaction_group_index = version.build_compaction_group_info();
+        let compaction_group_index = version.state_table_info.build_table_compaction_group_id();
 
         PinnedVersion {
             version: Arc::new(version),
@@ -109,7 +109,7 @@ impl PinnedVersion {
             self.version.id
         );
         let version_id = version.id;
-        let compaction_group_index = version.build_compaction_group_info();
+        let compaction_group_index = version.state_table_info.build_table_compaction_group_id();
 
         PinnedVersion {
             version: Arc::new(version),
@@ -135,9 +135,10 @@ impl PinnedVersion {
 
     pub fn levels(&self, table_id: TableId) -> impl Iterator<Item = &PbLevel> {
         #[auto_enum(Iterator)]
-        match self.compaction_group_index.get(&table_id) {
-            Some(compaction_group_id) => {
-                let levels = self.levels_by_compaction_groups_id(*compaction_group_id);
+        match self.version.state_table_info.info().get(&table_id) {
+            Some(info) => {
+                let compaction_group_id = info.compaction_group_id;
+                let levels = self.levels_by_compaction_groups_id(compaction_group_id);
                 levels
                     .l0
                     .as_ref()
@@ -155,10 +156,6 @@ impl PinnedVersion {
         self.version.max_committed_epoch
     }
 
-    pub fn safe_epoch(&self) -> u64 {
-        self.version.safe_epoch
-    }
-
     /// ret value can't be used as `HummockVersion`. it must be modified with delta
     pub fn version(&self) -> &HummockVersion {
         &self.version
diff --git a/src/storage/src/hummock/mod.rs b/src/storage/src/hummock/mod.rs
index 5cf2d831d325e..21eb4a13e8c31 100644
--- a/src/storage/src/hummock/mod.rs
+++ b/src/storage/src/hummock/mod.rs
@@ -25,9 +25,6 @@ use risingwave_pb::hummock::SstableInfo;
 pub mod block_cache;
 pub use block_cache::*;
 
-pub mod file_cache;
-pub use file_cache::*;
-
 pub mod sstable;
 pub use sstable::*;
 
@@ -53,6 +50,9 @@ mod validator;
 pub mod value;
 pub mod write_limiter;
 
+pub mod recent_filter;
+pub use recent_filter::*;
+
 pub mod block_stream;
 
 pub use error::*;
diff --git a/src/storage/src/hummock/observer_manager.rs b/src/storage/src/hummock/observer_manager.rs
index 4e10d9a523950..0725424aaca76 100644
--- a/src/storage/src/hummock/observer_manager.rs
+++ b/src/storage/src/hummock/observer_manager.rs
@@ -121,13 +121,13 @@ impl ObserverState for HummockObserverNode {
         );
         let _ = self
             .version_update_sender
-            .send(HummockVersionUpdate::PinnedVersion(
+            .send(HummockVersionUpdate::PinnedVersion(Box::new(
                 HummockVersion::from_rpc_protobuf(
                     &snapshot
                         .hummock_version
                         .expect("should get hummock version"),
                 ),
-            ))
+            )))
             .inspect_err(|e| {
                 tracing::error!(event = ?e.0, "unable to send full version");
             });
diff --git a/src/storage/src/hummock/file_cache/recent_filter.rs b/src/storage/src/hummock/recent_filter.rs
similarity index 100%
rename from src/storage/src/hummock/file_cache/recent_filter.rs
rename to src/storage/src/hummock/recent_filter.rs
diff --git a/src/storage/src/hummock/shared_buffer/shared_buffer_batch.rs b/src/storage/src/hummock/shared_buffer/shared_buffer_batch.rs
index 5857c5d2f8bd2..1f8b17fb6c662 100644
--- a/src/storage/src/hummock/shared_buffer/shared_buffer_batch.rs
+++ b/src/storage/src/hummock/shared_buffer/shared_buffer_batch.rs
@@ -16,6 +16,7 @@ use std::cmp::Ordering;
 use std::fmt::Debug;
 use std::future::Future;
 use std::marker::PhantomData;
+use std::mem::size_of_val;
 use std::ops::Bound::Included;
 use std::ops::{Bound, RangeBounds};
 use std::sync::atomic::AtomicU64;
@@ -23,12 +24,12 @@ use std::sync::atomic::Ordering::Relaxed;
 use std::sync::{Arc, LazyLock};
 
 use bytes::Bytes;
+use prometheus::IntGauge;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_hummock_sdk::key::{FullKey, PointRange, TableKey, TableKeyRange, UserKey};
 use risingwave_hummock_sdk::EpochWithGap;
 
-use crate::hummock::event_handler::LocalInstanceId;
 use crate::hummock::iterator::{
     Backward, DeleteRangeIterator, DirectionEnum, Forward, HummockIterator,
     HummockIteratorDirection, ValueMeta,
@@ -37,18 +38,56 @@ use crate::hummock::utils::{range_overlap, MemoryTracker};
 use crate::hummock::value::HummockValue;
 use crate::hummock::{HummockEpoch, HummockResult, MonotonicDeleteEvent};
 use crate::mem_table::ImmId;
-use crate::storage_value::StorageValue;
 use crate::store::ReadOptions;
 
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum SharedBufferValue<T> {
+    Insert(T),
+    Update(T),
+    Delete,
+}
+
+impl<T> SharedBufferValue<T> {
+    fn to_ref(&self) -> SharedBufferValue<&T> {
+        match self {
+            SharedBufferValue::Insert(val) => SharedBufferValue::Insert(val),
+            SharedBufferValue::Update(val) => SharedBufferValue::Update(val),
+            SharedBufferValue::Delete => SharedBufferValue::Delete,
+        }
+    }
+}
+
+impl<T> From<SharedBufferValue<T>> for HummockValue<T> {
+    fn from(val: SharedBufferValue<T>) -> HummockValue<T> {
+        match val {
+            SharedBufferValue::Insert(val) | SharedBufferValue::Update(val) => {
+                HummockValue::Put(val)
+            }
+            SharedBufferValue::Delete => HummockValue::Delete,
+        }
+    }
+}
+
+impl<'a, T: AsRef<[u8]>> SharedBufferValue<&'a T> {
+    pub(crate) fn to_slice(self) -> SharedBufferValue<&'a [u8]> {
+        match self {
+            SharedBufferValue::Insert(val) => SharedBufferValue::Insert(val.as_ref()),
+            SharedBufferValue::Update(val) => SharedBufferValue::Update(val.as_ref()),
+            SharedBufferValue::Delete => SharedBufferValue::Delete,
+        }
+    }
+}
+
 /// The key is `table_key`, which does not contain table id or epoch.
-pub(crate) type SharedBufferItem = (TableKey<Bytes>, HummockValue<Bytes>);
+pub(crate) type SharedBufferItem = (TableKey<Bytes>, SharedBufferValue<Bytes>);
 pub type SharedBufferBatchId = u64;
 
-pub(crate) type VersionedSharedBufferValue = (EpochWithGap, HummockValue<Bytes>);
+pub(crate) type VersionedSharedBufferValue = (EpochWithGap, SharedBufferValue<Bytes>);
 
 pub(crate) struct SharedBufferVersionedEntryRef<'a> {
     pub(crate) key: &'a TableKey<Bytes>,
     pub(crate) new_values: &'a [VersionedSharedBufferValue],
+    pub(crate) old_values: Option<&'a [Bytes]>,
 }
 
 #[derive(PartialEq, Debug)]
@@ -63,10 +102,10 @@ pub(crate) struct SharedBufferKeyEntry {
 
 impl SharedBufferKeyEntry {
     /// Return an exclusive offset of the values of key of index `i`
-    fn value_end_offset<'a>(
+    fn value_end_offset<'a, T>(
         i: usize,
         entries: &'a [SharedBufferKeyEntry],
-        values: &'a [VersionedSharedBufferValue],
+        values: &'a [T],
     ) -> usize {
         entries
             .get(i + 1)
@@ -74,19 +113,46 @@ impl SharedBufferKeyEntry {
             .unwrap_or(values.len())
     }
 
-    fn values<'a>(
-        i: usize,
-        entries: &'a [SharedBufferKeyEntry],
-        values: &'a [VersionedSharedBufferValue],
-    ) -> &'a [VersionedSharedBufferValue] {
+    fn values<'a, T>(i: usize, entries: &'a [SharedBufferKeyEntry], values: &'a [T]) -> &'a [T] {
         &values[entries[i].value_offset..Self::value_end_offset(i, entries, values)]
     }
 }
 
+#[derive(Debug)]
+pub(crate) struct SharedBufferBatchOldValues {
+    /// Store the old values. If some, the length should be the same as `new_values`. It contains empty `Bytes` when the
+    /// corresponding `new_value` is `Insert`, and contains the old values of `Update` and `Delete`.
+    values: Vec<Bytes>,
+    pub size: usize,
+    pub global_old_value_size: IntGauge,
+}
+
+impl Drop for SharedBufferBatchOldValues {
+    fn drop(&mut self) {
+        self.global_old_value_size.sub(self.size as _);
+    }
+}
+
+impl SharedBufferBatchOldValues {
+    pub(crate) fn new(values: Vec<Bytes>, size: usize, global_old_value_size: IntGauge) -> Self {
+        global_old_value_size.add(size as _);
+        Self {
+            values,
+            size,
+            global_old_value_size,
+        }
+    }
+
+    pub(crate) fn for_test(values: Vec<Bytes>, size: usize) -> Self {
+        Self::new(values, size, IntGauge::new("test", "test").unwrap())
+    }
+}
+
 #[derive(Debug)]
 pub(crate) struct SharedBufferBatchInner {
     entries: Vec<SharedBufferKeyEntry>,
     new_values: Vec<VersionedSharedBufferValue>,
+    old_values: Option<SharedBufferBatchOldValues>,
     /// The epochs of the data in batch, sorted in ascending order (old to new)
     epochs: Vec<HummockEpoch>,
     /// Total size of all key-value items (excluding the `epoch` of value versions)
@@ -102,11 +168,15 @@ impl SharedBufferBatchInner {
         epoch: HummockEpoch,
         spill_offset: u16,
         payload: Vec<SharedBufferItem>,
+        old_values: Option<SharedBufferBatchOldValues>,
         size: usize,
         _tracker: Option<MemoryTracker>,
     ) -> Self {
         assert!(!payload.is_empty());
         debug_assert!(payload.iter().is_sorted_by_key(|(key, _)| key));
+        if let Some(old_values) = &old_values {
+            assert_eq!(old_values.values.len(), payload.len());
+        }
 
         let epoch_with_gap = EpochWithGap::new(epoch, spill_offset);
         let mut entries = Vec::with_capacity(payload.len());
@@ -123,6 +193,7 @@ impl SharedBufferBatchInner {
         SharedBufferBatchInner {
             entries,
             new_values,
+            old_values,
             epochs: vec![epoch],
             size,
             _tracker,
@@ -139,6 +210,7 @@ impl SharedBufferBatchInner {
         epochs: Vec<HummockEpoch>,
         entries: Vec<SharedBufferKeyEntry>,
         new_values: Vec<VersionedSharedBufferValue>,
+        old_values: Option<SharedBufferBatchOldValues>,
         size: usize,
         imm_id: ImmId,
         tracker: Option<MemoryTracker>,
@@ -161,6 +233,7 @@ impl SharedBufferBatchInner {
         Self {
             entries,
             new_values,
+            old_values,
             epochs,
             size,
             _tracker: tracker,
@@ -188,7 +261,7 @@ impl SharedBufferBatchInner {
                 if read_epoch < e.pure_epoch() {
                     continue;
                 }
-                return Some((v.clone(), *e));
+                return Some((v.clone().into(), *e));
             }
             // cannot find a visible version
         }
@@ -207,11 +280,10 @@ pub static SHARED_BUFFER_BATCH_ID_GENERATOR: LazyLock<AtomicU64> =
     LazyLock::new(|| AtomicU64::new(0));
 
 /// A write batch stored in the shared buffer.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
 pub struct SharedBufferBatch {
     pub(crate) inner: Arc<SharedBufferBatchInner>,
     pub table_id: TableId,
-    pub instance_id: LocalInstanceId,
 }
 
 impl SharedBufferBatch {
@@ -220,18 +292,39 @@ impl SharedBufferBatch {
         epoch: HummockEpoch,
         table_id: TableId,
     ) -> Self {
-        let size = Self::measure_batch_size(&sorted_items);
+        Self::for_test_inner(sorted_items, None, epoch, table_id)
+    }
+
+    pub fn for_test_with_old_values(
+        sorted_items: Vec<SharedBufferItem>,
+        old_values: Vec<Bytes>,
+        epoch: HummockEpoch,
+        table_id: TableId,
+    ) -> Self {
+        Self::for_test_inner(sorted_items, Some(old_values), epoch, table_id)
+    }
+
+    fn for_test_inner(
+        sorted_items: Vec<SharedBufferItem>,
+        old_values: Option<Vec<Bytes>>,
+        epoch: HummockEpoch,
+        table_id: TableId,
+    ) -> Self {
+        let (size, old_value_size) = Self::measure_batch_size(&sorted_items, old_values.as_deref());
+
+        let old_values = old_values
+            .map(|old_values| SharedBufferBatchOldValues::for_test(old_values, old_value_size));
 
         Self {
             inner: Arc::new(SharedBufferBatchInner::new(
                 epoch,
                 0,
                 sorted_items,
+                old_values,
                 size,
                 None,
             )),
             table_id,
-            instance_id: SHARED_BUFFER_BATCH_ID_GENERATOR.fetch_add(1, Relaxed),
         }
     }
 
@@ -253,19 +346,30 @@ impl SharedBufferBatch {
             .sum()
     }
 
-    pub fn measure_batch_size(batch_items: &[SharedBufferItem]) -> usize {
+    /// Return (total size, old value size or 0)
+    pub fn measure_batch_size(
+        batch_items: &[SharedBufferItem],
+        old_values: Option<&[Bytes]>,
+    ) -> (usize, usize) {
+        let old_value_size = old_values
+            .iter()
+            .flat_map(|slice| slice.iter().map(|value| size_of_val(value) + value.len()))
+            .sum::<usize>();
         // size = Sum(length of full key + length of user value)
-        batch_items
+        let kv_size = batch_items
             .iter()
             .map(|(k, v)| {
                 k.len() + {
                     match v {
-                        HummockValue::Put(val) => val.len(),
-                        HummockValue::Delete => 0,
+                        SharedBufferValue::Insert(val) | SharedBufferValue::Update(val) => {
+                            val.len()
+                        }
+                        SharedBufferValue::Delete => 0,
                     }
                 }
             })
-            .sum()
+            .sum::<usize>();
+        (kv_size + old_value_size, old_value_size)
     }
 
     pub fn filter<R, B>(&self, table_id: TableId, table_key_range: &R) -> bool
@@ -309,6 +413,10 @@ impl SharedBufferBatch {
         self.inner.new_values.len()
     }
 
+    pub fn has_old_value(&self) -> bool {
+        self.inner.old_values.is_some()
+    }
+
     pub fn get(
         &self,
         table_key: TableKey<&[u8]>,
@@ -346,8 +454,14 @@ impl SharedBufferBatch {
             .is_ok()
     }
 
-    pub fn into_directed_iter<D: HummockIteratorDirection>(self) -> SharedBufferBatchIterator<D> {
-        SharedBufferBatchIterator::<D>::new(self.inner, self.table_id)
+    pub fn into_directed_iter<D: HummockIteratorDirection, const IS_NEW_VALUE: bool>(
+        self,
+    ) -> SharedBufferBatchIterator<D, IS_NEW_VALUE> {
+        SharedBufferBatchIterator::<D, IS_NEW_VALUE>::new(self.inner, self.table_id)
+    }
+
+    pub fn into_old_value_iter(self) -> SharedBufferBatchIterator<Forward, false> {
+        self.into_directed_iter()
     }
 
     pub fn into_forward_iter(self) -> SharedBufferBatchIterator<Forward> {
@@ -383,6 +497,10 @@ impl SharedBufferBatch {
         self.inner.size
     }
 
+    pub(crate) fn old_values(&self) -> Option<&SharedBufferBatchOldValues> {
+        self.inner.old_values.as_ref()
+    }
+
     pub fn batch_id(&self) -> SharedBufferBatchId {
         self.inner.batch_id
     }
@@ -391,29 +509,26 @@ impl SharedBufferBatch {
         &self.inner.epochs
     }
 
-    pub fn build_shared_buffer_item_batches(
-        kv_pairs: Vec<(TableKey<Bytes>, StorageValue)>,
-    ) -> Vec<SharedBufferItem> {
-        kv_pairs
-            .into_iter()
-            .map(|(key, value)| (key, value.into()))
-            .collect()
-    }
-
-    pub fn build_shared_buffer_batch(
+    pub(crate) fn build_shared_buffer_batch(
         epoch: HummockEpoch,
         spill_offset: u16,
         sorted_items: Vec<SharedBufferItem>,
+        old_values: Option<SharedBufferBatchOldValues>,
         size: usize,
         table_id: TableId,
-        instance_id: LocalInstanceId,
         tracker: Option<MemoryTracker>,
     ) -> Self {
-        let inner = SharedBufferBatchInner::new(epoch, spill_offset, sorted_items, size, tracker);
+        let inner = SharedBufferBatchInner::new(
+            epoch,
+            spill_offset,
+            sorted_items,
+            old_values,
+            size,
+            tracker,
+        );
         SharedBufferBatch {
             inner: Arc::new(inner),
             table_id,
-            instance_id,
         }
     }
 
@@ -456,18 +571,18 @@ impl SharedBufferBatch {
         size: usize,
         table_id: TableId,
     ) -> Self {
-        let inner = SharedBufferBatchInner::new(epoch, spill_offset, sorted_items, size, None);
+        let inner =
+            SharedBufferBatchInner::new(epoch, spill_offset, sorted_items, None, size, None);
         SharedBufferBatch {
             inner: Arc::new(inner),
             table_id,
-            instance_id: LocalInstanceId::default(),
         }
     }
 }
 
 /// Iterate all the items in the shared buffer batch
 /// If there are multiple versions of a key, the iterator will return all versions
-pub struct SharedBufferBatchIterator<D: HummockIteratorDirection> {
+pub struct SharedBufferBatchIterator<D: HummockIteratorDirection, const IS_NEW_VALUE: bool = true> {
     inner: Arc<SharedBufferBatchInner>,
     /// The index of the current entry in the payload
     current_entry_idx: usize,
@@ -479,8 +594,17 @@ pub struct SharedBufferBatchIterator<D: HummockIteratorDirection> {
     _phantom: PhantomData<D>,
 }
 
-impl<D: HummockIteratorDirection> SharedBufferBatchIterator<D> {
+impl<D: HummockIteratorDirection, const IS_NEW_VALUE: bool>
+    SharedBufferBatchIterator<D, IS_NEW_VALUE>
+{
     pub(crate) fn new(inner: Arc<SharedBufferBatchInner>, table_id: TableId) -> Self {
+        if !IS_NEW_VALUE {
+            assert!(
+                inner.old_values.is_some(),
+                "create old value iter with no old value: {:?}",
+                table_id
+            );
+        }
         Self {
             inner,
             current_entry_idx: 0,
@@ -533,6 +657,12 @@ impl<D: HummockIteratorDirection> SharedBufferBatchIterator<D> {
         );
         debug_assert_eq!(self.value_end_offset, self.get_value_end_offset());
         debug_assert!(self.current_value_idx < self.value_end_offset);
+        if !IS_NEW_VALUE {
+            debug_assert!(!matches!(
+                &self.inner.new_values[self.current_value_idx].1,
+                SharedBufferValue::Insert(_)
+            ));
+        }
     }
 
     fn advance_to_next_value(&mut self) {
@@ -547,6 +677,35 @@ impl<D: HummockIteratorDirection> SharedBufferBatchIterator<D> {
             }
         }
     }
+
+    fn advance_until_valid_old_value(&mut self) {
+        debug_assert!(!IS_NEW_VALUE);
+        if !self.is_valid_entry_idx() {
+            return;
+        }
+        loop {
+            while self.current_value_idx < self.value_end_offset
+                && matches!(
+                    &self.inner.new_values[self.current_value_idx].1,
+                    SharedBufferValue::Insert(_)
+                )
+            {
+                self.current_value_idx += 1;
+            }
+            if self.current_value_idx >= self.value_end_offset {
+                debug_assert_eq!(self.current_value_idx, self.value_end_offset);
+                self.advance_to_next_entry();
+                if self.is_valid_entry_idx() {
+                    self.reset_value_idx();
+                    continue;
+                } else {
+                    break;
+                }
+            } else {
+                break;
+            }
+        }
+    }
 }
 
 impl SharedBufferBatchIterator<Forward> {
@@ -566,15 +725,23 @@ impl SharedBufferBatchIterator<Forward> {
         SharedBufferVersionedEntryRef {
             key: &self.inner.entries[self.current_entry_idx].key,
             new_values: &self.inner.new_values[self.current_value_idx..self.value_end_offset],
+            old_values: self.inner.old_values.as_ref().map(|old_values| {
+                &old_values.values[self.current_value_idx..self.value_end_offset]
+            }),
         }
     }
 }
 
-impl<D: HummockIteratorDirection> HummockIterator for SharedBufferBatchIterator<D> {
+impl<D: HummockIteratorDirection, const IS_NEW_VALUE: bool> HummockIterator
+    for SharedBufferBatchIterator<D, IS_NEW_VALUE>
+{
     type Direction = D;
 
     async fn next(&mut self) -> HummockResult<()> {
         self.advance_to_next_value();
+        if !IS_NEW_VALUE {
+            self.advance_until_valid_old_value();
+        }
         Ok(())
     }
 
@@ -587,7 +754,17 @@ impl<D: HummockIteratorDirection> HummockIterator for SharedBufferBatchIterator<
 
     fn value(&self) -> HummockValue<&[u8]> {
         self.assert_valid_idx();
-        self.inner.new_values[self.current_value_idx].1.as_slice()
+        if IS_NEW_VALUE {
+            self.inner.new_values[self.current_value_idx]
+                .1
+                .to_ref()
+                .to_slice()
+                .into()
+        } else {
+            HummockValue::put(
+                self.inner.old_values.as_ref().unwrap().values[self.current_value_idx].as_ref(),
+            )
+        }
     }
 
     fn is_valid(&self) -> bool {
@@ -604,6 +781,9 @@ impl<D: HummockIteratorDirection> HummockIterator for SharedBufferBatchIterator<
             }
         };
         self.reset_value_idx();
+        if !IS_NEW_VALUE {
+            self.advance_until_valid_old_value();
+        }
         Ok(())
     }
 
@@ -651,6 +831,9 @@ impl<D: HummockIteratorDirection> HummockIterator for SharedBufferBatchIterator<
                 }
             },
         };
+        if !IS_NEW_VALUE {
+            self.advance_until_valid_old_value();
+        }
         Ok(())
     }
 
@@ -779,8 +962,9 @@ impl DeleteRangeIterator for SharedBufferDeleteRangeIterator {
 
 #[cfg(test)]
 mod tests {
-    use std::ops::Bound::{Excluded, Included};
+    use std::ops::Bound::Excluded;
 
+    use itertools::{zip_eq, Itertools};
     use risingwave_common::util::epoch::{test_epoch, EpochExt};
     use risingwave_hummock_sdk::key::map_table_key_range;
 
@@ -790,21 +974,27 @@ mod tests {
         iterator_test_key_of_epoch, iterator_test_table_key_of, transform_shared_buffer,
     };
 
+    fn to_hummock_value_batch(
+        items: Vec<(Vec<u8>, SharedBufferValue<Bytes>)>,
+    ) -> Vec<(Vec<u8>, HummockValue<Bytes>)> {
+        items.into_iter().map(|(k, v)| (k, v.into())).collect()
+    }
+
     #[tokio::test]
     async fn test_shared_buffer_batch_basic() {
         let epoch = test_epoch(1);
-        let shared_buffer_items: Vec<(Vec<u8>, HummockValue<Bytes>)> = vec![
+        let shared_buffer_items: Vec<(Vec<u8>, SharedBufferValue<Bytes>)> = vec![
             (
                 iterator_test_table_key_of(0),
-                HummockValue::put(Bytes::from("value1")),
+                SharedBufferValue::Insert(Bytes::from("value1")),
             ),
             (
                 iterator_test_table_key_of(1),
-                HummockValue::put(Bytes::from("value1")),
+                SharedBufferValue::Insert(Bytes::from("value1")),
             ),
             (
                 iterator_test_table_key_of(2),
-                HummockValue::put(Bytes::from("value1")),
+                SharedBufferValue::Insert(Bytes::from("value1")),
             ),
         ];
         let shared_buffer_batch = SharedBufferBatch::for_test(
@@ -812,6 +1002,7 @@ mod tests {
             epoch,
             Default::default(),
         );
+        let shared_buffer_items = to_hummock_value_batch(shared_buffer_items);
 
         // Sketch
         assert_eq!(
@@ -884,15 +1075,15 @@ mod tests {
         let shared_buffer_items = vec![
             (
                 iterator_test_table_key_of(1),
-                HummockValue::put(Bytes::from("value1")),
+                SharedBufferValue::Insert(Bytes::from("value1")),
             ),
             (
                 iterator_test_table_key_of(2),
-                HummockValue::put(Bytes::from("value2")),
+                SharedBufferValue::Insert(Bytes::from("value2")),
             ),
             (
                 iterator_test_table_key_of(3),
-                HummockValue::put(Bytes::from("value3")),
+                SharedBufferValue::Insert(Bytes::from("value3")),
             ),
         ];
         let shared_buffer_batch = SharedBufferBatch::for_test(
@@ -900,6 +1091,7 @@ mod tests {
             epoch,
             Default::default(),
         );
+        let shared_buffer_items = to_hummock_value_batch(shared_buffer_items);
 
         // FORWARD: Seek to a key < 1st key, expect all three items to return
         let mut iter = shared_buffer_batch.clone().into_forward_iter();
@@ -1018,6 +1210,178 @@ mod tests {
         assert!(!iter.is_valid());
     }
 
+    #[tokio::test]
+    async fn test_shared_buffer_batch_old_value_iter() {
+        let epoch = test_epoch(1);
+        let key_values = vec![
+            (
+                iterator_test_table_key_of(1),
+                SharedBufferValue::Insert(Bytes::from("value1")),
+            ),
+            (
+                iterator_test_table_key_of(2),
+                SharedBufferValue::Update(Bytes::from("value2")),
+            ),
+            (
+                iterator_test_table_key_of(3),
+                SharedBufferValue::Insert(Bytes::from("value3")),
+            ),
+            (iterator_test_table_key_of(4), SharedBufferValue::Delete),
+        ];
+        let old_values = vec![
+            Bytes::new(),
+            Bytes::from("old_value2"),
+            Bytes::new(),
+            Bytes::from("old_value4"),
+        ];
+        let shared_buffer_batch = SharedBufferBatch::for_test_with_old_values(
+            transform_shared_buffer(key_values.clone()),
+            old_values.clone(),
+            epoch,
+            Default::default(),
+        );
+        let shared_buffer_items = to_hummock_value_batch(key_values.clone());
+        let expected_old_value_iter_items = zip_eq(&key_values, &old_values)
+            .filter(|((_, new_value), _)| !matches!(new_value, SharedBufferValue::Insert(_)))
+            .map(|((key, _), old_value)| (key.clone(), HummockValue::Put(old_value)))
+            .collect_vec();
+
+        let mut iter = shared_buffer_batch.clone().into_old_value_iter();
+        iter.rewind().await.unwrap();
+        for item in &expected_old_value_iter_items {
+            assert!(iter.is_valid());
+            assert_eq!(*iter.key().user_key.table_key, item.0);
+            assert_eq!(iter.value(), item.1.as_slice());
+            iter.next().await.unwrap();
+        }
+        assert!(!iter.is_valid());
+
+        // FORWARD: Seek to a key < 1st key, expect all three items to return
+        let mut iter = shared_buffer_batch.clone().into_forward_iter();
+        iter.seek(iterator_test_key_of_epoch(0, epoch).to_ref())
+            .await
+            .unwrap();
+        for item in &shared_buffer_items {
+            assert!(iter.is_valid());
+            assert_eq!(*iter.key().user_key.table_key, item.0);
+            assert_eq!(iter.value(), item.1.as_slice());
+            iter.next().await.unwrap();
+        }
+        assert!(!iter.is_valid());
+
+        let mut iter = shared_buffer_batch.clone().into_old_value_iter();
+        iter.seek(iterator_test_key_of_epoch(0, epoch).to_ref())
+            .await
+            .unwrap();
+        for item in &expected_old_value_iter_items {
+            assert!(iter.is_valid());
+            assert_eq!(*iter.key().user_key.table_key, item.0);
+            assert_eq!(iter.value(), item.1.as_slice());
+            iter.next().await.unwrap();
+        }
+        assert!(!iter.is_valid());
+
+        // FORWARD: Seek to a key > the last key, expect no items to return
+        let mut iter = shared_buffer_batch.clone().into_forward_iter();
+        iter.seek(iterator_test_key_of_epoch(5, epoch).to_ref())
+            .await
+            .unwrap();
+        assert!(!iter.is_valid());
+
+        let mut iter = shared_buffer_batch.clone().into_old_value_iter();
+        iter.seek(iterator_test_key_of_epoch(5, epoch).to_ref())
+            .await
+            .unwrap();
+        assert!(!iter.is_valid());
+
+        // FORWARD: Seek to 2nd key with current epoch, expect last two items to return
+        let mut iter = shared_buffer_batch.clone().into_forward_iter();
+        iter.seek(iterator_test_key_of_epoch(2, epoch).to_ref())
+            .await
+            .unwrap();
+        for item in &shared_buffer_items[1..] {
+            assert!(iter.is_valid());
+            assert_eq!(*iter.key().user_key.table_key, item.0);
+            assert_eq!(iter.value(), item.1.as_slice());
+            iter.next().await.unwrap();
+        }
+        assert!(!iter.is_valid());
+
+        let mut iter = shared_buffer_batch.clone().into_old_value_iter();
+        iter.seek(iterator_test_key_of_epoch(2, epoch).to_ref())
+            .await
+            .unwrap();
+        for item in &expected_old_value_iter_items {
+            assert!(iter.is_valid());
+            assert_eq!(*iter.key().user_key.table_key, item.0);
+            assert_eq!(iter.value(), item.1.as_slice());
+            iter.next().await.unwrap();
+        }
+        assert!(!iter.is_valid());
+
+        // FORWARD: Seek to 2nd key with future epoch, expect last two items to return
+        let mut iter = shared_buffer_batch.clone().into_forward_iter();
+        iter.seek(iterator_test_key_of_epoch(2, epoch.next_epoch()).to_ref())
+            .await
+            .unwrap();
+        for item in &shared_buffer_items[1..] {
+            assert!(iter.is_valid());
+            assert_eq!(*iter.key().user_key.table_key, item.0.as_slice());
+            assert_eq!(iter.value(), item.1.as_slice());
+            iter.next().await.unwrap();
+        }
+        assert!(!iter.is_valid());
+
+        let mut iter = shared_buffer_batch.clone().into_old_value_iter();
+        iter.seek(iterator_test_key_of_epoch(2, epoch.next_epoch()).to_ref())
+            .await
+            .unwrap();
+        for item in &expected_old_value_iter_items {
+            assert!(iter.is_valid());
+            assert_eq!(*iter.key().user_key.table_key, item.0);
+            assert_eq!(iter.value(), item.1.as_slice());
+            iter.next().await.unwrap();
+        }
+        assert!(!iter.is_valid());
+
+        let mut iter = shared_buffer_batch.clone().into_old_value_iter();
+        iter.seek(iterator_test_key_of_epoch(2, epoch.prev_epoch()).to_ref())
+            .await
+            .unwrap();
+        for item in &expected_old_value_iter_items[1..] {
+            assert!(iter.is_valid());
+            assert_eq!(*iter.key().user_key.table_key, item.0);
+            assert_eq!(iter.value(), item.1.as_slice());
+            iter.next().await.unwrap();
+        }
+        assert!(!iter.is_valid());
+
+        // FORWARD: Seek to 2nd key with old epoch, expect last item to return
+        let mut iter = shared_buffer_batch.clone().into_forward_iter();
+        iter.seek(iterator_test_key_of_epoch(3, epoch.prev_epoch()).to_ref())
+            .await
+            .unwrap();
+        let item = shared_buffer_items.last().unwrap();
+        assert!(iter.is_valid());
+        assert_eq!(*iter.key().user_key.table_key, item.0.as_slice());
+        assert_eq!(iter.value(), item.1.as_slice());
+        iter.next().await.unwrap();
+        assert!(!iter.is_valid());
+
+        // Seek to an insert key
+        let mut iter = shared_buffer_batch.clone().into_old_value_iter();
+        iter.seek(iterator_test_key_of_epoch(3, epoch).to_ref())
+            .await
+            .unwrap();
+        for item in &expected_old_value_iter_items[1..] {
+            assert!(iter.is_valid());
+            assert_eq!(*iter.key().user_key.table_key, item.0);
+            assert_eq!(iter.value(), item.1.as_slice());
+            iter.next().await.unwrap();
+        }
+        assert!(!iter.is_valid());
+    }
+
     #[tokio::test]
     #[should_panic]
     async fn test_invalid_table_id() {
@@ -1034,10 +1398,22 @@ mod tests {
     async fn test_shared_buffer_batch_range_existx() {
         let epoch = test_epoch(1);
         let shared_buffer_items = vec![
-            (Vec::from("a_1"), HummockValue::put(Bytes::from("value1"))),
-            (Vec::from("a_3"), HummockValue::put(Bytes::from("value2"))),
-            (Vec::from("a_5"), HummockValue::put(Bytes::from("value3"))),
-            (Vec::from("b_2"), HummockValue::put(Bytes::from("value3"))),
+            (
+                Vec::from("a_1"),
+                SharedBufferValue::Insert(Bytes::from("value1")),
+            ),
+            (
+                Vec::from("a_3"),
+                SharedBufferValue::Insert(Bytes::from("value2")),
+            ),
+            (
+                Vec::from("a_5"),
+                SharedBufferValue::Insert(Bytes::from("value3")),
+            ),
+            (
+                Vec::from("b_2"),
+                SharedBufferValue::Insert(Bytes::from("value3")),
+            ),
         ];
         let shared_buffer_batch = SharedBufferBatch::for_test(
             transform_shared_buffer(shared_buffer_items),
@@ -1077,18 +1453,18 @@ mod tests {
     #[tokio::test]
     async fn test_merge_imms_basic() {
         let table_id = TableId { table_id: 1004 };
-        let shared_buffer_items1: Vec<(Vec<u8>, HummockValue<Bytes>)> = vec![
+        let shared_buffer_items1: Vec<(Vec<u8>, SharedBufferValue<Bytes>)> = vec![
             (
                 iterator_test_table_key_of(1),
-                HummockValue::put(Bytes::from("value1")),
+                SharedBufferValue::Insert(Bytes::from("value1")),
             ),
             (
                 iterator_test_table_key_of(2),
-                HummockValue::put(Bytes::from("value2")),
+                SharedBufferValue::Insert(Bytes::from("value2")),
             ),
             (
                 iterator_test_table_key_of(3),
-                HummockValue::put(Bytes::from("value3")),
+                SharedBufferValue::Insert(Bytes::from("value3")),
             ),
         ];
         let epoch = test_epoch(1);
@@ -1097,18 +1473,19 @@ mod tests {
             epoch,
             table_id,
         );
-        let shared_buffer_items2: Vec<(Vec<u8>, HummockValue<Bytes>)> = vec![
+        let shared_buffer_items1 = to_hummock_value_batch(shared_buffer_items1);
+        let shared_buffer_items2: Vec<(Vec<u8>, SharedBufferValue<Bytes>)> = vec![
             (
                 iterator_test_table_key_of(1),
-                HummockValue::put(Bytes::from("value12")),
+                SharedBufferValue::Insert(Bytes::from("value12")),
             ),
             (
                 iterator_test_table_key_of(2),
-                HummockValue::put(Bytes::from("value22")),
+                SharedBufferValue::Insert(Bytes::from("value22")),
             ),
             (
                 iterator_test_table_key_of(3),
-                HummockValue::put(Bytes::from("value32")),
+                SharedBufferValue::Insert(Bytes::from("value32")),
             ),
         ];
         let epoch = test_epoch(2);
@@ -1117,19 +1494,20 @@ mod tests {
             epoch,
             table_id,
         );
+        let shared_buffer_items2 = to_hummock_value_batch(shared_buffer_items2);
 
-        let shared_buffer_items3: Vec<(Vec<u8>, HummockValue<Bytes>)> = vec![
+        let shared_buffer_items3: Vec<(Vec<u8>, SharedBufferValue<Bytes>)> = vec![
             (
                 iterator_test_table_key_of(1),
-                HummockValue::put(Bytes::from("value13")),
+                SharedBufferValue::Insert(Bytes::from("value13")),
             ),
             (
                 iterator_test_table_key_of(2),
-                HummockValue::put(Bytes::from("value23")),
+                SharedBufferValue::Insert(Bytes::from("value23")),
             ),
             (
                 iterator_test_table_key_of(3),
-                HummockValue::put(Bytes::from("value33")),
+                SharedBufferValue::Insert(Bytes::from("value33")),
             ),
         ];
         let epoch = test_epoch(3);
@@ -1138,6 +1516,192 @@ mod tests {
             epoch,
             table_id,
         );
+        let shared_buffer_items3 = to_hummock_value_batch(shared_buffer_items3);
+
+        let batch_items = [
+            shared_buffer_items1,
+            shared_buffer_items2,
+            shared_buffer_items3,
+        ];
+        // newer data comes first
+        let imms = vec![imm3, imm2, imm1];
+        let merged_imm = merge_imms_in_memory(table_id, imms.clone(), None).await;
+
+        // Point lookup
+        for (i, items) in batch_items.iter().enumerate() {
+            for (key, value) in items {
+                assert_eq!(
+                    merged_imm
+                        .get(
+                            TableKey(key.as_slice()),
+                            test_epoch(i as u64 + 1),
+                            &ReadOptions::default()
+                        )
+                        .unwrap()
+                        .0,
+                    value.clone(),
+                    "epoch: {}, key: {:?}",
+                    test_epoch(i as u64 + 1),
+                    String::from_utf8(key.clone())
+                );
+            }
+        }
+        assert_eq!(
+            merged_imm.get(
+                TableKey(iterator_test_table_key_of(4).as_slice()),
+                test_epoch(1),
+                &ReadOptions::default()
+            ),
+            None
+        );
+        assert_eq!(
+            merged_imm.get(
+                TableKey(iterator_test_table_key_of(5).as_slice()),
+                test_epoch(1),
+                &ReadOptions::default()
+            ),
+            None
+        );
+
+        // Forward iterator
+        for snapshot_epoch in 1..=3 {
+            let mut iter = merged_imm.clone().into_forward_iter();
+            iter.rewind().await.unwrap();
+            let mut output = vec![];
+            while iter.is_valid() {
+                let epoch = iter.key().epoch_with_gap.pure_epoch();
+                if test_epoch(snapshot_epoch) == epoch {
+                    output.push((
+                        iter.key().user_key.table_key.to_vec(),
+                        iter.value().to_bytes(),
+                    ));
+                }
+                iter.next().await.unwrap();
+            }
+            assert_eq!(output, batch_items[snapshot_epoch as usize - 1]);
+        }
+
+        // Forward and Backward iterator
+        {
+            let mut iter = merged_imm.clone().into_forward_iter();
+            iter.rewind().await.unwrap();
+            let mut output = vec![];
+            while iter.is_valid() {
+                output.push((
+                    iter.key().user_key.table_key.to_vec(),
+                    iter.value().to_bytes(),
+                ));
+                iter.next().await.unwrap();
+            }
+
+            let mut expected = vec![];
+            for key_idx in 0..=2 {
+                for epoch in (1..=3).rev() {
+                    let item = batch_items[epoch - 1][key_idx].clone();
+                    expected.push(item);
+                }
+            }
+            assert_eq!(expected, output);
+
+            let mut backward_iter = merged_imm.clone().into_backward_iter();
+            backward_iter.rewind().await.unwrap();
+            let mut output = vec![];
+            while backward_iter.is_valid() {
+                output.push((
+                    backward_iter.key().user_key.table_key.to_vec(),
+                    backward_iter.value().to_bytes(),
+                ));
+                backward_iter.next().await.unwrap();
+            }
+            let mut expected = vec![];
+            for key_idx in (0..=2).rev() {
+                for epoch in (1..=3).rev() {
+                    let item = batch_items[epoch - 1][key_idx].clone();
+                    expected.push(item);
+                }
+            }
+            assert_eq!(expected, output);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_merge_imms_with_old_values() {
+        let table_id = TableId { table_id: 1004 };
+        let key_value1: Vec<(Vec<u8>, SharedBufferValue<Bytes>)> = vec![
+            (
+                iterator_test_table_key_of(1),
+                SharedBufferValue::Insert(Bytes::from("value1")),
+            ),
+            (
+                iterator_test_table_key_of(2),
+                SharedBufferValue::Update(Bytes::from("value2")),
+            ),
+            (iterator_test_table_key_of(3), SharedBufferValue::Delete),
+        ];
+        let old_value1 = vec![
+            Bytes::new(),
+            Bytes::from("old_value2"),
+            Bytes::from("old_value3"),
+        ];
+        let epoch = test_epoch(1);
+        let imm1 = SharedBufferBatch::for_test_with_old_values(
+            transform_shared_buffer(key_value1.clone()),
+            old_value1.clone(),
+            epoch,
+            table_id,
+        );
+        let shared_buffer_items1 = to_hummock_value_batch(key_value1.clone());
+        let key_value2: Vec<(Vec<u8>, SharedBufferValue<Bytes>)> = vec![
+            (
+                iterator_test_table_key_of(1),
+                SharedBufferValue::Update(Bytes::from("value12")),
+            ),
+            (
+                iterator_test_table_key_of(2),
+                SharedBufferValue::Update(Bytes::from("value22")),
+            ),
+            (
+                iterator_test_table_key_of(3),
+                SharedBufferValue::Insert(Bytes::from("value32")),
+            ),
+        ];
+        let old_value2 = vec![Bytes::from("value1"), Bytes::from("value2"), Bytes::new()];
+        let epoch = epoch.next_epoch();
+        let imm2 = SharedBufferBatch::for_test_with_old_values(
+            transform_shared_buffer(key_value2.clone()),
+            old_value2.clone(),
+            epoch,
+            table_id,
+        );
+        let shared_buffer_items2 = to_hummock_value_batch(key_value2.clone());
+
+        let key_value3: Vec<(Vec<u8>, SharedBufferValue<Bytes>)> = vec![
+            (iterator_test_table_key_of(1), SharedBufferValue::Delete),
+            (iterator_test_table_key_of(2), SharedBufferValue::Delete),
+            (
+                iterator_test_table_key_of(3),
+                SharedBufferValue::Update(Bytes::from("value33")),
+            ),
+        ];
+        let old_value3 = vec![
+            Bytes::from("value12"),
+            Bytes::from("value22"),
+            Bytes::from("value32"),
+        ];
+        let epoch = epoch.next_epoch();
+        let imm3 = SharedBufferBatch::for_test_with_old_values(
+            transform_shared_buffer(key_value3.clone()),
+            old_value3.clone(),
+            epoch,
+            table_id,
+        );
+        let shared_buffer_items3 = to_hummock_value_batch(key_value3.clone());
+
+        let key_values = [
+            (key_value1, old_value1),
+            (key_value2, old_value2),
+            (key_value3, old_value3),
+        ];
 
         let batch_items = [
             shared_buffer_items1,
@@ -1146,7 +1710,7 @@ mod tests {
         ];
         // newer data comes first
         let imms = vec![imm3, imm2, imm1];
-        let merged_imm = merge_imms_in_memory(table_id, 0, imms.clone(), None).await;
+        let merged_imm = merge_imms_in_memory(table_id, imms.clone(), None).await;
 
         // Point lookup
         for (i, items) in batch_items.iter().enumerate() {
@@ -1244,5 +1808,33 @@ mod tests {
             }
             assert_eq!(expected, output);
         }
+
+        // old value iter
+        {
+            let mut iter = merged_imm.clone().into_old_value_iter();
+            iter.rewind().await.unwrap();
+            let mut output = vec![];
+            while iter.is_valid() {
+                output.push((
+                    iter.key().user_key.table_key.to_vec(),
+                    iter.value().to_bytes(),
+                ));
+                iter.next().await.unwrap();
+            }
+
+            let mut expected = vec![];
+            for key_idx in 0..=2 {
+                for epoch in (0..=2).rev() {
+                    let (key_values, old_values) = &key_values[epoch];
+                    let (key, new_value) = &key_values[key_idx];
+                    let old_value = &old_values[key_idx];
+                    if matches!(new_value, SharedBufferValue::Insert(_)) {
+                        continue;
+                    }
+                    expected.push((key.clone(), HummockValue::Put(old_value.clone())));
+                }
+            }
+            assert_eq!(expected, output);
+        }
     }
 }
diff --git a/src/storage/src/hummock/sstable/backward_sstable_iterator.rs b/src/storage/src/hummock/sstable/backward_sstable_iterator.rs
index 58c2439771cff..d923750de7859 100644
--- a/src/storage/src/hummock/sstable/backward_sstable_iterator.rs
+++ b/src/storage/src/hummock/sstable/backward_sstable_iterator.rs
@@ -15,7 +15,7 @@
 use std::cmp::Ordering::{Equal, Less};
 use std::sync::Arc;
 
-use foyer::memory::CacheContext;
+use foyer::CacheContext;
 use risingwave_hummock_sdk::key::FullKey;
 
 use crate::hummock::iterator::{Backward, HummockIterator, ValueMeta};
@@ -187,7 +187,7 @@ mod tests {
     #[tokio::test]
     async fn test_backward_sstable_iterator() {
         // build remote sstable
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let handle =
             gen_default_test_sstable(default_builder_opt_for_test(), 0, sstable_store.clone())
                 .await;
@@ -212,7 +212,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_backward_sstable_seek() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let sstable =
             gen_default_test_sstable(default_builder_opt_for_test(), 0, sstable_store.clone())
                 .await;
diff --git a/src/storage/src/hummock/sstable/block.rs b/src/storage/src/hummock/sstable/block.rs
index 1028348a7d349..9ba69882db663 100644
--- a/src/storage/src/hummock/sstable/block.rs
+++ b/src/storage/src/hummock/sstable/block.rs
@@ -22,7 +22,7 @@ use bytes::{Buf, BufMut, Bytes, BytesMut};
 use risingwave_common::catalog::TableId;
 use risingwave_hummock_sdk::key::FullKey;
 use risingwave_hummock_sdk::KeyComparator;
-use {lz4, zstd};
+use serde::{Deserialize, Serialize};
 
 use super::utils::{bytes_diff_below_max_key_length, xxhash64_verify, CompressionAlgorithm};
 use crate::hummock::sstable::utils;
@@ -35,7 +35,7 @@ pub const DEFAULT_RESTART_INTERVAL: usize = 16;
 pub const DEFAULT_ENTRY_SIZE: usize = 24; // table_id(u64) + primary_key(u64) + epoch(u64)
 
 #[allow(non_camel_case_types)]
-#[derive(Clone, Copy, PartialEq, Debug)]
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub enum LenType {
     u8 = 1,
     u16 = 2,
@@ -125,7 +125,7 @@ impl LenType {
     }
 }
 
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub struct RestartPoint {
     pub offset: u32,
     pub key_len_type: LenType,
@@ -177,6 +177,26 @@ impl Debug for Block {
     }
 }
 
+impl Serialize for Block {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serde_bytes::serialize(&self.data[..], serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for Block {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let data: Vec<u8> = serde_bytes::deserialize(deserializer)?;
+        let data = Bytes::from(data);
+        Ok(Block::decode_from_raw(data))
+    }
+}
+
 impl Block {
     pub const HITMAP_ELEMS: usize = 4;
 
@@ -793,9 +813,9 @@ impl BlockBuilder {
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::catalog::TableId;
+
     use risingwave_common::util::epoch::test_epoch;
-    use risingwave_hummock_sdk::key::{FullKey, MAX_KEY_LEN};
+    use risingwave_hummock_sdk::key::MAX_KEY_LEN;
 
     use super::*;
     use crate::hummock::{BlockHolder, BlockIterator};
@@ -1003,4 +1023,35 @@ mod tests {
             builder.clear();
         }
     }
+
+    #[test]
+    fn test_block_serde() {
+        fn assmut_serde<'de, T: Serialize + Deserialize<'de>>() {}
+
+        assmut_serde::<Block>();
+        assmut_serde::<Box<Block>>();
+
+        let options = BlockBuilderOptions::default();
+        let mut builder = BlockBuilder::new(options);
+        for i in 0..100 {
+            builder.add_for_test(
+                construct_full_key_struct_for_test(0, format!("k{i:8}").as_bytes(), i),
+                format!("v{i:8}").as_bytes(),
+            );
+        }
+
+        let capacity = builder.uncompressed_block_size();
+        assert_eq!(capacity, builder.approximate_len() - 9);
+        let buf = builder.build().to_vec();
+
+        let block = Box::new(Block::decode(buf.into(), capacity).unwrap());
+
+        let buffer = bincode::serialize(&block).unwrap();
+        let blk: Block = bincode::deserialize(&buffer).unwrap();
+
+        assert_eq!(block.data, blk.data);
+        assert_eq!(block.data_len, blk.data_len);
+        assert_eq!(block.table_id, blk.table_id,);
+        assert_eq!(block.restart_points, blk.restart_points);
+    }
 }
diff --git a/src/storage/src/hummock/sstable/block_iterator.rs b/src/storage/src/hummock/sstable/block_iterator.rs
index 7344d033ddde4..daf25024846c4 100644
--- a/src/storage/src/hummock/sstable/block_iterator.rs
+++ b/src/storage/src/hummock/sstable/block_iterator.rs
@@ -330,11 +330,10 @@ impl BlockIterator {
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::catalog::TableId;
     use risingwave_common::util::epoch::test_epoch;
 
     use super::*;
-    use crate::hummock::{Block, BlockBuilder, BlockBuilderOptions};
+    use crate::hummock::{BlockBuilder, BlockBuilderOptions};
 
     fn build_iterator_for_test() -> BlockIterator {
         let options = BlockBuilderOptions::default();
diff --git a/src/storage/src/hummock/sstable/builder.rs b/src/storage/src/hummock/sstable/builder.rs
index f3f6c2345c70c..9b007f629f74a 100644
--- a/src/storage/src/hummock/sstable/builder.rs
+++ b/src/storage/src/hummock/sstable/builder.rs
@@ -626,9 +626,7 @@ pub(super) mod tests {
         default_builder_opt_for_test, gen_test_sstable_impl, mock_sst_writer, test_key_of,
         test_value_of, TEST_KEYS_COUNT,
     };
-    use crate::hummock::{
-        CachePolicy, Sstable, SstableWriterOptions, Xor16FilterBuilder, Xor8FilterBuilder,
-    };
+    use crate::hummock::{CachePolicy, Sstable, SstableWriterOptions, Xor8FilterBuilder};
     use crate::monitor::StoreLocalStatistic;
 
     #[tokio::test]
@@ -690,7 +688,7 @@ pub(super) mod tests {
         };
 
         // build remote table
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let sst_info = gen_test_sstable_impl::<Vec<u8>, F>(
             opts,
             0,
@@ -734,7 +732,7 @@ pub(super) mod tests {
     async fn test_no_bloom_filter_block() {
         let opts = SstableBuilderOptions::default();
         // build remote table
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let writer_opts = SstableWriterOptions::default();
         let object_id = 1;
         let writer = sstable_store
diff --git a/src/storage/src/hummock/sstable/delete_range_aggregator.rs b/src/storage/src/hummock/sstable/delete_range_aggregator.rs
index 58f4958919e04..a0c01d8cb80ea 100644
--- a/src/storage/src/hummock/sstable/delete_range_aggregator.rs
+++ b/src/storage/src/hummock/sstable/delete_range_aggregator.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::cmp::Ordering;
 use std::future::Future;
 
 #[cfg(test)]
@@ -25,33 +24,6 @@ use crate::hummock::iterator::{DeleteRangeIterator, ForwardMergeRangeIterator};
 use crate::hummock::sstable_store::TableHolder;
 use crate::hummock::{HummockResult, Sstable};
 
-pub struct SortedBoundary {
-    sequence: HummockEpoch,
-    user_key: UserKey<Vec<u8>>,
-}
-
-impl PartialEq<Self> for SortedBoundary {
-    fn eq(&self, other: &Self) -> bool {
-        self.user_key.eq(&other.user_key) && self.sequence == other.sequence
-    }
-}
-
-impl PartialOrd for SortedBoundary {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Eq for SortedBoundary {}
-
-impl Ord for SortedBoundary {
-    fn cmp(&self, other: &Self) -> Ordering {
-        self.user_key
-            .cmp(&other.user_key)
-            .then_with(|| other.sequence.cmp(&self.sequence))
-    }
-}
-
 pub struct CompactionDeleteRangeIterator {
     inner: ForwardMergeRangeIterator,
 }
diff --git a/src/storage/src/hummock/sstable/forward_sstable_iterator.rs b/src/storage/src/hummock/sstable/forward_sstable_iterator.rs
index bcfd7eb86f081..fe258db352143 100644
--- a/src/storage/src/hummock/sstable/forward_sstable_iterator.rs
+++ b/src/storage/src/hummock/sstable/forward_sstable_iterator.rs
@@ -316,7 +316,7 @@ mod tests {
     use std::collections::Bound;
 
     use bytes::Bytes;
-    use foyer::memory::CacheContext;
+    use foyer::CacheContext;
     use itertools::Itertools;
     use rand::prelude::*;
     use risingwave_common::catalog::TableId;
@@ -359,7 +359,7 @@ mod tests {
     #[tokio::test]
     async fn test_table_iterator() {
         // Build remote sstable
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let sstable =
             gen_default_test_sstable(default_builder_opt_for_test(), 0, sstable_store.clone())
                 .await;
@@ -372,7 +372,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_table_seek() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let sstable =
             gen_default_test_sstable(default_builder_opt_for_test(), 0, sstable_store.clone())
                 .await;
@@ -463,7 +463,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_prefetch_table_read() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         // when upload data is successful, but upload meta is fail and delete is fail
         let kv_iter =
             (0..TEST_KEYS_COUNT).map(|i| (test_key_of(i), HummockValue::put(test_value_of(i))));
diff --git a/src/storage/src/hummock/sstable/mod.rs b/src/storage/src/hummock/sstable/mod.rs
index adc1e5f4885f7..1125cc919bd58 100644
--- a/src/storage/src/hummock/sstable/mod.rs
+++ b/src/storage/src/hummock/sstable/mod.rs
@@ -27,6 +27,7 @@ pub use block_iterator::*;
 mod bloom;
 mod xor_filter;
 pub use bloom::BloomFilterBuilder;
+use serde::{Deserialize, Serialize};
 pub use xor_filter::{
     BlockedXor16FilterBuilder, Xor16FilterBuilder, Xor8FilterBuilder, XorFilterReader,
 };
@@ -147,7 +148,7 @@ impl DeleteRangeTombstone {
 /// next event key wmk2 (7) (not inclusive).
 /// If there is no range deletes between current event key and next event key, `new_epoch` will be
 /// `HummockEpoch::MAX`.
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct MonotonicDeleteEvent {
     pub event_key: PointRange<Vec<u8>>,
     pub new_epoch: HummockEpoch,
@@ -209,11 +210,25 @@ impl Display for MonotonicDeleteEvent {
     }
 }
 
+#[derive(Serialize, Deserialize)]
+struct SerdeSstable {
+    id: HummockSstableObjectId,
+    meta: SstableMeta,
+}
+
+impl From<SerdeSstable> for Sstable {
+    fn from(SerdeSstable { id, meta }: SerdeSstable) -> Self {
+        Sstable::new(id, meta)
+    }
+}
+
 /// [`Sstable`] is a handle for accessing SST.
-#[derive(Clone)]
+#[derive(Clone, Serialize, Deserialize)]
+#[serde(from = "SerdeSstable")]
 pub struct Sstable {
     pub id: HummockSstableObjectId,
     pub meta: SstableMeta,
+    #[serde(skip)]
     pub filter_reader: XorFilterReader,
 }
 
@@ -280,7 +295,7 @@ impl Sstable {
     }
 }
 
-#[derive(Clone, Default, Debug, Eq, PartialEq)]
+#[derive(Clone, Default, Debug, Eq, PartialEq, Serialize, Deserialize)]
 pub struct BlockMeta {
     pub smallest_key: Vec<u8>,
     pub offset: u32,
@@ -350,7 +365,7 @@ impl BlockMeta {
     }
 }
 
-#[derive(Default, Clone, PartialEq, Eq, Debug)]
+#[derive(Default, Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
 pub struct SstableMeta {
     pub block_metas: Vec<BlockMeta>,
     pub bloom_filter: Vec<u8>,
@@ -562,9 +577,14 @@ impl SstableIteratorReadOptions {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::hummock::iterator::test_utils::{
+        default_builder_opt_for_test, iterator_test_key_of,
+    };
+    use crate::hummock::test_utils::gen_test_sstable_data;
+    use crate::hummock::HummockValue;
 
     #[test]
-    pub fn test_sstable_meta_enc_dec() {
+    fn test_sstable_meta_enc_dec() {
         let meta = SstableMeta {
             block_metas: vec![
                 BlockMeta {
@@ -593,5 +613,29 @@ mod tests {
         assert_eq!(sz, buf.len());
         let decoded_meta = SstableMeta::decode(&buf[..]).unwrap();
         assert_eq!(decoded_meta, meta);
+
+        println!("buf: {}", buf.len());
+    }
+
+    #[tokio::test]
+    async fn test_sstable_serde() {
+        let (_, meta) = gen_test_sstable_data(
+            default_builder_opt_for_test(),
+            (0..100).clone().map(|x| {
+                (
+                    iterator_test_key_of(x),
+                    HummockValue::put(format!("overlapped_new_{}", x).as_bytes().to_vec()),
+                )
+            }),
+        )
+        .await;
+
+        let buffer = bincode::serialize(&meta).unwrap();
+
+        let m: SstableMeta = bincode::deserialize(&buffer).unwrap();
+
+        assert_eq!(meta, m);
+
+        println!("{} vs {}", buffer.len(), meta.encoded_size());
     }
 }
diff --git a/src/storage/src/hummock/sstable/multi_builder.rs b/src/storage/src/hummock/sstable/multi_builder.rs
index 4049811f78c9e..4e364ce9f94f5 100644
--- a/src/storage/src/hummock/sstable/multi_builder.rs
+++ b/src/storage/src/hummock/sstable/multi_builder.rs
@@ -18,6 +18,8 @@ use std::sync::atomic::Ordering::SeqCst;
 use std::sync::Arc;
 
 use bytes::Bytes;
+use futures::stream::FuturesUnordered;
+use futures::StreamExt;
 use num_integer::Integer;
 use risingwave_common::hash::VirtualNode;
 use risingwave_hummock_sdk::key::{FullKey, UserKey};
@@ -29,8 +31,8 @@ use crate::hummock::sstable::filter::FilterBuilder;
 use crate::hummock::sstable_store::SstableStoreRef;
 use crate::hummock::value::HummockValue;
 use crate::hummock::{
-    BatchUploadWriter, BlockMeta, CachePolicy, HummockResult, MemoryLimiter, SstableBuilder,
-    SstableBuilderOptions, SstableWriter, SstableWriterOptions, Xor16FilterBuilder,
+    BatchUploadWriter, BlockMeta, CachePolicy, HummockError, HummockResult, MemoryLimiter,
+    SstableBuilder, SstableBuilderOptions, SstableWriter, SstableWriterOptions, Xor16FilterBuilder,
 };
 use crate::monitor::CompactorMetrics;
 
@@ -43,11 +45,6 @@ pub trait TableBuilderFactory {
     async fn open_builder(&mut self) -> HummockResult<SstableBuilder<Self::Writer, Self::Filter>>;
 }
 
-pub struct SplitTableOutput {
-    pub sst_info: LocalSstableInfo,
-    pub upload_join_handle: UploadJoinHandle,
-}
-
 /// A wrapper for [`SstableBuilder`] which automatically split key-value pairs into multiple tables,
 /// based on their target capacity set in options.
 ///
@@ -59,7 +56,7 @@ where
     /// When creating a new [`SstableBuilder`], caller use this factory to generate it.
     builder_factory: F,
 
-    sst_outputs: Vec<SplitTableOutput>,
+    sst_outputs: Vec<LocalSstableInfo>,
 
     current_builder: Option<SstableBuilder<F::Writer, F::Filter>>,
 
@@ -75,6 +72,10 @@ where
     /// When vnode of the coming key is greater than `largest_vnode_in_current_partition`, we will
     /// switch SST.
     largest_vnode_in_current_partition: usize,
+
+    concurrent_upload_join_handle: FuturesUnordered<UploadJoinHandle>,
+
+    concurrent_uploading_sst_count: Option<usize>,
 }
 
 impl<F> CapacitySplitTableBuilder<F>
@@ -88,6 +89,7 @@ where
         compactor_metrics: Arc<CompactorMetrics>,
         task_progress: Option<Arc<TaskProgress>>,
         table_partition_vnode: BTreeMap<u32, u32>,
+        concurrent_uploading_sst_count: Option<usize>,
     ) -> Self {
         Self {
             builder_factory,
@@ -99,6 +101,8 @@ where
             table_partition_vnode,
             split_weight_by_vnode: 0,
             largest_vnode_in_current_partition: VirtualNode::MAX.to_index(),
+            concurrent_upload_join_handle: FuturesUnordered::new(),
+            concurrent_uploading_sst_count,
         }
     }
 
@@ -113,6 +117,8 @@ where
             table_partition_vnode: BTreeMap::default(),
             split_weight_by_vnode: 0,
             largest_vnode_in_current_partition: VirtualNode::MAX.to_index(),
+            concurrent_upload_join_handle: FuturesUnordered::new(),
+            concurrent_uploading_sst_count: None,
         }
     }
 
@@ -264,6 +270,7 @@ where
     /// If there's no builder created, or current one is already sealed before, then this function
     /// will be no-op.
     pub async fn seal_current(&mut self) -> HummockResult<()> {
+        use await_tree::InstrumentAwait;
         if let Some(builder) = self.current_builder.take() {
             let builder_output = builder.finish().await?;
             {
@@ -302,17 +309,33 @@ where
                         .observe(builder_output.epoch_count as _);
                 }
             }
-            self.sst_outputs.push(SplitTableOutput {
-                upload_join_handle: builder_output.writer_output,
-                sst_info: builder_output.sst_info,
-            });
+
+            self.concurrent_upload_join_handle
+                .push(builder_output.writer_output);
+
+            self.sst_outputs.push(builder_output.sst_info);
+
+            if let Some(concurrent_uploading_sst_count) = self.concurrent_uploading_sst_count
+                && self.concurrent_upload_join_handle.len() >= concurrent_uploading_sst_count
+            {
+                self.concurrent_upload_join_handle
+                    .next()
+                    .verbose_instrument_await("upload")
+                    .await
+                    .unwrap()
+                    .map_err(HummockError::sstable_upload_error)??;
+            }
         }
         Ok(())
     }
 
     /// Finalizes all the tables to be ids, blocks and metadata.
-    pub async fn finish(mut self) -> HummockResult<Vec<SplitTableOutput>> {
+    pub async fn finish(mut self) -> HummockResult<Vec<LocalSstableInfo>> {
+        use futures::future::try_join_all;
         self.seal_current().await?;
+        try_join_all(self.concurrent_upload_join_handle.into_iter())
+            .await
+            .map_err(HummockError::sstable_upload_error)?;
         Ok(self.sst_outputs)
     }
 }
@@ -370,13 +393,12 @@ impl TableBuilderFactory for LocalTableBuilderFactory {
 #[cfg(test)]
 mod tests {
     use risingwave_common::catalog::TableId;
-    use risingwave_common::hash::VirtualNode;
     use risingwave_common::util::epoch::{test_epoch, EpochExt};
 
     use super::*;
     use crate::hummock::iterator::test_utils::mock_sstable_store;
     use crate::hummock::test_utils::{default_builder_opt_for_test, test_key_of, test_user_key_of};
-    use crate::hummock::{SstableBuilderOptions, DEFAULT_RESTART_INTERVAL};
+    use crate::hummock::DEFAULT_RESTART_INTERVAL;
 
     #[tokio::test]
     async fn test_empty() {
@@ -389,7 +411,7 @@ mod tests {
             bloom_false_positive: 0.1,
             ..Default::default()
         };
-        let builder_factory = LocalTableBuilderFactory::new(1001, mock_sstable_store(), opts);
+        let builder_factory = LocalTableBuilderFactory::new(1001, mock_sstable_store().await, opts);
         let builder = CapacitySplitTableBuilder::for_test(builder_factory);
         let results = builder.finish().await.unwrap();
         assert!(results.is_empty());
@@ -406,7 +428,7 @@ mod tests {
             bloom_false_positive: 0.1,
             ..Default::default()
         };
-        let builder_factory = LocalTableBuilderFactory::new(1001, mock_sstable_store(), opts);
+        let builder_factory = LocalTableBuilderFactory::new(1001, mock_sstable_store().await, opts);
         let mut builder = CapacitySplitTableBuilder::for_test(builder_factory);
 
         for i in 0..table_capacity {
@@ -432,7 +454,7 @@ mod tests {
         let opts = default_builder_opt_for_test();
         let mut builder = CapacitySplitTableBuilder::for_test(LocalTableBuilderFactory::new(
             1001,
-            mock_sstable_store(),
+            mock_sstable_store().await,
             opts,
         ));
         let mut epoch = test_epoch(100);
@@ -476,7 +498,7 @@ mod tests {
         let opts = default_builder_opt_for_test();
         let mut builder = CapacitySplitTableBuilder::for_test(LocalTableBuilderFactory::new(
             1001,
-            mock_sstable_store(),
+            mock_sstable_store().await,
             opts,
         ));
         builder
@@ -501,10 +523,11 @@ mod tests {
             BTreeMap::from([(1_u32, 4_u32), (2_u32, 4_u32), (3_u32, 4_u32)]);
 
         let mut builder = CapacitySplitTableBuilder::new(
-            LocalTableBuilderFactory::new(1001, mock_sstable_store(), opts),
+            LocalTableBuilderFactory::new(1001, mock_sstable_store().await, opts),
             Arc::new(CompactorMetrics::unused()),
             None,
             table_partition_vnode,
+            None,
         );
 
         let mut table_key = VirtualNode::from_index(0).to_be_bytes().to_vec();
diff --git a/src/storage/src/hummock/sstable/xor_filter.rs b/src/storage/src/hummock/sstable/xor_filter.rs
index e096676008f89..1df4333fca459 100644
--- a/src/storage/src/hummock/sstable/xor_filter.rs
+++ b/src/storage/src/hummock/sstable/xor_filter.rs
@@ -442,7 +442,7 @@ impl Clone for XorFilterReader {
 
 #[cfg(test)]
 mod tests {
-    use foyer::memory::CacheContext;
+    use foyer::CacheContext;
     use rand::RngCore;
     use risingwave_common::util::epoch::test_epoch;
     use risingwave_hummock_sdk::EpochWithGap;
@@ -453,12 +453,12 @@ mod tests {
     use crate::hummock::sstable::{SstableBuilder, SstableBuilderOptions};
     use crate::hummock::test_utils::{test_user_key_of, test_value_of, TEST_KEYS_COUNT};
     use crate::hummock::value::HummockValue;
-    use crate::hummock::{BlockIterator, CachePolicy, Sstable, SstableWriterOptions};
+    use crate::hummock::{BlockIterator, CachePolicy, SstableWriterOptions};
     use crate::monitor::StoreLocalStatistic;
 
     #[tokio::test]
     async fn test_blocked_bloom_filter() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let writer_opts = SstableWriterOptions {
             capacity_hint: None,
             tracker: None,
diff --git a/src/storage/src/hummock/sstable_store.rs b/src/storage/src/hummock/sstable_store.rs
index 3d4a4d1e642a0..a6ab2a6162e0d 100644
--- a/src/storage/src/hummock/sstable_store.rs
+++ b/src/storage/src/hummock/sstable_store.rs
@@ -14,46 +14,71 @@
 use std::clone::Clone;
 use std::collections::VecDeque;
 use std::future::Future;
-use std::ops::Deref;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 
-use ahash::RandomState;
 use await_tree::InstrumentAwait;
 use bytes::Bytes;
 use fail::fail_point;
-use foyer::memory::{
-    Cache, CacheContext, CacheEntry, CacheEventListener, EntryState, Key, LfuCacheConfig,
-    LruCacheConfig, LruConfig, S3FifoCacheConfig, Value,
+use foyer::{
+    CacheContext, EventListener, FetchState, HybridCache, HybridCacheBuilder, HybridCacheEntry,
 };
 use futures::{future, StreamExt};
 use itertools::Itertools;
-use risingwave_common::config::{EvictionConfig, StorageMemoryConfig};
+use risingwave_common::config::StorageMemoryConfig;
 use risingwave_hummock_sdk::{HummockSstableObjectId, OBJECT_SUFFIX};
 use risingwave_hummock_trace::TracedCachePolicy;
 use risingwave_object_store::object::{
     ObjectError, ObjectMetadataIter, ObjectStoreRef, ObjectStreamingUploader,
 };
 use risingwave_pb::hummock::SstableInfo;
+use serde::{Deserialize, Serialize};
 use thiserror_ext::AsReport;
 use tokio::task::JoinHandle;
 use tokio::time::Instant;
 use zstd::zstd_safe::WriteBuf;
 
 use super::utils::MemoryTracker;
-use super::{
-    Block, BlockCache, BlockCacheConfig, BlockMeta, BlockResponse, CachedBlock, CachedSstable,
-    FileCache, RecentFilter, Sstable, SstableBlockIndex, SstableMeta, SstableWriter,
-};
+use super::{Block, BlockMeta, BlockResponse, RecentFilter, Sstable, SstableMeta, SstableWriter};
 use crate::hummock::block_stream::{
     BlockDataStream, BlockStream, MemoryUsageTracker, PrefetchBlockStream,
 };
-use crate::hummock::file_cache::preclude::*;
 use crate::hummock::multi_builder::UploadJoinHandle;
 use crate::hummock::{BlockHolder, HummockError, HummockResult, MemoryLimiter};
 use crate::monitor::{HummockStateStoreMetrics, MemoryCollector, StoreLocalStatistic};
 
-pub type TableHolder = CacheEntry<HummockSstableObjectId, Box<Sstable>, MetaCacheEventListener>;
+pub type TableHolder = HybridCacheEntry<HummockSstableObjectId, Box<Sstable>>;
+
+#[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Ord, Hash, Serialize, Deserialize)]
+pub struct SstableBlockIndex {
+    pub sst_id: HummockSstableObjectId,
+    pub block_idx: u64,
+}
+
+pub struct BlockCacheEventListener {
+    metrics: Arc<HummockStateStoreMetrics>,
+}
+
+impl BlockCacheEventListener {
+    pub fn new(metrics: Arc<HummockStateStoreMetrics>) -> Self {
+        Self { metrics }
+    }
+}
+
+impl EventListener for BlockCacheEventListener {
+    type Key = SstableBlockIndex;
+    type Value = Box<Block>;
+
+    fn on_memory_release(&self, _key: Self::Key, value: Self::Value)
+    where
+        Self::Key: foyer::Key,
+        Self::Value: foyer::Value,
+    {
+        self.metrics
+            .block_efficiency_histogram
+            .observe(value.efficiency());
+    }
+}
 
 // TODO: Define policy based on use cases (read / compaction / ...).
 #[derive(Clone, Copy, Eq, PartialEq)]
@@ -62,8 +87,6 @@ pub enum CachePolicy {
     Disable,
     /// Try reading the cache and fill the cache afterwards.
     Fill(CacheContext),
-    /// Fill file cache only.
-    FillFileCache,
     /// Read the cache but not fill the cache afterwards.
     NotFill,
 }
@@ -79,7 +102,6 @@ impl From<TracedCachePolicy> for CachePolicy {
         match policy {
             TracedCachePolicy::Disable => Self::Disable,
             TracedCachePolicy::Fill(priority) => Self::Fill(priority.into()),
-            TracedCachePolicy::FileFileCache => Self::FillFileCache,
             TracedCachePolicy::NotFill => Self::NotFill,
         }
     }
@@ -89,130 +111,32 @@ impl From<CachePolicy> for TracedCachePolicy {
     fn from(policy: CachePolicy) -> Self {
         match policy {
             CachePolicy::Disable => Self::Disable,
-            CachePolicy::FillFileCache => Self::FileFileCache,
             CachePolicy::Fill(priority) => Self::Fill(priority.into()),
             CachePolicy::NotFill => Self::NotFill,
         }
     }
 }
 
-#[derive(Debug)]
-pub struct BlockCacheEventListener {
-    data_file_cache: FileCache<SstableBlockIndex, CachedBlock>,
-    metrics: Arc<HummockStateStoreMetrics>,
-}
-
-impl BlockCacheEventListener {
-    pub fn new(
-        data_file_cache: FileCache<SstableBlockIndex, CachedBlock>,
-        metrics: Arc<HummockStateStoreMetrics>,
-    ) -> Self {
-        Self {
-            data_file_cache,
-            metrics,
-        }
-    }
-}
-
-impl CacheEventListener<(HummockSstableObjectId, u64), Box<Block>> for BlockCacheEventListener {
-    fn on_release(
-        &self,
-        key: (HummockSstableObjectId, u64),
-        value: Box<Block>,
-        _context: CacheContext,
-        _charges: usize,
-    ) {
-        let key = SstableBlockIndex {
-            sst_id: key.0,
-            block_idx: key.1,
-        };
-        self.metrics
-            .block_efficiency_histogram
-            .with_label_values(&[&value.table_id().to_string()])
-            .observe(value.efficiency());
-        // temporarily avoid spawn task while task drop with madsim
-        // FYI: https://github.com/madsim-rs/madsim/issues/182
-        #[cfg(not(madsim))]
-        self.data_file_cache
-            .insert_if_not_exists_async(key, CachedBlock::Loaded { block: value });
-    }
-}
-
-pub struct MetaCacheEventListener(FileCache<HummockSstableObjectId, CachedSstable>);
-
-impl From<FileCache<HummockSstableObjectId, CachedSstable>> for MetaCacheEventListener {
-    fn from(value: FileCache<HummockSstableObjectId, CachedSstable>) -> Self {
-        Self(value)
-    }
-}
-
-impl CacheEventListener<HummockSstableObjectId, Box<Sstable>> for MetaCacheEventListener {
-    fn on_release(
-        &self,
-        key: HummockSstableObjectId,
-        value: Box<Sstable>,
-        _context: CacheContext,
-        _charges: usize,
-    ) {
-        // temporarily avoid spawn task while task drop with madsim
-        // FYI: https://github.com/madsim-rs/madsim/issues/182
-        #[cfg(not(madsim))]
-        self.0.insert_if_not_exists_async(key, value.into());
-    }
-}
-
-pub enum CachedOrShared<K, V, L>
-where
-    K: Key,
-    V: Value,
-    L: CacheEventListener<K, Box<V>>,
-{
-    Cached(CacheEntry<K, Box<V>, L>),
-    Shared(Arc<V>),
-}
-
-impl<K, V, L> Deref for CachedOrShared<K, V, L>
-where
-    K: Key,
-    V: Value,
-    L: CacheEventListener<K, Box<V>>,
-{
-    type Target = V;
-
-    fn deref(&self) -> &Self::Target {
-        match self {
-            CachedOrShared::Cached(entry) => entry,
-            CachedOrShared::Shared(v) => v,
-        }
-    }
-}
-
 pub struct SstableStoreConfig {
     pub store: ObjectStoreRef,
     pub path: String,
-    pub block_cache_capacity: usize,
-    pub block_cache_shard_num: usize,
-    pub block_cache_eviction: EvictionConfig,
-    pub meta_cache_capacity: usize,
-    pub meta_cache_shard_num: usize,
-    pub meta_cache_eviction: EvictionConfig,
+
     pub prefetch_buffer_capacity: usize,
     pub max_prefetch_block_number: usize,
-    pub data_file_cache: FileCache<SstableBlockIndex, CachedBlock>,
-    pub meta_file_cache: FileCache<HummockSstableObjectId, CachedSstable>,
     pub recent_filter: Option<Arc<RecentFilter<(HummockSstableObjectId, usize)>>>,
     pub state_store_metrics: Arc<HummockStateStoreMetrics>,
+
+    pub meta_cache: HybridCache<HummockSstableObjectId, Box<Sstable>>,
+    pub block_cache: HybridCache<SstableBlockIndex, Box<Block>>,
 }
 
 pub struct SstableStore {
     path: String,
     store: ObjectStoreRef,
-    block_cache: BlockCache,
-    // TODO(MrCroxx): use no hash random state
-    meta_cache: Arc<Cache<HummockSstableObjectId, Box<Sstable>, MetaCacheEventListener>>,
 
-    data_file_cache: FileCache<SstableBlockIndex, CachedBlock>,
-    meta_file_cache: FileCache<HummockSstableObjectId, CachedSstable>,
+    meta_cache: HybridCache<HummockSstableObjectId, Box<Sstable>>,
+    block_cache: HybridCache<SstableBlockIndex, Box<Block>>,
+
     /// Recent filter for `(sst_obj_id, blk_idx)`.
     ///
     /// `blk_idx == USIZE::MAX` stands for `sst_obj_id` only entry.
@@ -227,60 +151,12 @@ impl SstableStore {
         // TODO: We should validate path early. Otherwise object store won't report invalid path
         // error until first write attempt.
 
-        let block_cache = BlockCache::new(BlockCacheConfig {
-            capacity: config.block_cache_capacity,
-            shard_num: config.block_cache_shard_num,
-            eviction: config.block_cache_eviction,
-            listener: BlockCacheEventListener::new(
-                config.data_file_cache.clone(),
-                config.state_store_metrics.clone(),
-            ),
-        });
-
-        // TODO(MrCroxx): reuse BlockCacheConfig here?
-        let meta_cache = {
-            let capacity = config.meta_cache_capacity;
-            let shards = config.meta_cache_shard_num;
-            let object_pool_capacity = config.meta_cache_shard_num * 1024;
-            let hash_builder = RandomState::default();
-            let event_listener = MetaCacheEventListener::from(config.meta_file_cache.clone());
-            match config.meta_cache_eviction {
-                EvictionConfig::Lru(eviction_config) => Cache::lru(LruCacheConfig {
-                    capacity,
-                    shards,
-                    eviction_config,
-                    object_pool_capacity,
-                    hash_builder,
-                    event_listener,
-                }),
-                EvictionConfig::Lfu(eviction_config) => Cache::lfu(LfuCacheConfig {
-                    capacity,
-                    shards,
-                    eviction_config,
-                    object_pool_capacity,
-                    hash_builder,
-                    event_listener,
-                }),
-                EvictionConfig::S3Fifo(eviction_config) => Cache::s3fifo(S3FifoCacheConfig {
-                    capacity,
-                    shards,
-                    eviction_config,
-                    object_pool_capacity,
-                    hash_builder,
-                    event_listener,
-                }),
-            }
-        };
-        let meta_cache = Arc::new(meta_cache);
-
         Self {
             path: config.path,
             store: config.store,
-            block_cache,
-            meta_cache,
 
-            data_file_cache: config.data_file_cache,
-            meta_file_cache: config.meta_file_cache,
+            meta_cache: config.meta_cache,
+            block_cache: config.block_cache,
 
             recent_filter: config.recent_filter,
             prefetch_buffer_usage: Arc::new(AtomicUsize::new(0)),
@@ -291,55 +167,56 @@ impl SstableStore {
 
     /// For compactor, we do not need a high concurrency load for cache. Instead, we need the cache
     ///  can be evict more effective.
-    pub fn for_compactor(
+    #[expect(clippy::borrowed_box)]
+    pub async fn for_compactor(
         store: ObjectStoreRef,
         path: String,
         block_cache_capacity: usize,
         meta_cache_capacity: usize,
-    ) -> Self {
-        let meta_cache = Arc::new(Cache::lru(LruCacheConfig {
-            capacity: meta_cache_capacity,
-            shards: 1,
-            eviction_config: LruConfig {
-                high_priority_pool_ratio: 0.0,
-            },
-            object_pool_capacity: 1024,
-            hash_builder: RandomState::default(),
-            event_listener: FileCache::none().into(),
-        }));
-        Self {
+    ) -> HummockResult<Self> {
+        let meta_cache = HybridCacheBuilder::new()
+            .memory(meta_cache_capacity)
+            .with_shards(1)
+            .with_weighter(|_: &HummockSstableObjectId, value: &Box<Sstable>| {
+                u64::BITS as usize / 8 + value.estimate_size()
+            })
+            .storage()
+            .build()
+            .await
+            .map_err(HummockError::foyer_error)?;
+
+        let block_cache = HybridCacheBuilder::new()
+            .memory(block_cache_capacity)
+            .with_shards(1)
+            .with_weighter(|_: &SstableBlockIndex, value: &Box<Block>| {
+                // FIXME(MrCroxx): Calculate block weight more accurately.
+                u64::BITS as usize * 2 / 8 + value.raw().len()
+            })
+            .storage()
+            .build()
+            .await
+            .map_err(HummockError::foyer_error)?;
+
+        Ok(Self {
             path,
             store,
-            block_cache: BlockCache::new(BlockCacheConfig {
-                capacity: block_cache_capacity,
-                shard_num: 1,
-                eviction: EvictionConfig::Lru(LruConfig {
-                    high_priority_pool_ratio: 0.0,
-                }),
-                listener: BlockCacheEventListener::new(
-                    FileCache::none(),
-                    Arc::new(HummockStateStoreMetrics::unused()),
-                ),
-            }),
-            meta_cache,
-            data_file_cache: FileCache::none(),
-            meta_file_cache: FileCache::none(),
+
             prefetch_buffer_usage: Arc::new(AtomicUsize::new(0)),
             prefetch_buffer_capacity: block_cache_capacity,
             max_prefetch_block_number: 16, /* compactor won't use this parameter, so just assign a default value. */
             recent_filter: None,
-        }
+
+            meta_cache,
+            block_cache,
+        })
     }
 
     pub async fn delete(&self, object_id: HummockSstableObjectId) -> HummockResult<()> {
-        // Data
         self.store
             .delete(self.get_sst_data_path(object_id).as_str())
             .await?;
         self.meta_cache.remove(&object_id);
-        self.meta_file_cache
-            .remove(&object_id)
-            .map_err(HummockError::file_cache)?;
+        // TODO(MrCroxx): support group remove in foyer.
         Ok(())
     }
 
@@ -357,21 +234,16 @@ impl SstableStore {
         self.store.delete_objects(&paths).await?;
 
         // Delete from cache.
-        for &object_id in object_id_list {
-            self.meta_cache.remove(&object_id);
-            self.meta_file_cache
-                .remove(&object_id)
-                .map_err(HummockError::file_cache)?;
+        for object_id in object_id_list {
+            self.meta_cache.remove(object_id);
         }
 
         Ok(())
     }
 
-    pub fn delete_cache(&self, object_id: HummockSstableObjectId) {
+    pub fn delete_cache(&self, object_id: HummockSstableObjectId) -> HummockResult<()> {
         self.meta_cache.remove(&object_id);
-        if let Err(e) = self.meta_file_cache.remove(&object_id) {
-            tracing::warn!(error = %e.as_report(), "meta file cache remove error");
-        }
+        Ok(())
     }
 
     async fn put_sst_data(
@@ -404,7 +276,16 @@ impl SstableStore {
             )));
         }
         stats.cache_data_block_total += 1;
-        if let Some(block) = self.block_cache.get(object_id, block_index as u64) {
+        if let Some(entry) = self
+            .block_cache
+            .get(&SstableBlockIndex {
+                sst_id: object_id,
+                block_idx: block_index as _,
+            })
+            .await
+            .map_err(HummockError::foyer_error)?
+        {
+            let block = BlockHolder::from_hybrid_cache_entry(entry);
             return Ok(Box::new(PrefetchBlockStream::new(
                 VecDeque::from([block]),
                 block_index,
@@ -417,7 +298,10 @@ impl SstableStore {
         let mut min_hit_index = end_index;
         let mut hit_count = 0;
         for idx in block_index..end_index {
-            if self.block_cache.exists_block(object_id, idx as u64) {
+            if self.block_cache.contains(&SstableBlockIndex {
+                sst_id: object_id,
+                block_idx: idx as _,
+            }) {
                 if min_hit_index > idx && idx > block_index {
                     min_hit_index = idx;
                 }
@@ -480,10 +364,17 @@ impl SstableStore {
                 let cache_priority = if idx == block_index {
                     priority
                 } else {
-                    CacheContext::LruPriorityLow
+                    CacheContext::LowPriority
                 };
-                self.block_cache
-                    .insert(object_id, idx as u64, Box::new(block), cache_priority)
+                let entry = self.block_cache.insert_with_context(
+                    SstableBlockIndex {
+                        sst_id: object_id,
+                        block_idx: idx as _,
+                    },
+                    Box::new(block),
+                    cache_priority,
+                );
+                BlockHolder::from_hybrid_cache_entry(entry)
             } else {
                 BlockHolder::from_owned_block(Box::new(block))
             };
@@ -507,32 +398,28 @@ impl SstableStore {
     ) -> HummockResult<BlockResponse> {
         let object_id = sst.id;
         let (range, uncompressed_capacity) = sst.calculate_block_info(block_index);
+        let store = self.store.clone();
 
         stats.cache_data_block_total += 1;
         let file_size = sst.meta.estimated_size;
-        let mut fetch_block = || {
-            let file_cache = self.data_file_cache.clone();
-            stats.cache_data_block_miss += 1;
-            let data_path = self.get_sst_data_path(object_id);
-            let store = self.store.clone();
-            let use_file_cache = !matches!(policy, CachePolicy::Disable);
+        let data_path = self.get_sst_data_path(object_id);
+
+        let disable_cache: fn() -> bool = || {
+            fail_point!("disable_block_cache", |_| true);
+            false
+        };
+
+        let policy = if disable_cache() {
+            CachePolicy::Disable
+        } else {
+            policy
+        };
+
+        // future: fetch block if hybrid cache miss
+        let fetch_block = move |context: CacheContext| {
             let range = range.clone();
 
             async move {
-                let key = SstableBlockIndex {
-                    sst_id: object_id,
-                    block_idx: block_index as u64,
-                };
-                if use_file_cache
-                    && let Some(block) = file_cache
-                        .lookup(&key)
-                        .await
-                        .map_err(HummockError::file_cache)?
-                {
-                    let block = block.try_into_block()?;
-                    return Ok(block);
-                }
-
                 let block_data = match store
                     .read(&data_path, range.clone())
                     .verbose_instrument_await("get_block_response")
@@ -541,66 +428,66 @@ impl SstableStore {
                     Ok(data) => data,
                     Err(e) => {
                         tracing::error!(
-                            "get_block_response meet error when read {:?} from sst-{}, total length: {}",
-                            range,
-                            object_id,
-                            file_size
-                        );
-                        return Err(HummockError::from(e));
+                        "get_block_response meet error when read {:?} from sst-{}, total length: {}",
+                        range,
+                        object_id,
+                        file_size
+                    );
+                        return Err(anyhow::Error::from(HummockError::from(e)));
                     }
                 };
-                let block = Box::new(Block::decode(block_data, uncompressed_capacity)?);
-
-                Ok(block)
+                let block = Box::new(
+                    Block::decode(block_data, uncompressed_capacity)
+                        .map_err(anyhow::Error::from)?,
+                );
+                Ok((block, context))
             }
         };
 
-        let disable_cache: fn() -> bool = || {
-            fail_point!("disable_block_cache", |_| true);
-            false
-        };
-
-        let policy = if disable_cache() {
-            CachePolicy::Disable
-        } else {
-            policy
-        };
-
         if let Some(filter) = self.recent_filter.as_ref() {
             filter.extend([(object_id, usize::MAX), (object_id, block_index)]);
         }
 
         match policy {
-            CachePolicy::Fill(priority) => Ok(self.block_cache.get_or_insert_with(
-                object_id,
-                block_index as u64,
-                priority,
-                fetch_block,
-            )),
-            CachePolicy::FillFileCache => {
-                let block = fetch_block().await?;
-                self.data_file_cache.insert_async(
+            CachePolicy::Fill(context) => {
+                let entry = self.block_cache.fetch(
                     SstableBlockIndex {
                         sst_id: object_id,
-                        block_idx: block_index as u64,
-                    },
-                    CachedBlock::Loaded {
-                        block: block.clone(),
+                        block_idx: block_index as _,
                     },
+                    move || fetch_block(context),
                 );
-                Ok(BlockResponse::Block(BlockHolder::from_owned_block(block)))
+                if matches!(entry.state(), FetchState::Miss) {
+                    stats.cache_data_block_miss += 1;
+                }
+                Ok(BlockResponse::Entry(entry))
             }
-            CachePolicy::NotFill => match self.block_cache.get(object_id, block_index as u64) {
-                Some(block) => Ok(BlockResponse::Block(block)),
-                None => fetch_block()
+            CachePolicy::NotFill => {
+                if let Some(entry) = self
+                    .block_cache
+                    .get(&SstableBlockIndex {
+                        sst_id: object_id,
+                        block_idx: block_index as _,
+                    })
                     .await
-                    .map(BlockHolder::from_owned_block)
-                    .map(BlockResponse::Block),
-            },
-            CachePolicy::Disable => fetch_block()
-                .await
-                .map(BlockHolder::from_owned_block)
-                .map(BlockResponse::Block),
+                    .map_err(HummockError::foyer_error)?
+                {
+                    Ok(BlockResponse::Block(BlockHolder::from_hybrid_cache_entry(
+                        entry,
+                    )))
+                } else {
+                    let (block, _) = fetch_block(CacheContext::default())
+                        .await
+                        .map_err(HummockError::foyer_error)?;
+                    Ok(BlockResponse::Block(BlockHolder::from_owned_block(block)))
+                }
+            }
+            CachePolicy::Disable => {
+                let (block, _) = fetch_block(CacheContext::default())
+                    .await
+                    .map_err(HummockError::foyer_error)?;
+                Ok(BlockResponse::Block(BlockHolder::from_owned_block(block)))
+            }
         }
     }
 
@@ -642,41 +529,29 @@ impl SstableStore {
     }
 
     #[cfg(any(test, feature = "test"))]
-    pub fn clear_block_cache(&self) {
-        self.block_cache.clear();
-        if let Err(e) = self.data_file_cache.clear() {
-            tracing::warn!(error = %e.as_report(), "data file cache clear error");
-        }
+    pub async fn clear_block_cache(&self) -> HummockResult<()> {
+        self.block_cache
+            .clear()
+            .await
+            .map_err(HummockError::foyer_error)
     }
 
     #[cfg(any(test, feature = "test"))]
-    pub fn clear_meta_cache(&self) {
-        self.meta_cache.clear();
-        if let Err(e) = self.meta_file_cache.clear() {
-            tracing::warn!(error = %e.as_report(), "meta file cache clear error");
-        }
+    pub async fn clear_meta_cache(&self) -> HummockResult<()> {
+        self.meta_cache
+            .clear()
+            .await
+            .map_err(HummockError::foyer_error)
     }
 
     pub async fn sstable_cached(
         &self,
         sst_obj_id: HummockSstableObjectId,
-    ) -> HummockResult<
-        Option<CachedOrShared<HummockSstableObjectId, Sstable, MetaCacheEventListener>>,
-    > {
-        if let Some(sst) = self.meta_cache.get(&sst_obj_id) {
-            return Ok(Some(CachedOrShared::Cached(sst)));
-        }
-
-        if let Some(sst) = self
-            .meta_file_cache
-            .lookup(&sst_obj_id)
+    ) -> HummockResult<Option<HybridCacheEntry<HummockSstableObjectId, Box<Sstable>>>> {
+        self.meta_cache
+            .get(&sst_obj_id)
             .await
-            .map_err(HummockError::file_cache)?
-        {
-            return Ok(Some(CachedOrShared::Shared(sst.into_inner())));
-        }
-
-        Ok(None)
+            .map_err(HummockError::foyer_error)
     }
 
     /// Returns `table_holder`
@@ -686,43 +561,31 @@ impl SstableStore {
         stats: &mut StoreLocalStatistic,
     ) -> impl Future<Output = HummockResult<TableHolder>> + Send + 'static {
         let object_id = sst.get_object_id();
-        let entry = self.meta_cache.entry(object_id, || {
-            let meta_file_cache = self.meta_file_cache.clone();
+
+        let entry = self.meta_cache.fetch(object_id, || {
             let store = self.store.clone();
             let meta_path = self.get_sst_data_path(object_id);
             let stats_ptr = stats.remote_io_time.clone();
             let range = sst.meta_offset as usize..sst.file_size as usize;
             async move {
-                if let Some(sst) = meta_file_cache
-                    .lookup(&object_id)
-                    .await
-                    .map_err(HummockError::file_cache)?
-                {
-                    // TODO(MrCroxx): Make meta cache receives Arc<Sstable> to reduce copy?
-                    let sst: Box<Sstable> = sst.into();
-                    let charge = sst.estimate_size();
-                    return Ok((sst, charge, CacheContext::Default));
-                }
-
                 let now = Instant::now();
                 let buf = store.read(&meta_path, range).await?;
                 let meta = SstableMeta::decode(&buf[..])?;
 
                 let sst = Sstable::new(object_id, meta);
-                let charge = sst.estimate_size();
                 let add = (now.elapsed().as_secs_f64() * 1000.0).ceil();
                 stats_ptr.fetch_add(add as u64, Ordering::Relaxed);
-                Ok((Box::new(sst), charge, CacheContext::Default))
+                Ok((Box::new(sst), CacheContext::Default))
             }
         });
 
-        if matches! { entry.state(), EntryState::Wait | EntryState::Miss } {
+        if matches! { entry.state(), FetchState::Wait | FetchState::Miss } {
             stats.cache_meta_block_miss += 1;
         }
 
         stats.cache_meta_block_total += 1;
 
-        entry
+        async move { entry.await.map_err(HummockError::foyer_error) }
     }
 
     pub async fn list_object_metadata_from_object_store(
@@ -746,13 +609,7 @@ impl SstableStore {
 
     pub fn insert_meta_cache(&self, object_id: HummockSstableObjectId, meta: SstableMeta) {
         let sst = Sstable::new(object_id, meta);
-        let charge = sst.estimate_size();
-        self.meta_cache.insert_with_context(
-            object_id,
-            Box::new(sst),
-            charge,
-            CacheContext::Default,
-        );
+        self.meta_cache.insert(object_id, Box::new(sst));
     }
 
     pub fn insert_block_cache(
@@ -761,15 +618,21 @@ impl SstableStore {
         block_index: u64,
         block: Box<Block>,
     ) {
-        if let Some(filter) = self.recent_filter.as_ref() {
-            filter.extend([(object_id, usize::MAX), (object_id, block_index as usize)]);
-        }
-        self.block_cache
-            .insert(object_id, block_index, block, CacheContext::Default);
+        self.block_cache.insert(
+            SstableBlockIndex {
+                sst_id: object_id,
+                block_idx: block_index,
+            },
+            block,
+        );
     }
 
     pub fn get_meta_memory_usage(&self) -> u64 {
-        self.meta_cache.usage() as u64
+        self.meta_cache.memory().usage() as _
+    }
+
+    pub fn get_prefetch_memory_usage(&self) -> usize {
+        self.prefetch_buffer_usage.load(Ordering::Acquire)
     }
 
     pub async fn get_stream_for_blocks(
@@ -806,11 +669,11 @@ impl SstableStore {
         self.recent_filter.as_ref()
     }
 
-    pub fn data_file_cache(&self) -> &FileCache<SstableBlockIndex, CachedBlock> {
-        &self.data_file_cache
+    pub fn meta_cache(&self) -> &HybridCache<HummockSstableObjectId, Box<Sstable>> {
+        &self.meta_cache
     }
 
-    pub fn data_cache(&self) -> &BlockCache {
+    pub fn block_cache(&self) -> &HybridCache<SstableBlockIndex, Box<Block>> {
         &self.block_cache
     }
 }
@@ -843,20 +706,24 @@ impl MemoryCollector for HummockMemoryCollector {
     }
 
     fn get_data_memory_usage(&self) -> u64 {
-        self.sstable_store.block_cache.size() as u64
+        self.sstable_store.block_cache.memory().usage() as _
     }
 
     fn get_uploading_memory_usage(&self) -> u64 {
         self.limiter.get_memory_usage()
     }
 
+    fn get_prefetch_memory_usage(&self) -> usize {
+        self.sstable_store.get_prefetch_memory_usage()
+    }
+
     fn get_meta_cache_memory_usage_ratio(&self) -> f64 {
         self.sstable_store.get_meta_memory_usage() as f64
             / (self.storage_memory_config.meta_cache_capacity_mb * 1024 * 1024) as f64
     }
 
     fn get_block_cache_memory_usage_ratio(&self) -> f64 {
-        self.sstable_store.block_cache.size() as f64
+        self.get_data_memory_usage() as f64
             / (self.storage_memory_config.block_cache_capacity_mb * 1024 * 1024) as f64
     }
 
@@ -1002,9 +869,11 @@ impl SstableWriter for BatchUploadWriter {
                 // The `block_info` may be empty when there is only range-tombstones, because we
                 //  store them in meta-block.
                 for (block_idx, block) in self.block_info.into_iter().enumerate() {
-                    self.sstable_store.block_cache.insert(
-                        self.object_id,
-                        block_idx as u64,
+                    self.sstable_store.block_cache.insert_with_context(
+                        SstableBlockIndex {
+                            sst_id: self.object_id,
+                            block_idx: block_idx as _,
+                        },
                         Box::new(block),
                         fill_cache_priority,
                     );
@@ -1111,9 +980,11 @@ impl SstableWriter for StreamingUploadWriter {
                 && !self.blocks.is_empty()
             {
                 for (block_idx, block) in self.blocks.into_iter().enumerate() {
-                    self.sstable_store.block_cache.insert(
-                        self.object_id,
-                        block_idx as u64,
+                    self.sstable_store.block_cache.insert_with_context(
+                        SstableBlockIndex {
+                            sst_id: self.object_id,
+                            block_idx: block_idx as _,
+                        },
                         Box::new(block),
                         fill_high_priority_cache,
                     );
@@ -1295,7 +1166,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_batch_upload() {
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let x_range = 0..100;
         let (data, meta) = gen_test_sstable_data(
             default_builder_opt_for_test(),
@@ -1325,7 +1196,7 @@ mod tests {
     #[tokio::test]
     async fn test_streaming_upload() {
         // Generate test data.
-        let sstable_store = mock_sstable_store();
+        let sstable_store = mock_sstable_store().await;
         let x_range = 0..100;
         let (data, meta) = gen_test_sstable_data(
             default_builder_opt_for_test(),
@@ -1352,9 +1223,9 @@ mod tests {
         validate_sst(sstable_store, &info, meta, x_range).await;
     }
 
-    #[test]
-    fn test_basic() {
-        let sstable_store = mock_sstable_store();
+    #[tokio::test]
+    async fn test_basic() {
+        let sstable_store = mock_sstable_store().await;
         let object_id = 123;
         let data_path = sstable_store.get_sst_data_path(object_id);
         assert_eq!(data_path, "test/123.data");
diff --git a/src/storage/src/hummock/store/hummock_storage.rs b/src/storage/src/hummock/store/hummock_storage.rs
index 8c18b647ca819..7f9f956b62ddf 100644
--- a/src/storage/src/hummock/store/hummock_storage.rs
+++ b/src/storage/src/hummock/store/hummock_storage.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
 
 use arc_swap::ArcSwap;
 use bytes::Bytes;
+use futures::FutureExt;
 use itertools::Itertools;
 use more_asserts::assert_gt;
 use risingwave_common::catalog::TableId;
@@ -28,7 +29,7 @@ use risingwave_hummock_sdk::key::{
     is_empty_key_range, vnode, vnode_range, TableKey, TableKeyRange,
 };
 use risingwave_hummock_sdk::table_watermark::TableWatermarksIndex;
-use risingwave_hummock_sdk::{HummockReadEpoch, SyncResult};
+use risingwave_hummock_sdk::HummockReadEpoch;
 use risingwave_pb::hummock::SstableInfo;
 use risingwave_rpc_client::HummockMetaClient;
 use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
@@ -47,20 +48,19 @@ use crate::hummock::event_handler::hummock_event_handler::{BufferTracker, Hummoc
 use crate::hummock::event_handler::{
     HummockEvent, HummockEventHandler, HummockVersionUpdate, ReadOnlyReadVersionMapping,
 };
-use crate::hummock::iterator::ChangeLogIterator;
+use crate::hummock::iterator::change_log::ChangeLogIterator;
 use crate::hummock::local_version::pinned_version::{start_pinned_version_worker, PinnedVersion};
 use crate::hummock::observer_manager::HummockObserverNode;
 use crate::hummock::utils::{validate_safe_epoch, wait_for_epoch};
 use crate::hummock::write_limiter::{WriteLimiter, WriteLimiterRef};
 use crate::hummock::{
-    HummockEpoch, HummockError, HummockResult, HummockStorageIterator, MemoryLimiter,
-    SstableObjectIdManager, SstableObjectIdManagerRef, SstableStoreRef,
+    HummockEpoch, HummockError, HummockResult, HummockStorageIterator, HummockStorageRevIterator,
+    MemoryLimiter, SstableObjectIdManager, SstableObjectIdManagerRef, SstableStoreRef,
 };
 use crate::mem_table::ImmutableMemtable;
 use crate::monitor::{CompactorMetrics, HummockStateStoreMetrics, StoreLocalStatistic};
 use crate::opts::StorageOpts;
 use crate::store::*;
-use crate::StateStore;
 
 struct HummockStorageShutdownGuard {
     shutdown_sender: HummockEventSender,
@@ -171,7 +171,7 @@ impl HummockStorage {
         observer_manager.start().await;
 
         let hummock_version = match version_update_rx.recv().await {
-            Some(HummockVersionUpdate::PinnedVersion(version)) => version,
+            Some(HummockVersionUpdate::PinnedVersion(version)) => *version,
             _ => unreachable!("the hummock observer manager is the first one to take the event tx. Should be full hummock version")
         };
 
@@ -287,15 +287,38 @@ impl HummockStorage {
             .await
     }
 
+    async fn rev_iter_inner(
+        &self,
+        key_range: TableKeyRange,
+        epoch: u64,
+        read_options: ReadOptions,
+    ) -> StorageResult<HummockStorageRevIterator> {
+        let (key_range, read_version_tuple) = if read_options.read_version_from_backup {
+            self.build_read_version_tuple_from_backup(epoch, read_options.table_id, key_range)
+                .await?
+        } else {
+            self.build_read_version_tuple(epoch, read_options.table_id, key_range)?
+        };
+
+        self.hummock_version_reader
+            .rev_iter(key_range, epoch, read_options, read_version_tuple, None)
+            .await
+    }
+
     async fn build_read_version_tuple_from_backup(
         &self,
         epoch: u64,
         table_id: TableId,
         key_range: TableKeyRange,
     ) -> StorageResult<(TableKeyRange, ReadVersionTuple)> {
-        match self.backup_reader.try_get_hummock_version(epoch).await {
+        match self
+            .backup_reader
+            .try_get_hummock_version(table_id, epoch)
+            .await
+        {
             Ok(Some(backup_version)) => {
-                validate_safe_epoch(backup_version.safe_epoch(), epoch)?;
+                validate_safe_epoch(backup_version.version(), table_id, epoch)?;
+
                 Ok(get_committed_read_version_tuple(
                     backup_version,
                     table_id,
@@ -319,7 +342,7 @@ impl HummockStorage {
         key_range: TableKeyRange,
     ) -> StorageResult<(TableKeyRange, ReadVersionTuple)> {
         let pinned_version = self.pinned_version.load();
-        validate_safe_epoch(pinned_version.safe_epoch(), epoch)?;
+        validate_safe_epoch(pinned_version.version(), table_id, epoch)?;
 
         // check epoch if lower mce
         let ret = if epoch <= pinned_version.max_committed_epoch() {
@@ -462,6 +485,7 @@ impl HummockStorage {
 impl StateStoreRead for HummockStorage {
     type ChangeLogIter = ChangeLogIterator;
     type Iter = HummockStorageIterator;
+    type RevIter = HummockStorageRevIterator;
 
     fn get(
         &self,
@@ -489,6 +513,23 @@ impl StateStoreRead for HummockStorage {
         self.iter_inner(key_range, epoch, read_options)
     }
 
+    fn rev_iter(
+        &self,
+        key_range: TableKeyRange,
+        epoch: u64,
+        read_options: ReadOptions,
+    ) -> impl Future<Output = StorageResult<Self::RevIter>> + '_ {
+        let (l_vnode_inclusive, r_vnode_exclusive) = vnode_range(&key_range);
+        assert_eq!(
+            r_vnode_exclusive - l_vnode_inclusive,
+            1,
+            "read range {:?} for table {} iter contains more than one vnode",
+            key_range,
+            read_options.table_id
+        );
+        self.rev_iter_inner(key_range, epoch, read_options)
+    }
+
     async fn iter_log(
         &self,
         epoch_range: (u64, u64),
@@ -521,15 +562,15 @@ impl StateStore for HummockStorage {
         wait_for_epoch(&self.version_update_notifier_tx, wait_epoch).await
     }
 
-    async fn sync(&self, epoch: u64) -> StorageResult<SyncResult> {
+    fn sync(&self, epoch: u64) -> impl SyncFuture {
         let (tx, rx) = oneshot::channel();
         self.hummock_event_sender
-            .send(HummockEvent::AwaitSyncEpoch {
+            .send(HummockEvent::SyncEpoch {
                 new_sync_epoch: epoch,
                 sync_result_sender: tx,
             })
             .expect("should send success");
-        Ok(rx.await.expect("should wait success")?)
+        rx.map(|recv_result| Ok(recv_result.expect("should wait success")?))
     }
 
     fn seal_epoch(&self, epoch: u64, is_checkpoint: bool) {
@@ -546,12 +587,6 @@ impl StateStore for HummockStorage {
                 MemOrdering::SeqCst,
             );
         }
-        self.hummock_event_sender
-            .send(HummockEvent::SealEpoch {
-                epoch,
-                is_checkpoint,
-            })
-            .expect("should send success");
         StoreLocalStatistic::flush_all();
     }
 
@@ -607,7 +642,10 @@ use risingwave_hummock_sdk::version::HummockVersion;
 
 #[cfg(any(test, feature = "test"))]
 impl HummockStorage {
-    pub async fn seal_and_sync_epoch(&self, epoch: u64) -> StorageResult<SyncResult> {
+    pub async fn seal_and_sync_epoch(
+        &self,
+        epoch: u64,
+    ) -> StorageResult<risingwave_hummock_sdk::SyncResult> {
         self.seal_epoch(epoch, true);
         self.sync(epoch).await
     }
@@ -618,7 +656,7 @@ impl HummockStorage {
         use tokio::task::yield_now;
         let version_id = version.id;
         self._version_update_sender
-            .send(HummockVersionUpdate::PinnedVersion(version))
+            .send(HummockVersionUpdate::PinnedVersion(Box::new(version)))
             .unwrap();
         loop {
             if self.pinned_version.load().id() >= version_id {
diff --git a/src/storage/src/hummock/store/local_hummock_storage.rs b/src/storage/src/hummock/store/local_hummock_storage.rs
index d9a83c8695b14..a14f3b450adff 100644
--- a/src/storage/src/hummock/store/local_hummock_storage.rs
+++ b/src/storage/src/hummock/store/local_hummock_storage.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::future::Future;
+use std::iter::once;
 use std::ops::Bound;
 use std::sync::Arc;
 
@@ -23,29 +24,34 @@ use risingwave_common::catalog::{TableId, TableOption};
 use risingwave_common::util::epoch::MAX_SPILL_TIMES;
 use risingwave_hummock_sdk::key::{is_empty_key_range, vnode_range, TableKey, TableKeyRange};
 use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch};
+use risingwave_pb::hummock::SstableInfo;
 use tracing::{warn, Instrument};
 
 use super::version::{StagingData, VersionUpdate};
 use crate::error::StorageResult;
 use crate::hummock::event_handler::hummock_event_handler::HummockEventSender;
 use crate::hummock::event_handler::{HummockEvent, HummockReadVersionRef, LocalInstanceGuard};
+use crate::hummock::iterator::change_log::ChangeLogIterator;
 use crate::hummock::iterator::{
-    ChangeLogIterator, ConcatIteratorInner, Forward, HummockIteratorUnion, MergeIterator,
-    UserIterator,
+    Backward, BackwardUserIterator, ConcatIteratorInner, Forward, HummockIteratorUnion,
+    IteratorFactory, MergeIterator, UserIterator,
 };
 use crate::hummock::shared_buffer::shared_buffer_batch::{
-    SharedBufferBatch, SharedBufferBatchIterator,
+    SharedBufferBatch, SharedBufferBatchIterator, SharedBufferBatchOldValues, SharedBufferItem,
+    SharedBufferValue,
 };
 use crate::hummock::store::version::{read_filter_for_version, HummockVersionReader};
 use crate::hummock::utils::{
-    do_delete_sanity_check, do_insert_sanity_check, do_update_sanity_check, wait_for_epoch,
-    ENABLE_SANITY_CHECK,
+    do_delete_sanity_check, do_insert_sanity_check, do_update_sanity_check, sanity_check_enabled,
+    wait_for_epoch,
 };
 use crate::hummock::write_limiter::WriteLimiterRef;
-use crate::hummock::{MemoryLimiter, SstableIterator};
-use crate::mem_table::{KeyOp, MemTable, MemTableHummockIterator};
+use crate::hummock::{
+    BackwardSstableIterator, MemoryLimiter, SstableIterator, SstableIteratorReadOptions,
+    SstableStoreRef,
+};
+use crate::mem_table::{KeyOp, MemTable, MemTableHummockIterator, MemTableHummockRevIterator};
 use crate::monitor::{HummockStateStoreMetrics, IterLocalMetricsGuard, StoreLocalStatistic};
-use crate::storage_value::StorageValue;
 use crate::store::*;
 
 /// `LocalHummockStorage` is a handle for a state table shard to access data from and write data to
@@ -151,6 +157,26 @@ impl LocalHummockStorage {
             .await
     }
 
+    pub async fn rev_iter_flushed(
+        &self,
+        table_key_range: TableKeyRange,
+        epoch: u64,
+        read_options: ReadOptions,
+    ) -> StorageResult<HummockStorageRevIterator> {
+        let (table_key_range, read_snapshot) = read_filter_for_version(
+            epoch,
+            read_options.table_id,
+            table_key_range,
+            &self.read_version,
+        )?;
+
+        let table_key_range = table_key_range;
+
+        self.hummock_version_reader
+            .rev_iter(table_key_range, epoch, read_options, read_snapshot, None)
+            .await
+    }
+
     fn mem_table_iter(&self) -> MemTableHummockIterator<'_> {
         MemTableHummockIterator::new(
             &self.mem_table.buffer,
@@ -159,6 +185,14 @@ impl LocalHummockStorage {
         )
     }
 
+    fn mem_table_rev_iter(&self) -> MemTableHummockRevIterator<'_> {
+        MemTableHummockRevIterator::new(
+            &self.mem_table.buffer,
+            EpochWithGap::new(self.epoch(), self.spill_offset),
+            self.table_id,
+        )
+    }
+
     pub async fn iter_all(
         &self,
         table_key_range: TableKeyRange,
@@ -178,7 +212,31 @@ impl LocalHummockStorage {
                 epoch,
                 read_options,
                 read_snapshot,
-                self.mem_table_iter(),
+                Some(self.mem_table_iter()),
+            )
+            .await
+    }
+
+    pub async fn rev_iter_all(
+        &self,
+        table_key_range: TableKeyRange,
+        epoch: u64,
+        read_options: ReadOptions,
+    ) -> StorageResult<LocalHummockStorageRevIterator<'_>> {
+        let (table_key_range, read_snapshot) = read_filter_for_version(
+            epoch,
+            read_options.table_id,
+            table_key_range,
+            &self.read_version,
+        )?;
+
+        self.hummock_version_reader
+            .rev_iter(
+                table_key_range,
+                epoch,
+                read_options,
+                read_snapshot,
+                Some(self.mem_table_rev_iter()),
             )
             .await
     }
@@ -208,6 +266,7 @@ impl LocalHummockStorage {
 impl StateStoreRead for LocalHummockStorage {
     type ChangeLogIter = ChangeLogIterator;
     type Iter = HummockStorageIterator;
+    type RevIter = HummockStorageRevIterator;
 
     fn get(
         &self,
@@ -230,6 +289,17 @@ impl StateStoreRead for LocalHummockStorage {
             .instrument(tracing::trace_span!("hummock_iter"))
     }
 
+    fn rev_iter(
+        &self,
+        key_range: TableKeyRange,
+        epoch: u64,
+        read_options: ReadOptions,
+    ) -> impl Future<Output = StorageResult<Self::RevIter>> + '_ {
+        assert!(epoch <= self.epoch());
+        self.rev_iter_flushed(key_range, epoch, read_options)
+            .instrument(tracing::trace_span!("hummock_rev_iter"))
+    }
+
     async fn iter_log(
         &self,
         epoch_range: (u64, u64),
@@ -247,6 +317,7 @@ impl StateStoreRead for LocalHummockStorage {
 
 impl LocalStateStore for LocalHummockStorage {
     type Iter<'a> = LocalHummockStorageIterator<'a>;
+    type RevIter<'a> = LocalHummockStorageRevIterator<'a>;
 
     fn may_exist(
         &self,
@@ -287,6 +358,23 @@ impl LocalStateStore for LocalHummockStorage {
             .await
     }
 
+    async fn rev_iter(
+        &self,
+        key_range: TableKeyRange,
+        read_options: ReadOptions,
+    ) -> StorageResult<Self::RevIter<'_>> {
+        let (l_vnode_inclusive, r_vnode_exclusive) = vnode_range(&key_range);
+        assert_eq!(
+            r_vnode_exclusive - l_vnode_inclusive,
+            1,
+            "read range {:?} for table {} iter contains more than one vnode",
+            key_range,
+            read_options.table_id
+        );
+        self.rev_iter_all(key_range.clone(), self.epoch(), read_options)
+            .await
+    }
+
     fn insert(
         &mut self,
         key: TableKey<Bytes>,
@@ -310,13 +398,18 @@ impl LocalStateStore for LocalHummockStorage {
     async fn flush(&mut self) -> StorageResult<usize> {
         let buffer = self.mem_table.drain().into_parts();
         let mut kv_pairs = Vec::with_capacity(buffer.len());
+        let mut old_values = if self.is_flush_old_value() {
+            Some(Vec::with_capacity(buffer.len()))
+        } else {
+            None
+        };
         for (key, key_op) in buffer {
             match key_op {
                 // Currently, some executors do not strictly comply with these semantics. As
                 // a workaround you may call disable the check by initializing the
                 // state store with `is_consistent_op=false`.
                 KeyOp::Insert(value) => {
-                    if ENABLE_SANITY_CHECK {
+                    if sanity_check_enabled() {
                         do_insert_sanity_check(
                             &key,
                             &value,
@@ -328,10 +421,13 @@ impl LocalStateStore for LocalHummockStorage {
                         )
                         .await?;
                     }
-                    kv_pairs.push((key, StorageValue::new_put(value)));
+                    kv_pairs.push((key, SharedBufferValue::Insert(value)));
+                    if let Some(old_values) = &mut old_values {
+                        old_values.push(Bytes::new());
+                    }
                 }
                 KeyOp::Delete(old_value) => {
-                    if ENABLE_SANITY_CHECK {
+                    if sanity_check_enabled() {
                         do_delete_sanity_check(
                             &key,
                             &old_value,
@@ -343,10 +439,13 @@ impl LocalStateStore for LocalHummockStorage {
                         )
                         .await?;
                     }
-                    kv_pairs.push((key, StorageValue::new_delete()));
+                    kv_pairs.push((key, SharedBufferValue::Delete));
+                    if let Some(old_values) = &mut old_values {
+                        old_values.push(old_value);
+                    }
                 }
                 KeyOp::Update((old_value, new_value)) => {
-                    if ENABLE_SANITY_CHECK {
+                    if sanity_check_enabled() {
                         do_update_sanity_check(
                             &key,
                             &old_value,
@@ -359,12 +458,16 @@ impl LocalStateStore for LocalHummockStorage {
                         )
                         .await?;
                     }
-                    kv_pairs.push((key, StorageValue::new_put(new_value)));
+                    kv_pairs.push((key, SharedBufferValue::Update(new_value)));
+                    if let Some(old_values) = &mut old_values {
+                        old_values.push(old_value);
+                    }
                 }
             }
         }
         self.flush_inner(
             kv_pairs,
+            old_values,
             WriteOptions {
                 epoch: self.epoch(),
                 table_id: self.table_id,
@@ -410,6 +513,14 @@ impl LocalStateStore for LocalHummockStorage {
             "local state store of table id {:?} is init for more than once",
             self.table_id
         );
+        if !self.is_replicated {
+            self.event_sender
+                .send(HummockEvent::InitEpoch {
+                    instance_id: self.instance_id(),
+                    init_epoch: options.epoch.curr,
+                })
+                .expect("should succeed");
+        }
         Ok(())
     }
 
@@ -441,14 +552,15 @@ impl LocalStateStore for LocalHummockStorage {
                 });
             }
         }
-        self.event_sender
-            .send(HummockEvent::LocalSealEpoch {
-                instance_id: self.instance_id(),
-                table_id: self.table_id,
-                epoch: prev_epoch,
-                opts,
-            })
-            .expect("should be able to send")
+        if !self.is_replicated {
+            self.event_sender
+                .send(HummockEvent::LocalSealEpoch {
+                    instance_id: self.instance_id(),
+                    next_epoch,
+                    opts,
+                })
+                .expect("should be able to send");
+        }
     }
 
     fn update_vnode_bitmap(&mut self, vnodes: Arc<Bitmap>) -> Arc<Bitmap> {
@@ -463,7 +575,8 @@ impl LocalStateStore for LocalHummockStorage {
 impl LocalHummockStorage {
     async fn flush_inner(
         &mut self,
-        kv_pairs: Vec<(TableKey<Bytes>, StorageValue)>,
+        sorted_items: Vec<SharedBufferItem>,
+        old_values: Option<Vec<Bytes>>,
         write_options: WriteOptions,
     ) -> StorageResult<usize> {
         let epoch = write_options.epoch;
@@ -473,16 +586,17 @@ impl LocalHummockStorage {
         self.stats
             .write_batch_tuple_counts
             .with_label_values(&[table_id_label.as_str()])
-            .inc_by(kv_pairs.len() as _);
+            .inc_by(sorted_items.len() as _);
         let timer = self
             .stats
             .write_batch_duration
             .with_label_values(&[table_id_label.as_str()])
             .start_timer();
 
-        let imm_size = if !kv_pairs.is_empty() {
-            let sorted_items = SharedBufferBatch::build_shared_buffer_item_batches(kv_pairs);
-            let size = SharedBufferBatch::measure_batch_size(&sorted_items);
+        let imm_size = if !sorted_items.is_empty() {
+            let (size, old_value_size) =
+                SharedBufferBatch::measure_batch_size(&sorted_items, old_values.as_deref());
+
             self.write_limiter.wait_permission(self.table_id).await;
             let limiter = self.memory_limiter.as_ref();
             let tracker = if let Some(tracker) = limiter.try_require_memory(size as u64) {
@@ -508,14 +622,22 @@ impl LocalHummockStorage {
                 tracker
             };
 
+            let old_values = old_values.map(|old_values| {
+                SharedBufferBatchOldValues::new(
+                    old_values,
+                    old_value_size,
+                    self.stats.old_value_size.clone(),
+                )
+            });
+
             let instance_id = self.instance_guard.instance_id;
             let imm = SharedBufferBatch::build_shared_buffer_batch(
                 epoch,
                 self.spill_offset,
                 sorted_items,
+                old_values,
                 size,
                 table_id,
-                instance_id,
                 Some(tracker),
             );
             self.spill_offset += 1;
@@ -525,7 +647,7 @@ impl LocalHummockStorage {
             // insert imm to uploader
             if !self.is_replicated {
                 self.event_sender
-                    .send(HummockEvent::ImmToUploader(imm))
+                    .send(HummockEvent::ImmToUploader { instance_id, imm })
                     .unwrap();
             }
             imm_size
@@ -589,11 +711,24 @@ impl LocalHummockStorage {
     pub fn instance_id(&self) -> u64 {
         self.instance_guard.instance_id
     }
+
+    fn is_flush_old_value(&self) -> bool {
+        matches!(
+            &self.op_consistency_level,
+            OpConsistencyLevel::ConsistentOldValue {
+                is_log_store: true,
+                ..
+            }
+        )
+    }
 }
 
 pub type StagingDataIterator = MergeIterator<
     HummockIteratorUnion<Forward, SharedBufferBatchIterator<Forward>, SstableIterator>,
 >;
+pub type StagingDataRevIterator = MergeIterator<
+    HummockIteratorUnion<Backward, SharedBufferBatchIterator<Backward>, BackwardSstableIterator>,
+>;
 pub type HummockStorageIteratorPayloadInner<'a> = MergeIterator<
     HummockIteratorUnion<
         Forward,
@@ -604,8 +739,20 @@ pub type HummockStorageIteratorPayloadInner<'a> = MergeIterator<
     >,
 >;
 
+pub type StorageRevIteratorPayloadInner<'a> = MergeIterator<
+    HummockIteratorUnion<
+        Backward,
+        StagingDataRevIterator,
+        BackwardSstableIterator,
+        ConcatIteratorInner<BackwardSstableIterator>,
+        MemTableHummockRevIterator<'a>,
+    >,
+>;
+
 pub type HummockStorageIterator = HummockStorageIteratorInner<'static>;
+pub type HummockStorageRevIterator = HummockStorageRevIteratorInner<'static>;
 pub type LocalHummockStorageIterator<'a> = HummockStorageIteratorInner<'a>;
+pub type LocalHummockStorageRevIterator<'a> = HummockStorageRevIteratorInner<'a>;
 
 pub struct HummockStorageIteratorInner<'a> {
     inner: UserIterator<HummockStorageIteratorPayloadInner<'a>>,
@@ -635,8 +782,13 @@ impl<'a> HummockStorageIteratorInner<'a> {
         inner: UserIterator<HummockStorageIteratorPayloadInner<'a>>,
         metrics: Arc<HummockStateStoreMetrics>,
         table_id: TableId,
-        local_stats: StoreLocalStatistic,
+        mut local_stats: StoreLocalStatistic,
     ) -> Self {
+        local_stats.found_key = inner.is_valid();
+        local_stats.sub_iter_count = local_stats.staging_imm_iter_count
+            + local_stats.staging_sst_iter_count
+            + local_stats.overlapping_iter_count
+            + local_stats.non_overlapping_iter_count;
         Self {
             inner,
             initial_read: false,
@@ -651,3 +803,187 @@ impl<'a> Drop for HummockStorageIteratorInner<'a> {
             .collect_local_statistic(&mut self.stats_guard.local_stats);
     }
 }
+
+#[derive(Default)]
+pub struct ForwardIteratorFactory {
+    non_overlapping_iters: Vec<ConcatIteratorInner<SstableIterator>>,
+    overlapping_iters: Vec<SstableIterator>,
+    staging_iters:
+        Vec<HummockIteratorUnion<Forward, SharedBufferBatchIterator<Forward>, SstableIterator>>,
+}
+
+impl ForwardIteratorFactory {
+    pub fn build(
+        self,
+        mem_table: Option<MemTableHummockIterator<'_>>,
+    ) -> HummockStorageIteratorPayloadInner<'_> {
+        // 3. build user_iterator
+        let staging_iter = StagingDataIterator::new(self.staging_iters);
+        MergeIterator::new(
+            once(HummockIteratorUnion::First(staging_iter))
+                .chain(
+                    self.overlapping_iters
+                        .into_iter()
+                        .map(HummockIteratorUnion::Second),
+                )
+                .chain(
+                    self.non_overlapping_iters
+                        .into_iter()
+                        .map(HummockIteratorUnion::Third),
+                )
+                .chain(mem_table.into_iter().map(HummockIteratorUnion::Fourth)),
+        )
+    }
+}
+
+pub struct HummockStorageRevIteratorInner<'a> {
+    inner: BackwardUserIterator<StorageRevIteratorPayloadInner<'a>>,
+    initial_read: bool,
+    stats_guard: IterLocalMetricsGuard,
+}
+
+impl<'a> StateStoreIter for HummockStorageRevIteratorInner<'a> {
+    async fn try_next<'b>(&'b mut self) -> StorageResult<Option<StateStoreIterItemRef<'b>>> {
+        let iter = &mut self.inner;
+        if !self.initial_read {
+            self.initial_read = true;
+        } else {
+            iter.next().await?;
+        }
+
+        if iter.is_valid() {
+            Ok(Some((iter.key(), iter.value())))
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+impl<'a> HummockStorageRevIteratorInner<'a> {
+    pub fn new(
+        inner: BackwardUserIterator<StorageRevIteratorPayloadInner<'a>>,
+        metrics: Arc<HummockStateStoreMetrics>,
+        table_id: TableId,
+        mut local_stats: StoreLocalStatistic,
+    ) -> Self {
+        local_stats.found_key = inner.is_valid();
+        local_stats.sub_iter_count = local_stats.staging_imm_iter_count
+            + local_stats.staging_sst_iter_count
+            + local_stats.overlapping_iter_count
+            + local_stats.non_overlapping_iter_count;
+        Self {
+            inner,
+            initial_read: false,
+            stats_guard: IterLocalMetricsGuard::new(metrics, table_id, local_stats),
+        }
+    }
+}
+
+impl<'a> Drop for HummockStorageRevIteratorInner<'a> {
+    fn drop(&mut self) {
+        self.inner
+            .collect_local_statistic(&mut self.stats_guard.local_stats);
+    }
+}
+
+impl IteratorFactory for ForwardIteratorFactory {
+    type Direction = Forward;
+    type SstableIteratorType = SstableIterator;
+
+    fn add_batch_iter(&mut self, batch: SharedBufferBatch) {
+        self.staging_iters
+            .push(HummockIteratorUnion::First(batch.into_forward_iter()));
+    }
+
+    fn add_staging_sst_iter(&mut self, iter: Self::SstableIteratorType) {
+        self.staging_iters.push(HummockIteratorUnion::Second(iter));
+    }
+
+    fn add_overlapping_sst_iter(&mut self, iter: Self::SstableIteratorType) {
+        self.overlapping_iters.push(iter);
+    }
+
+    fn add_concat_sst_iter(
+        &mut self,
+        tables: Vec<SstableInfo>,
+        sstable_store: SstableStoreRef,
+        read_options: Arc<SstableIteratorReadOptions>,
+    ) {
+        self.non_overlapping_iters
+            .push(ConcatIteratorInner::<Self::SstableIteratorType>::new(
+                tables,
+                sstable_store,
+                read_options,
+            ));
+    }
+}
+
+#[derive(Default)]
+pub struct BackwardIteratorFactory {
+    non_overlapping_iters: Vec<ConcatIteratorInner<BackwardSstableIterator>>,
+    overlapping_iters: Vec<BackwardSstableIterator>,
+    staging_iters: Vec<
+        HummockIteratorUnion<
+            Backward,
+            SharedBufferBatchIterator<Backward>,
+            BackwardSstableIterator,
+        >,
+    >,
+}
+
+impl BackwardIteratorFactory {
+    pub fn build(
+        self,
+        mem_table: Option<MemTableHummockRevIterator<'_>>,
+    ) -> StorageRevIteratorPayloadInner<'_> {
+        // 3. build user_iterator
+        let staging_iter = StagingDataRevIterator::new(self.staging_iters);
+        MergeIterator::new(
+            once(HummockIteratorUnion::First(staging_iter))
+                .chain(
+                    self.overlapping_iters
+                        .into_iter()
+                        .map(HummockIteratorUnion::Second),
+                )
+                .chain(
+                    self.non_overlapping_iters
+                        .into_iter()
+                        .map(HummockIteratorUnion::Third),
+                )
+                .chain(mem_table.into_iter().map(HummockIteratorUnion::Fourth)),
+        )
+    }
+}
+
+impl IteratorFactory for BackwardIteratorFactory {
+    type Direction = Backward;
+    type SstableIteratorType = BackwardSstableIterator;
+
+    fn add_batch_iter(&mut self, batch: SharedBufferBatch) {
+        self.staging_iters
+            .push(HummockIteratorUnion::First(batch.into_backward_iter()));
+    }
+
+    fn add_staging_sst_iter(&mut self, iter: Self::SstableIteratorType) {
+        self.staging_iters.push(HummockIteratorUnion::Second(iter));
+    }
+
+    fn add_overlapping_sst_iter(&mut self, iter: Self::SstableIteratorType) {
+        self.overlapping_iters.push(iter);
+    }
+
+    fn add_concat_sst_iter(
+        &mut self,
+        mut tables: Vec<SstableInfo>,
+        sstable_store: SstableStoreRef,
+        read_options: Arc<SstableIteratorReadOptions>,
+    ) {
+        tables.reverse();
+        self.non_overlapping_iters
+            .push(ConcatIteratorInner::<Self::SstableIteratorType>::new(
+                tables,
+                sstable_store,
+                read_options,
+            ));
+    }
+}
diff --git a/src/storage/src/hummock/store/version.rs b/src/storage/src/hummock/store/version.rs
index 91b6394ed160a..9831f57448560 100644
--- a/src/storage/src/hummock/store/version.rs
+++ b/src/storage/src/hummock/store/version.rs
@@ -14,8 +14,7 @@
 
 use std::cmp::Ordering;
 use std::collections::vec_deque::VecDeque;
-use std::collections::HashSet;
-use std::iter::once;
+use std::collections::HashMap;
 use std::ops::Bound::Included;
 use std::sync::Arc;
 use std::time::Instant;
@@ -28,7 +27,6 @@ use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::util::epoch::MAX_SPILL_TIMES;
-use risingwave_hummock_sdk::change_log::EpochNewChangeLog;
 use risingwave_hummock_sdk::key::{
     bound_table_key_range, is_empty_key_range, FullKey, TableKey, TableKeyRange, UserKey,
 };
@@ -36,29 +34,33 @@ use risingwave_hummock_sdk::key_range::KeyRangeCommon;
 use risingwave_hummock_sdk::table_watermark::{
     TableWatermarksIndex, VnodeWatermark, WatermarkDirection,
 };
-use risingwave_hummock_sdk::version::HummockVersionDelta;
 use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, LocalSstableInfo};
-use risingwave_pb::hummock::{LevelType, SstableInfo};
+use risingwave_pb::hummock::{EpochNewChangeLog, LevelType, SstableInfo};
 use sync_point::sync_point;
+use tracing::warn;
 
-use super::StagingDataIterator;
 use crate::error::StorageResult;
+use crate::hummock::event_handler::LocalInstanceId;
+use crate::hummock::iterator::change_log::ChangeLogIterator;
 use crate::hummock::iterator::{
-    ChangeLogIterator, ConcatIterator, HummockIteratorUnion, MergeIterator, UserIterator,
+    BackwardUserIterator, IteratorFactory, MergeIterator, UserIterator,
 };
 use crate::hummock::local_version::pinned_version::PinnedVersion;
-use crate::hummock::sstable::SstableIteratorReadOptions;
+use crate::hummock::sstable::{SstableIteratorReadOptions, SstableIteratorType};
 use crate::hummock::sstable_store::SstableStoreRef;
 use crate::hummock::utils::{
-    check_subset_preserve_order, filter_single_sst, prune_nonoverlapping_ssts,
-    prune_overlapping_ssts, range_overlap, search_sst_idx,
+    filter_single_sst, prune_nonoverlapping_ssts, prune_overlapping_ssts, range_overlap,
+    search_sst_idx,
 };
 use crate::hummock::{
-    get_from_batch, get_from_sstable_info, hit_sstable_bloom_filter, HummockError, HummockResult,
-    HummockStorageIterator, HummockStorageIteratorInner, LocalHummockStorageIterator,
-    ReadVersionTuple, Sstable, SstableIterator,
+    get_from_batch, get_from_sstable_info, hit_sstable_bloom_filter, BackwardIteratorFactory,
+    ForwardIteratorFactory, HummockError, HummockResult, HummockStorageIterator,
+    HummockStorageIteratorInner, HummockStorageRevIteratorInner, ReadVersionTuple, Sstable,
+    SstableIterator,
+};
+use crate::mem_table::{
+    ImmId, ImmutableMemtable, MemTableHummockIterator, MemTableHummockRevIterator,
 };
-use crate::mem_table::{ImmId, ImmutableMemtable, MemTableHummockIterator};
 use crate::monitor::{
     GetLocalMetricsGuard, HummockStateStoreMetrics, MayExistLocalMetricsGuard, StoreLocalStatistic,
 };
@@ -75,24 +77,28 @@ pub type CommittedVersion = PinnedVersion;
 pub struct StagingSstableInfo {
     // newer data comes first
     sstable_infos: Vec<LocalSstableInfo>,
+    old_value_sstable_infos: Vec<LocalSstableInfo>,
     /// Epochs whose data are included in the Sstable. The newer epoch comes first.
     /// The field must not be empty.
     epochs: Vec<HummockEpoch>,
-    imm_ids: Vec<ImmId>,
+    // newer data at the front
+    imm_ids: HashMap<LocalInstanceId, Vec<ImmId>>,
     imm_size: usize,
 }
 
 impl StagingSstableInfo {
     pub fn new(
         sstable_infos: Vec<LocalSstableInfo>,
+        old_value_sstable_infos: Vec<LocalSstableInfo>,
         epochs: Vec<HummockEpoch>,
-        imm_ids: Vec<ImmId>,
+        imm_ids: HashMap<LocalInstanceId, Vec<ImmId>>,
         imm_size: usize,
     ) -> Self {
         // the epochs are sorted from higher epoch to lower epoch
-        assert!(epochs.is_sorted_by(|epoch1, epoch2| epoch2.partial_cmp(epoch1)));
+        assert!(epochs.is_sorted_by(|epoch1, epoch2| epoch2 <= epoch1));
         Self {
             sstable_infos,
+            old_value_sstable_infos,
             epochs,
             imm_ids,
             imm_size,
@@ -103,6 +109,10 @@ impl StagingSstableInfo {
         &self.sstable_infos
     }
 
+    pub fn old_value_sstable_infos(&self) -> &Vec<LocalSstableInfo> {
+        &self.old_value_sstable_infos
+    }
+
     pub fn imm_size(&self) -> usize {
         self.imm_size
     }
@@ -111,7 +121,7 @@ impl StagingSstableInfo {
         &self.epochs
     }
 
-    pub fn imm_ids(&self) -> &Vec<ImmId> {
+    pub fn imm_ids(&self) -> &HashMap<LocalInstanceId, Vec<ImmId>> {
         &self.imm_ids
     }
 }
@@ -119,14 +129,12 @@ impl StagingSstableInfo {
 #[derive(Clone)]
 pub enum StagingData {
     ImmMem(ImmutableMemtable),
-    MergedImmMem(ImmutableMemtable, Vec<ImmId>),
     Sst(Arc<StagingSstableInfo>),
 }
 
 pub enum VersionUpdate {
     /// a new staging data entry will be added.
     Staging(StagingData),
-    CommittedDelta(HummockVersionDelta),
     CommittedSnapshot(CommittedVersion),
     NewTableWatermark {
         direction: WatermarkDirection,
@@ -203,6 +211,7 @@ impl StagingVersion {
 /// A container of information required for reading from hummock.
 pub struct HummockReadVersion {
     table_id: TableId,
+    instance_id: LocalInstanceId,
 
     /// Local version for staging data.
     staging: StagingVersion,
@@ -226,6 +235,7 @@ pub struct HummockReadVersion {
 impl HummockReadVersion {
     pub fn new_with_replication_option(
         table_id: TableId,
+        instance_id: LocalInstanceId,
         committed_version: CommittedVersion,
         is_replicated: bool,
         vnodes: Arc<Bitmap>,
@@ -236,6 +246,7 @@ impl HummockReadVersion {
         assert!(committed_version.is_valid());
         Self {
             table_id,
+            instance_id,
             table_watermarks: committed_version
                 .version()
                 .table_watermarks
@@ -260,10 +271,11 @@ impl HummockReadVersion {
 
     pub fn new(
         table_id: TableId,
+        instance_id: LocalInstanceId,
         committed_version: CommittedVersion,
         vnodes: Arc<Bitmap>,
     ) -> Self {
-        Self::new_with_replication_option(table_id, committed_version, false, vnodes)
+        Self::new_with_replication_option(table_id, instance_id, committed_version, false, vnodes)
     }
 
     pub fn table_id(&self) -> TableId {
@@ -292,84 +304,44 @@ impl HummockReadVersion {
 
                     self.staging.imm.push_front(imm)
                 }
-                StagingData::MergedImmMem(merged_imm, imm_ids) => {
-                    self.add_merged_imm(merged_imm, imm_ids);
-                }
                 StagingData::Sst(staging_sst_ref) => {
-                    // The following properties must be ensured:
-                    // 1) self.staging.imm is sorted by imm id descendingly
-                    // 2) staging_sst.imm_ids preserves the imm id partial
-                    //    ordering of the participating read version imms. Example:
-                    //    If staging_sst contains two read versions r1: [i1, i3] and  r2: [i2, i4],
-                    //    then [i2, i1, i3, i4] is valid while [i3, i1, i2, i4] is invalid.
-                    // 3) The intersection between staging_sst.imm_ids and self.staging.imm
-                    //    are always the suffix of self.staging.imm
-
-                    // Check 1)
-                    debug_assert!(self
-                        .staging
-                        .imm
-                        .iter()
-                        .rev()
-                        .is_sorted_by_key(|imm| imm.batch_id()));
-
-                    // Calculate intersection
-                    let staging_imm_ids_from_imms: HashSet<u64> =
-                        self.staging.imm.iter().map(|imm| imm.batch_id()).collect();
-
-                    // intersected batch_id order from oldest to newest
-                    let intersect_imm_ids = staging_sst_ref
-                        .imm_ids
-                        .iter()
-                        .rev()
-                        .copied()
-                        .filter(|id| staging_imm_ids_from_imms.contains(id))
-                        .collect_vec();
-
-                    if !intersect_imm_ids.is_empty() {
-                        // Check 2)
-                        debug_assert!(check_subset_preserve_order(
-                            intersect_imm_ids.iter().copied(),
-                            self.staging.imm.iter().map(|imm| imm.batch_id()).rev(),
-                        ));
-
-                        // Check 3) and replace imms with a staging sst
-                        for imm_id in &intersect_imm_ids {
-                            if let Some(imm) = self.staging.imm.back() {
-                                if *imm_id == imm.batch_id() {
-                                    self.staging.imm.pop_back();
-                                }
-                            } else {
-                                let local_imm_ids = self
-                                    .staging
-                                    .imm
-                                    .iter()
-                                    .map(|imm| imm.batch_id())
-                                    .collect_vec();
-
-                                unreachable!(
-                                    "should not reach here staging_sst.size {},
+                    let Some(imms) = staging_sst_ref.imm_ids.get(&self.instance_id) else {
+                        warn!(
+                            instance_id = self.instance_id,
+                            "no related imm in sst input"
+                        );
+                        return;
+                    };
+
+                    // old data comes first
+                    for imm_id in imms.iter().rev() {
+                        let valid = match self.staging.imm.pop_back() {
+                            None => false,
+                            Some(prev_imm_id) => prev_imm_id.batch_id() == *imm_id,
+                        };
+                        assert!(
+                            valid,
+                            "should be valid staging_sst.size {},
                                     staging_sst.imm_ids {:?},
                                     staging_sst.epochs {:?},
                                     local_imm_ids {:?},
-                                    intersect_imm_ids {:?}",
-                                    staging_sst_ref.imm_size,
-                                    staging_sst_ref.imm_ids,
-                                    staging_sst_ref.epochs,
-                                    local_imm_ids,
-                                    intersect_imm_ids,
-                                );
-                            }
-                        }
-                        self.staging.sst.push_front(staging_sst_ref);
+                                    instance_id {}",
+                            staging_sst_ref.imm_size,
+                            staging_sst_ref.imm_ids,
+                            staging_sst_ref.epochs,
+                            self.staging
+                                .imm
+                                .iter()
+                                .map(|imm| imm.batch_id())
+                                .collect_vec(),
+                            self.instance_id,
+                        );
                     }
+
+                    self.staging.sst.push_front(staging_sst_ref);
                 }
             },
 
-            VersionUpdate::CommittedDelta(_) => {
-                unimplemented!()
-            }
-
             VersionUpdate::CommittedSnapshot(committed_version) => {
                 let max_committed_epoch = committed_version.max_committed_epoch();
                 self.committed = committed_version;
@@ -440,92 +412,6 @@ impl HummockReadVersion {
         }
     }
 
-    /// `imm_ids` is the list of imm ids that are merged into this batch
-    /// This field is immutable. Larger imm id at the front.
-    pub fn add_merged_imm(&mut self, merged_imm: ImmutableMemtable, imm_ids: Vec<ImmId>) {
-        assert!(imm_ids.iter().rev().is_sorted());
-        let min_imm_id = *imm_ids.last().expect("non-empty");
-
-        let back = self.staging.imm.back().expect("should not be empty");
-
-        // pop and save imms that are written earlier than the oldest imm if there is any
-        let earlier_imms = if back.batch_id() < min_imm_id {
-            let mut earlier_imms = VecDeque::with_capacity(self.staging.imm.len());
-            loop {
-                let batch_id = self
-                    .staging
-                    .imm
-                    .back()
-                    .expect("should not be empty")
-                    .batch_id();
-                match batch_id.cmp(&min_imm_id) {
-                    Ordering::Less => {
-                        let imm = self.staging.imm.pop_back().unwrap();
-                        earlier_imms.push_front(imm);
-                    }
-                    Ordering::Equal => {
-                        break;
-                    }
-                    Ordering::Greater => {
-                        let remaining_staging_imm_ids = self
-                            .staging
-                            .imm
-                            .iter()
-                            .map(|imm| imm.batch_id())
-                            .collect_vec();
-                        let earlier_imm_ids =
-                            earlier_imms.iter().map(|imm| imm.batch_id()).collect_vec();
-
-                        unreachable!(
-                            "must have break in equal: {:?} {:?} {:?}",
-                            remaining_staging_imm_ids, earlier_imm_ids, imm_ids
-                        )
-                    }
-                }
-            }
-            Some(earlier_imms)
-        } else {
-            assert_eq!(
-                back.batch_id(),
-                min_imm_id,
-                "{:?} {:?}",
-                {
-                    self.staging
-                        .imm
-                        .iter()
-                        .map(|imm| imm.batch_id())
-                        .collect_vec()
-                },
-                imm_ids
-            );
-            None
-        };
-
-        // iter from smaller imm and take the older imm at the back.
-        for imm_id in imm_ids.iter().rev() {
-            let imm = self.staging.imm.pop_back().expect("should exist");
-            assert_eq!(
-                imm.batch_id(),
-                *imm_id,
-                "{:?} {:?} {}",
-                {
-                    self.staging
-                        .imm
-                        .iter()
-                        .map(|imm| imm.batch_id())
-                        .collect_vec()
-                },
-                imm_ids,
-                imm_id,
-            );
-        }
-
-        self.staging.imm.push_back(merged_imm);
-        if let Some(earlier_imms) = earlier_imms {
-            self.staging.imm.extend(earlier_imms);
-        }
-    }
-
     pub fn is_replicated(&self) -> bool {
         self.is_replicated
     }
@@ -670,7 +556,7 @@ impl HummockVersionReader {
                 });
             }
         }
-
+        let single_table_key_range = table_key.clone()..=table_key.clone();
         // 3. read from committed_version sst file
         // Because SST meta records encoded key range,
         // the filter key needs to be encoded as well.
@@ -682,7 +568,6 @@ impl HummockVersionReader {
 
             match level.level_type() {
                 LevelType::Overlapping | LevelType::Unspecified => {
-                    let single_table_key_range = table_key.clone()..=table_key.clone();
                     let sstable_infos = prune_overlapping_ssts(
                         &level.table_infos,
                         read_options.table_id,
@@ -755,9 +640,9 @@ impl HummockVersionReader {
         table_key_range: TableKeyRange,
         epoch: u64,
         read_options: ReadOptions,
-        read_version_tuple: ReadVersionTuple,
+        read_version_tuple: (Vec<ImmutableMemtable>, Vec<SstableInfo>, CommittedVersion),
     ) -> StorageResult<HummockStorageIterator> {
-        self.iter_inner(
+        self.iter_with_memtable(
             table_key_range,
             epoch,
             read_options,
@@ -767,39 +652,114 @@ impl HummockVersionReader {
         .await
     }
 
-    pub async fn iter_with_memtable<'a>(
+    pub async fn iter_with_memtable<'a, 'b>(
         &'a self,
         table_key_range: TableKeyRange,
         epoch: u64,
         read_options: ReadOptions,
         read_version_tuple: (Vec<ImmutableMemtable>, Vec<SstableInfo>, CommittedVersion),
-        memtable_iter: MemTableHummockIterator<'a>,
-    ) -> StorageResult<LocalHummockStorageIterator<'_>> {
+        memtable_iter: Option<MemTableHummockIterator<'b>>,
+    ) -> StorageResult<HummockStorageIteratorInner<'b>> {
+        let user_key_range_ref = bound_table_key_range(read_options.table_id, &table_key_range);
+        let user_key_range = (
+            user_key_range_ref.0.map(|key| key.cloned()),
+            user_key_range_ref.1.map(|key| key.cloned()),
+        );
+        let mut factory = ForwardIteratorFactory::default();
+        let mut local_stats = StoreLocalStatistic::default();
+        let (imms, uncommitted_ssts, committed) = read_version_tuple;
+        let table_id = read_options.table_id;
+        let min_epoch = gen_min_epoch(epoch, read_options.retention_seconds.as_ref());
         self.iter_inner(
             table_key_range,
             epoch,
             read_options,
-            read_version_tuple,
-            Some(memtable_iter),
+            imms,
+            uncommitted_ssts,
+            &committed,
+            &mut local_stats,
+            &mut factory,
         )
-        .await
+        .await?;
+        let merge_iter = factory.build(memtable_iter);
+        // the epoch_range left bound for iterator read
+        let mut user_iter = UserIterator::new(
+            merge_iter,
+            user_key_range,
+            epoch,
+            min_epoch,
+            Some(committed),
+        );
+        user_iter.rewind().await?;
+        Ok(HummockStorageIteratorInner::new(
+            user_iter,
+            self.state_store_metrics.clone(),
+            table_id,
+            local_stats,
+        ))
     }
 
-    pub async fn iter_inner<'a, 'b>(
+    pub async fn rev_iter<'a, 'b>(
         &'a self,
         table_key_range: TableKeyRange,
         epoch: u64,
         read_options: ReadOptions,
-        read_version_tuple: ReadVersionTuple,
-        mem_table: Option<MemTableHummockIterator<'b>>,
-    ) -> StorageResult<HummockStorageIteratorInner<'b>> {
+        read_version_tuple: (Vec<ImmutableMemtable>, Vec<SstableInfo>, CommittedVersion),
+        memtable_iter: Option<MemTableHummockRevIterator<'b>>,
+    ) -> StorageResult<HummockStorageRevIteratorInner<'b>> {
+        let user_key_range_ref = bound_table_key_range(read_options.table_id, &table_key_range);
+        let user_key_range = (
+            user_key_range_ref.0.map(|key| key.cloned()),
+            user_key_range_ref.1.map(|key| key.cloned()),
+        );
+        let mut factory = BackwardIteratorFactory::default();
+        let mut local_stats = StoreLocalStatistic::default();
         let (imms, uncommitted_ssts, committed) = read_version_tuple;
+        let table_id = read_options.table_id;
+        let min_epoch = gen_min_epoch(epoch, read_options.retention_seconds.as_ref());
+        self.iter_inner(
+            table_key_range,
+            epoch,
+            read_options,
+            imms,
+            uncommitted_ssts,
+            &committed,
+            &mut local_stats,
+            &mut factory,
+        )
+        .await?;
+        let merge_iter = factory.build(memtable_iter);
+        // the epoch_range left bound for iterator read
+        let mut user_iter = BackwardUserIterator::new(
+            merge_iter,
+            user_key_range,
+            epoch,
+            min_epoch,
+            Some(committed),
+        );
+        user_iter.rewind().await?;
+        Ok(HummockStorageRevIteratorInner::new(
+            user_iter,
+            self.state_store_metrics.clone(),
+            table_id,
+            local_stats,
+        ))
+    }
 
-        let mut local_stats = StoreLocalStatistic::default();
-        let mut staging_iters = Vec::with_capacity(imms.len() + uncommitted_ssts.len());
+    pub async fn iter_inner<F: IteratorFactory>(
+        &self,
+        table_key_range: TableKeyRange,
+        epoch: u64,
+        read_options: ReadOptions,
+        imms: Vec<ImmutableMemtable>,
+        uncommitted_ssts: Vec<SstableInfo>,
+        committed: &CommittedVersion,
+        local_stats: &mut StoreLocalStatistic,
+        factory: &mut F,
+    ) -> StorageResult<()> {
         local_stats.staging_imm_iter_count = imms.len() as u64;
         for imm in imms {
-            staging_iters.push(HummockIteratorUnion::First(imm.into_forward_iter()));
+            factory.add_batch_iter(imm);
         }
 
         // 2. build iterator from committed
@@ -826,7 +786,7 @@ impl HummockVersionReader {
         for sstable_info in &uncommitted_ssts {
             let table_holder = self
                 .sstable_store
-                .sstable(sstable_info, &mut local_stats)
+                .sstable(sstable_info, local_stats)
                 .await?;
 
             if let Some(prefix_hash) = bloom_filter_prefix_hash.as_ref() {
@@ -834,24 +794,21 @@ impl HummockVersionReader {
                     &table_holder,
                     &user_key_range_ref,
                     *prefix_hash,
-                    &mut local_stats,
+                    local_stats,
                 ) {
                     continue;
                 }
             }
 
             staging_sst_iter_count += 1;
-            staging_iters.push(HummockIteratorUnion::Second(SstableIterator::new(
+            factory.add_staging_sst_iter(F::SstableIteratorType::create(
                 table_holder,
                 self.sstable_store.clone(),
                 sst_read_options.clone(),
-            )));
+            ));
         }
         local_stats.staging_sst_iter_count = staging_sst_iter_count;
-        let staging_iter: StagingDataIterator = MergeIterator::new(staging_iters);
 
-        let mut non_overlapping_iters = Vec::new();
-        let mut overlapping_iters = Vec::new();
         let timer = Instant::now();
 
         for level in committed.levels(read_options.table_id) {
@@ -874,16 +831,16 @@ impl HummockVersionReader {
                     continue;
                 }
                 if sstables.len() > 1 {
-                    non_overlapping_iters.push(ConcatIterator::new(
+                    factory.add_concat_sst_iter(
                         sstables,
                         self.sstable_store.clone(),
                         sst_read_options.clone(),
-                    ));
+                    );
                     local_stats.non_overlapping_iter_count += 1;
                 } else {
                     let sstable = self
                         .sstable_store
-                        .sstable(&sstables[0], &mut local_stats)
+                        .sstable(&sstables[0], local_stats)
                         .await?;
 
                     if let Some(dist_hash) = bloom_filter_prefix_hash.as_ref() {
@@ -891,7 +848,7 @@ impl HummockVersionReader {
                             &sstable,
                             &user_key_range_ref,
                             *dist_hash,
-                            &mut local_stats,
+                            local_stats,
                         ) {
                             continue;
                         }
@@ -901,7 +858,7 @@ impl HummockVersionReader {
                     // We put the SstableIterator in `overlapping_iters` just for convenience since
                     // it overlaps with SSTs in other levels. In metrics reporting, we still count
                     // it in `non_overlapping_iter_count`.
-                    overlapping_iters.push(SstableIterator::new(
+                    factory.add_overlapping_sst_iter(F::SstableIteratorType::create(
                         sstable,
                         self.sstable_store.clone(),
                         sst_read_options.clone(),
@@ -922,7 +879,7 @@ impl HummockVersionReader {
                 for sstable_info in fetch_meta_req {
                     let sstable = self
                         .sstable_store
-                        .sstable(sstable_info, &mut local_stats)
+                        .sstable(sstable_info, local_stats)
                         .await?;
                     assert_eq!(sstable_info.get_object_id(), sstable.id);
                     if let Some(dist_hash) = bloom_filter_prefix_hash.as_ref() {
@@ -930,12 +887,12 @@ impl HummockVersionReader {
                             &sstable,
                             &user_key_range_ref,
                             *dist_hash,
-                            &mut local_stats,
+                            local_stats,
                         ) {
                             continue;
                         }
                     }
-                    overlapping_iters.push(SstableIterator::new(
+                    factory.add_overlapping_sst_iter(F::SstableIteratorType::create(
                         sstable,
                         self.sstable_store.clone(),
                         sst_read_options.clone(),
@@ -953,50 +910,7 @@ impl HummockVersionReader {
                 .iter_slow_fetch_meta_cache_unhits
                 .set(local_stats.cache_meta_block_miss as i64);
         }
-
-        // 3. build user_iterator
-        let merge_iter = MergeIterator::new(
-            once(HummockIteratorUnion::First(staging_iter))
-                .chain(
-                    overlapping_iters
-                        .into_iter()
-                        .map(HummockIteratorUnion::Second),
-                )
-                .chain(
-                    non_overlapping_iters
-                        .into_iter()
-                        .map(HummockIteratorUnion::Third),
-                )
-                .chain(mem_table.into_iter().map(HummockIteratorUnion::Fourth)),
-        );
-
-        let user_key_range = (
-            user_key_range.0.map(|key| key.cloned()),
-            user_key_range.1.map(|key| key.cloned()),
-        );
-
-        // the epoch_range left bound for iterator read
-        let min_epoch = gen_min_epoch(epoch, read_options.retention_seconds.as_ref());
-        let mut user_iter = UserIterator::new(
-            merge_iter,
-            user_key_range,
-            epoch,
-            min_epoch,
-            Some(committed),
-        );
-        user_iter.rewind().await?;
-        local_stats.found_key = user_iter.is_valid();
-        local_stats.sub_iter_count = local_stats.staging_imm_iter_count
-            + local_stats.staging_sst_iter_count
-            + local_stats.overlapping_iter_count
-            + local_stats.non_overlapping_iter_count;
-
-        Ok(HummockStorageIteratorInner::new(
-            user_iter,
-            self.state_store_metrics.clone(),
-            read_options.table_id,
-            local_stats,
-        ))
+        Ok(())
     }
 
     // Note: this method will not check the kv tomestones and delete range tomestones
@@ -1170,7 +1084,6 @@ impl HummockVersionReader {
             .await?;
             Ok::<_, HummockError>(MergeIterator::new(iters))
         }
-
         let new_value_iter = make_iter(
             change_log
                 .iter()
diff --git a/src/storage/src/hummock/test_utils.rs b/src/storage/src/hummock/test_utils.rs
index 872e0877ac4a5..cfed5f0eb7824 100644
--- a/src/storage/src/hummock/test_utils.rs
+++ b/src/storage/src/hummock/test_utils.rs
@@ -17,7 +17,10 @@ use std::ops::Bound;
 use std::sync::Arc;
 
 use bytes::Bytes;
-use foyer::memory::CacheContext;
+use foyer::{
+    CacheContext, HybridCache, HybridCacheBuilder, StorageKey as HybridKey,
+    StorageValue as HybridValue,
+};
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
 use risingwave_common::config::EvictionConfig;
@@ -34,7 +37,9 @@ use super::{
 };
 use crate::filter_key_extractor::{FilterKeyExtractorImpl, FullKeyFilterKeyExtractor};
 use crate::hummock::iterator::ForwardMergeRangeIterator;
-use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferBatch;
+use crate::hummock::shared_buffer::shared_buffer_batch::{
+    SharedBufferBatch, SharedBufferItem, SharedBufferValue,
+};
 use crate::hummock::value::HummockValue;
 use crate::hummock::{
     BlockedXor16FilterBuilder, CachePolicy, CompactionDeleteRangeIterator, DeleteRangeTombstone,
@@ -67,10 +72,10 @@ pub fn default_opts_for_test() -> StorageOpts {
     }
 }
 
-pub fn gen_dummy_batch(n: u64) -> Vec<(TableKey<Bytes>, StorageValue)> {
+pub fn gen_dummy_batch(n: u64) -> Vec<SharedBufferItem> {
     vec![(
         TableKey(Bytes::from(iterator_test_table_key_of(n as usize))),
-        StorageValue::new_put(b"value1".to_vec()),
+        SharedBufferValue::Insert(Bytes::copy_from_slice(&b"value1"[..])),
     )]
 }
 
@@ -354,6 +359,19 @@ pub fn create_small_table_cache() -> Arc<LruCache<HummockSstableObjectId, Box<Ss
     Arc::new(LruCache::new(1, 4, 0))
 }
 
+pub async fn hybrid_cache_for_test<K, V>() -> HybridCache<K, V>
+where
+    K: HybridKey,
+    V: HybridValue,
+{
+    HybridCacheBuilder::new()
+        .memory(10)
+        .storage()
+        .build()
+        .await
+        .unwrap()
+}
+
 pub mod delete_range {
     use super::*;
     use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferDeleteRangeIterator;
@@ -473,26 +491,37 @@ pub mod delete_range {
     /// overlaps among range tombstones among different SSTs/batchs in compaction.
     /// The core idea contains two parts:
     /// 1) we only need to keep the smallest epoch of the overlapping
-    /// range tomstone intervals since the key covered by the range tombstone in lower level must have
-    /// smaller epoches;
+    ///    range tomstone intervals since the key covered by the range tombstone in lower level must have
+    ///    smaller epoches;
     /// 2) due to 1), we lose the information to delete a key by tombstone in a single
-    /// SST so we add a tombstone key in the data block.
-    /// We leverage `events` to calculate the epoch information mentioned above.
+    ///    SST so we add a tombstone key in the data block.
+    ///    We leverage `events` to calculate the epoch information mentioned above.
+    ///
     /// e.g. Delete range [1, 5) at epoch1, delete range [3, 7) at epoch2 and delete range [10, 12) at
     /// epoch3 will first be transformed into `events` below:
+    ///
     /// `<1, +epoch1> <5, -epoch1> <3, +epoch2> <7, -epoch2> <10, +epoch3> <12, -epoch3>`
+    ///
     /// Then `events` are sorted by user key:
+    ///
     /// `<1, +epoch1> <3, +epoch2> <5, -epoch1> <7, -epoch2> <10, +epoch3> <12, -epoch3>`
+    ///
     /// We rely on the fact that keys met in compaction are in order.
+    ///
     /// When user key 0 comes, no events have happened yet so no range delete epoch. (will be
     /// represented as range delete epoch MAX EPOCH)
+    ///
     /// When user key 1 comes, event `<1, +epoch1>` happens so there is currently one range delete
     /// epoch: epoch1.
+    ///
     /// When user key 2 comes, no more events happen so the set remains `{epoch1}`.
+    ///
     /// When user key 3 comes, event `<3, +epoch2>` happens so the range delete epoch set is now
     /// `{epoch1, epoch2}`.
+    ///
     /// When user key 5 comes, event `<5, -epoch1>` happens so epoch1 exits the set,
     /// therefore the current range delete epoch set is `{epoch2}`.
+    ///
     /// When user key 11 comes, events `<7, -epoch2>` and `<10, +epoch3>`
     /// both happen, one after another. The set changes to `{epoch3}` from `{epoch2}`.
     pub fn apply_event(epochs: &mut BTreeSet<HummockEpoch>, event: &CompactionDeleteRangeEvent) {
diff --git a/src/storage/src/hummock/utils.rs b/src/storage/src/hummock/utils.rs
index 82af4beadf761..4c270ee736b97 100644
--- a/src/storage/src/hummock/utils.rs
+++ b/src/storage/src/hummock/utils.rs
@@ -13,15 +13,17 @@
 // limitations under the License.
 
 use std::cmp::Ordering;
+use std::collections::VecDeque;
 use std::fmt::{Debug, Formatter};
 use std::ops::Bound::{Excluded, Included, Unbounded};
 use std::ops::{Bound, RangeBounds};
-use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering as AtomicOrdering};
 use std::sync::Arc;
 use std::time::Duration;
 
 use bytes::Bytes;
-use foyer::memory::CacheContext;
+use foyer::CacheContext;
+use parking_lot::Mutex;
 use risingwave_common::catalog::{TableId, TableOption};
 use risingwave_hummock_sdk::key::{
     bound_table_key_range, EmptySliceRef, FullKey, TableKey, UserKey,
@@ -29,8 +31,7 @@ use risingwave_hummock_sdk::key::{
 use risingwave_hummock_sdk::version::HummockVersion;
 use risingwave_hummock_sdk::{can_concat, HummockEpoch};
 use risingwave_pb::hummock::SstableInfo;
-use tokio::sync::watch::Sender;
-use tokio::sync::Notify;
+use tokio::sync::oneshot::{channel, Receiver, Sender};
 
 use super::{HummockError, HummockResult};
 use crate::error::StorageResult;
@@ -69,9 +70,19 @@ where
     !too_left && !too_right
 }
 
-pub fn validate_safe_epoch(safe_epoch: u64, epoch: u64) -> HummockResult<()> {
-    if epoch < safe_epoch {
-        return Err(HummockError::expired_epoch(safe_epoch, epoch));
+pub fn validate_safe_epoch(
+    version: &HummockVersion,
+    table_id: TableId,
+    epoch: u64,
+) -> HummockResult<()> {
+    if let Some(info) = version.state_table_info.info().get(&table_id)
+        && epoch < info.safe_epoch
+    {
+        return Err(HummockError::expired_epoch(
+            table_id,
+            info.safe_epoch,
+            epoch,
+        ));
     }
 
     Ok(())
@@ -166,23 +177,54 @@ pub fn prune_nonoverlapping_ssts<'a>(
     ssts[start_table_idx..=end_table_idx].iter()
 }
 
-#[derive(Debug)]
+type RequestQueue = VecDeque<(Sender<MemoryTracker>, u64)>;
+enum MemoryRequest {
+    Ready(MemoryTracker),
+    Pending(Receiver<MemoryTracker>),
+}
+
 struct MemoryLimiterInner {
     total_size: AtomicU64,
-    notify: Notify,
+    controller: Mutex<RequestQueue>,
+    has_waiter: AtomicBool,
     quota: u64,
 }
 
 impl MemoryLimiterInner {
     fn release_quota(&self, quota: u64) {
-        self.total_size.fetch_sub(quota, AtomicOrdering::Release);
-        self.notify.notify_waiters();
+        self.total_size.fetch_sub(quota, AtomicOrdering::SeqCst);
     }
 
     fn add_memory(&self, quota: u64) {
         self.total_size.fetch_add(quota, AtomicOrdering::SeqCst);
     }
 
+    fn may_notify_waiters(self: &Arc<Self>) {
+        // check `has_waiter` to avoid access lock every times drop `MemoryTracker`.
+        if !self.has_waiter.load(AtomicOrdering::Acquire) {
+            return;
+        }
+        let mut notify_waiters = vec![];
+        {
+            let mut waiters = self.controller.lock();
+            while let Some((_, quota)) = waiters.front() {
+                if !self.try_require_memory(*quota) {
+                    break;
+                }
+                let (tx, quota) = waiters.pop_front().unwrap();
+                notify_waiters.push((tx, quota));
+            }
+
+            if waiters.is_empty() {
+                self.has_waiter.store(false, AtomicOrdering::Release);
+            }
+        }
+
+        for (tx, quota) in notify_waiters {
+            let _ = tx.send(MemoryTracker::new(self.clone(), quota));
+        }
+    }
+
     fn try_require_memory(&self, quota: u64) -> bool {
         let mut current_quota = self.total_size.load(AtomicOrdering::Acquire);
         while self.permit_quota(current_quota, quota) {
@@ -203,44 +245,23 @@ impl MemoryLimiterInner {
         false
     }
 
-    async fn require_memory(&self, quota: u64) {
-        let current_quota = self.total_size.load(AtomicOrdering::Acquire);
-        if self.permit_quota(current_quota, quota)
-            && self
-                .total_size
-                .compare_exchange(
-                    current_quota,
-                    current_quota + quota,
-                    AtomicOrdering::SeqCst,
-                    AtomicOrdering::SeqCst,
-                )
-                .is_ok()
-        {
-            // fast path.
-            return;
+    fn require_memory(self: &Arc<Self>, quota: u64) -> MemoryRequest {
+        let mut waiters = self.controller.lock();
+        let first_req = waiters.is_empty();
+        if first_req {
+            // When this request is the first waiter but the previous `MemoryTracker` is just release a large quota, it may skip notifying this waiter because it has checked `has_waiter` and found it was false. So we must set it one and retry `try_require_memory` again to avoid deadlock.
+            self.has_waiter.store(true, AtomicOrdering::Release);
         }
-        loop {
-            let notified = self.notify.notified();
-            let current_quota = self.total_size.load(AtomicOrdering::Acquire);
-            if self.permit_quota(current_quota, quota) {
-                match self.total_size.compare_exchange(
-                    current_quota,
-                    current_quota + quota,
-                    AtomicOrdering::SeqCst,
-                    AtomicOrdering::SeqCst,
-                ) {
-                    Ok(_) => break,
-                    Err(old_quota) => {
-                        // The quota is enough but just changed by other threads. So just try to
-                        // update again without waiting notify.
-                        if self.permit_quota(old_quota, quota) {
-                            continue;
-                        }
-                    }
-                }
+        // We must require again with lock because some other MemoryTracker may drop just after this thread gets mutex but before it enters queue.
+        if self.try_require_memory(quota) {
+            if first_req {
+                self.has_waiter.store(false, AtomicOrdering::Release);
             }
-            notified.await;
+            return MemoryRequest::Ready(MemoryTracker::new(self.clone(), quota));
         }
+        let (tx, rx) = channel();
+        waiters.push_back((tx, quota));
+        MemoryRequest::Pending(rx)
     }
 
     fn permit_quota(&self, current_quota: u64, _request_quota: u64) -> bool {
@@ -248,38 +269,51 @@ impl MemoryLimiterInner {
     }
 }
 
-#[derive(Debug)]
 pub struct MemoryLimiter {
     inner: Arc<MemoryLimiterInner>,
 }
 
+impl Debug for MemoryLimiter {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("MemoryLimiter")
+            .field("quota", &self.inner.quota)
+            .field("usage", &self.inner.total_size)
+            .finish()
+    }
+}
+
 pub struct MemoryTracker {
     limiter: Arc<MemoryLimiterInner>,
-    quota: u64,
+    quota: Option<u64>,
+}
+impl MemoryTracker {
+    fn new(limiter: Arc<MemoryLimiterInner>, quota: u64) -> Self {
+        Self {
+            limiter,
+            quota: Some(quota),
+        }
+    }
 }
 
 impl Debug for MemoryTracker {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("quota").field("quota", &self.quota).finish()
+        f.debug_struct("MemoryTracker")
+            .field("quota", &self.quota)
+            .finish()
     }
 }
 
 impl MemoryLimiter {
     pub fn unlimit() -> Arc<Self> {
-        Arc::new(Self {
-            inner: Arc::new(MemoryLimiterInner {
-                total_size: AtomicU64::new(0),
-                notify: Notify::new(),
-                quota: u64::MAX - 1,
-            }),
-        })
+        Arc::new(Self::new(u64::MAX))
     }
 
     pub fn new(quota: u64) -> Self {
         Self {
             inner: Arc::new(MemoryLimiterInner {
                 total_size: AtomicU64::new(0),
-                notify: Notify::new(),
+                controller: Mutex::new(VecDeque::default()),
+                has_waiter: AtomicBool::new(false),
                 quota,
             }),
         }
@@ -287,10 +321,7 @@ impl MemoryLimiter {
 
     pub fn try_require_memory(&self, quota: u64) -> Option<MemoryTracker> {
         if self.inner.try_require_memory(quota) {
-            Some(MemoryTracker {
-                limiter: self.inner.clone(),
-                quota,
-            })
+            Some(MemoryTracker::new(self.inner.clone(), quota))
         } else {
             None
         }
@@ -309,32 +340,27 @@ impl MemoryLimiter {
             self.inner.add_memory(quota);
         }
 
-        MemoryTracker {
-            limiter: self.inner.clone(),
-            quota,
-        }
+        MemoryTracker::new(self.inner.clone(), quota)
     }
 }
 
 impl MemoryLimiter {
     pub async fn require_memory(&self, quota: u64) -> MemoryTracker {
-        // Since the over provision limiter gets blocked only when the current usage exceeds the
-        // memory quota, it is allowed to apply for more than the memory quota.
-        self.inner.require_memory(quota).await;
-        MemoryTracker {
-            limiter: self.inner.clone(),
-            quota,
+        match self.inner.require_memory(quota) {
+            MemoryRequest::Ready(tracker) => tracker,
+            MemoryRequest::Pending(rx) => rx.await.unwrap(),
         }
     }
 }
 
 impl MemoryTracker {
     pub fn try_increase_memory(&mut self, target: u64) -> bool {
-        if self.quota >= target {
+        let quota = self.quota.unwrap();
+        if quota >= target {
             return true;
         }
-        if self.limiter.try_require_memory(target - self.quota) {
-            self.quota = target;
+        if self.limiter.try_require_memory(target - quota) {
+            self.quota = Some(target);
             true
         } else {
             false
@@ -342,9 +368,13 @@ impl MemoryTracker {
     }
 }
 
+// We must notify waiters outside `MemoryTracker` to avoid dead-lock and loop-owner.
 impl Drop for MemoryTracker {
     fn drop(&mut self) {
-        self.limiter.release_quota(self.quota);
+        if let Some(quota) = self.quota.take() {
+            self.limiter.release_quota(quota);
+            self.limiter.may_notify_waiters();
+        }
     }
 }
 
@@ -369,7 +399,18 @@ pub fn check_subset_preserve_order<T: Eq>(
     true
 }
 
-pub(crate) const ENABLE_SANITY_CHECK: bool = cfg!(debug_assertions);
+static SANITY_CHECK_ENABLED: AtomicBool = AtomicBool::new(cfg!(debug_assertions));
+
+/// This function is intended to be called during compute node initialization if the storage
+/// sanity check is not desired. This controls a global flag so only need to be called once
+/// if need to disable the sanity check.
+pub fn disable_sanity_check() {
+    SANITY_CHECK_ENABLED.store(false, AtomicOrdering::Release);
+}
+
+pub(crate) fn sanity_check_enabled() -> bool {
+    SANITY_CHECK_ENABLED.load(AtomicOrdering::Acquire)
+}
 
 /// Make sure the key to insert should not exist in storage.
 pub(crate) async fn do_insert_sanity_check(
@@ -413,7 +454,11 @@ pub(crate) async fn do_delete_sanity_check(
     table_option: TableOption,
     op_consistency_level: &OpConsistencyLevel,
 ) -> StorageResult<()> {
-    let OpConsistencyLevel::ConsistentOldValue(old_value_checker) = op_consistency_level else {
+    let OpConsistencyLevel::ConsistentOldValue {
+        check_old_value: old_value_checker,
+        ..
+    } = op_consistency_level
+    else {
         return Ok(());
     };
     let read_options = ReadOptions {
@@ -455,7 +500,11 @@ pub(crate) async fn do_update_sanity_check(
     table_option: TableOption,
     op_consistency_level: &OpConsistencyLevel,
 ) -> StorageResult<()> {
-    let OpConsistencyLevel::ConsistentOldValue(old_value_checker) = op_consistency_level else {
+    let OpConsistencyLevel::ConsistentOldValue {
+        check_old_value: old_value_checker,
+        ..
+    } = op_consistency_level
+    else {
         return Ok(());
     };
     let read_options = ReadOptions {
@@ -568,7 +617,7 @@ pub(crate) fn filter_with_delete_range<'a>(
 }
 
 pub(crate) async fn wait_for_epoch(
-    notifier: &Sender<HummockEpoch>,
+    notifier: &tokio::sync::watch::Sender<HummockEpoch>,
     wait_epoch: u64,
 ) -> StorageResult<()> {
     let mut receiver = notifier.subscribe();
@@ -611,9 +660,12 @@ pub(crate) async fn wait_for_epoch(
 #[cfg(test)]
 mod tests {
     use std::future::{poll_fn, Future};
+    use std::sync::Arc;
     use std::task::Poll;
 
+    use futures::future::join_all;
     use futures::FutureExt;
+    use rand::random;
 
     use crate::hummock::utils::MemoryLimiter;
 
@@ -645,4 +697,42 @@ mod tests {
         drop(tracker3);
         assert_eq!(0, memory_limiter.get_memory_usage());
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
+    async fn test_multi_thread_acquire_memory() {
+        const QUOTA: u64 = 10;
+        let memory_limiter = Arc::new(MemoryLimiter::new(200));
+        let mut handles = vec![];
+        for _ in 0..40 {
+            let limiter = memory_limiter.clone();
+            let h = tokio::spawn(async move {
+                let mut buffers = vec![];
+                let mut current_buffer_usage = (random::<usize>() % 8) + 2;
+                for _ in 0..1000 {
+                    if buffers.len() < current_buffer_usage
+                        && let Some(tracker) = limiter.try_require_memory(QUOTA)
+                    {
+                        buffers.push(tracker);
+                    } else {
+                        buffers.clear();
+                        current_buffer_usage = (random::<usize>() % 8) + 2;
+                        let req = limiter.require_memory(QUOTA);
+                        match tokio::time::timeout(std::time::Duration::from_millis(1), req).await {
+                            Ok(tracker) => {
+                                buffers.push(tracker);
+                            }
+                            Err(_) => {
+                                continue;
+                            }
+                        }
+                    }
+                    let sleep_time = random::<u64>() % 3 + 1;
+                    tokio::time::sleep(std::time::Duration::from_millis(sleep_time)).await;
+                }
+            });
+            handles.push(h);
+        }
+        let h = join_all(handles);
+        let _ = h.await;
+    }
 }
diff --git a/src/storage/src/hummock/value.rs b/src/storage/src/hummock/value.rs
index 7a1987ad81b31..0572a5c164f6f 100644
--- a/src/storage/src/hummock/value.rs
+++ b/src/storage/src/hummock/value.rs
@@ -17,7 +17,6 @@ use std::fmt::Debug;
 use bytes::{Buf, BufMut, Bytes};
 
 use super::{HummockError, HummockResult};
-use crate::storage_value::StorageValue;
 
 pub const VALUE_DELETE: u8 = 1 << 0;
 pub const VALUE_PUT: u8 = 0;
@@ -45,6 +44,7 @@ impl<T: AsRef<[u8]>> Debug for HummockValue<T> {
 
 impl<T> Copy for HummockValue<T> where T: Copy {}
 
+#[cfg(any(test, feature = "test"))]
 impl<T: PartialEq> PartialEq for HummockValue<T> {
     fn eq(&self, other: &Self) -> bool {
         match (self, other) {
@@ -55,11 +55,12 @@ impl<T: PartialEq> PartialEq for HummockValue<T> {
     }
 }
 
+#[cfg(any(test, feature = "test"))]
 impl<T: Eq> Eq for HummockValue<T> {}
 
 impl<T> HummockValue<T>
 where
-    T: PartialEq + Eq + AsRef<[u8]>,
+    T: AsRef<[u8]>,
 {
     pub fn encoded_len(&self) -> usize {
         match self {
@@ -168,16 +169,6 @@ impl From<HummockValue<Vec<u8>>> for HummockValue<Bytes> {
     }
 }
 
-impl From<StorageValue> for HummockValue<Bytes> {
-    fn from(data: StorageValue) -> Self {
-        if data.is_some() {
-            HummockValue::Put(data.user_value.unwrap_or_default())
-        } else {
-            HummockValue::Delete
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/src/storage/src/lib.rs b/src/storage/src/lib.rs
index 505eec276fbf4..21c0c7f49ae4c 100644
--- a/src/storage/src/lib.rs
+++ b/src/storage/src/lib.rs
@@ -14,7 +14,6 @@
 
 #![feature(allocator_api)]
 #![feature(bound_as_ref)]
-#![feature(bound_map)]
 #![feature(custom_test_frameworks)]
 #![feature(extract_if)]
 #![feature(coroutines)]
@@ -37,7 +36,6 @@
 #![recursion_limit = "256"]
 #![feature(error_generic_member_access)]
 #![feature(let_chains)]
-#![feature(associated_type_bounds)]
 #![feature(exclusive_range_pattern)]
 #![feature(impl_trait_in_assoc_type)]
 #![feature(maybe_uninit_uninit_array)]
diff --git a/src/storage/src/mem_table.rs b/src/storage/src/mem_table.rs
index 48000ec702310..8562991943c1a 100644
--- a/src/storage/src/mem_table.rs
+++ b/src/storage/src/mem_table.rs
@@ -35,10 +35,10 @@ use thiserror_ext::AsReport;
 use tracing::error;
 
 use crate::error::{StorageError, StorageResult};
-use crate::hummock::iterator::{FromRustIterator, RustIteratorBuilder};
+use crate::hummock::iterator::{Backward, Forward, FromRustIterator, RustIteratorBuilder};
 use crate::hummock::shared_buffer::shared_buffer_batch::{SharedBufferBatch, SharedBufferBatchId};
 use crate::hummock::utils::{
-    do_delete_sanity_check, do_insert_sanity_check, do_update_sanity_check, ENABLE_SANITY_CHECK,
+    do_delete_sanity_check, do_insert_sanity_check, do_update_sanity_check, sanity_check_enabled,
 };
 use crate::hummock::value::HummockValue;
 use crate::row_serde::value_serde::ValueRowSerde;
@@ -78,6 +78,7 @@ type Result<T> = std::result::Result<T, Box<MemTableError>>;
 
 pub type MemTableStore = BTreeMap<TableKey<Bytes>, KeyOp>;
 pub struct MemTableIteratorBuilder;
+pub struct MemTableRevIteratorBuilder;
 
 fn map_to_hummock_value<'a>(
     (key, op): (&'a TableKey<Bytes>, &'a KeyOp),
@@ -91,7 +92,29 @@ fn map_to_hummock_value<'a>(
     )
 }
 
+impl RustIteratorBuilder for MemTableRevIteratorBuilder {
+    type Direction = Backward;
+    type Iterable = MemTableStore;
+
+    type RewindIter<'a> =
+        impl Iterator<Item = (TableKey<&'a [u8]>, HummockValue<&'a [u8]>)> + Send + 'a;
+    type SeekIter<'a> =
+        impl Iterator<Item = (TableKey<&'a [u8]>, HummockValue<&'a [u8]>)> + Send + 'a;
+
+    fn seek<'a>(iterable: &'a Self::Iterable, seek_key: TableKey<&[u8]>) -> Self::SeekIter<'a> {
+        iterable
+            .range::<[u8], _>((Unbounded, Included(seek_key.0)))
+            .rev()
+            .map(map_to_hummock_value)
+    }
+
+    fn rewind(iterable: &Self::Iterable) -> Self::RewindIter<'_> {
+        iterable.iter().rev().map(map_to_hummock_value)
+    }
+}
+
 impl RustIteratorBuilder for MemTableIteratorBuilder {
+    type Direction = Forward;
     type Iterable = MemTableStore;
 
     type RewindIter<'a> =
@@ -111,6 +134,7 @@ impl RustIteratorBuilder for MemTableIteratorBuilder {
 }
 
 pub type MemTableHummockIterator<'a> = FromRustIterator<'a, MemTableIteratorBuilder>;
+pub type MemTableHummockRevIterator<'a> = FromRustIterator<'a, MemTableRevIteratorBuilder>;
 
 impl MemTable {
     pub fn new(op_consistency_level: OpConsistencyLevel) -> Self {
@@ -134,39 +158,50 @@ impl MemTable {
     pub fn insert(&mut self, pk: TableKey<Bytes>, value: Bytes) -> Result<()> {
         if let OpConsistencyLevel::Inconsistent = &self.op_consistency_level {
             let key_len = std::mem::size_of::<Bytes>() + pk.len();
-            let insert_value = KeyOp::Insert(value);
-            self.kv_size.add(&pk, &insert_value);
-            let origin_value = self.buffer.insert(pk, insert_value);
-            self.sub_origin_size(origin_value, key_len);
-
+            let op = KeyOp::Insert(value);
+            self.kv_size.add(&pk, &op);
+            let old_op = self.buffer.insert(pk, op);
+            self.sub_old_op_size(old_op, key_len);
             return Ok(());
         };
         let entry = self.buffer.entry(pk);
         match entry {
             Entry::Vacant(e) => {
-                let insert_value = KeyOp::Insert(value);
-                self.kv_size.add(e.key(), &insert_value);
-                e.insert(insert_value);
+                let op = KeyOp::Insert(value);
+                self.kv_size.add(e.key(), &op);
+                e.insert(op);
                 Ok(())
             }
             Entry::Occupied(mut e) => {
-                let origin_value = e.get_mut();
-                self.kv_size.sub_val(origin_value);
-                match origin_value {
-                    KeyOp::Delete(ref mut old_value) => {
-                        let old_val = std::mem::take(old_value);
-                        let update_value = KeyOp::Update((old_val, value));
-                        self.kv_size.add_val(&update_value);
-                        e.insert(update_value);
+                let old_op = e.get_mut();
+                self.kv_size.sub_val(old_op);
+
+                match old_op {
+                    KeyOp::Delete(ref mut old_op_old_value) => {
+                        let new_op = KeyOp::Update((std::mem::take(old_op_old_value), value));
+                        self.kv_size.add_val(&new_op);
+                        e.insert(new_op);
                         Ok(())
                     }
                     KeyOp::Insert(_) | KeyOp::Update(_) => {
-                        Err(MemTableError::InconsistentOperation {
+                        let new_op = KeyOp::Insert(value);
+                        let err = MemTableError::InconsistentOperation {
                             key: e.key().clone(),
                             prev: e.get().clone(),
-                            new: KeyOp::Insert(value),
+                            new: new_op.clone(),
+                        };
+
+                        if sanity_check_enabled() {
+                            Err(err.into())
+                        } else {
+                            tracing::error!(
+                                error = %err.as_report(),
+                                "double insert / insert on updated, ignoring because sanity check is disabled"
+                            );
+                            self.kv_size.add_val(&new_op);
+                            e.insert(new_op);
+                            Ok(())
                         }
-                        .into())
                     }
                 }
             }
@@ -175,28 +210,32 @@ impl MemTable {
 
     pub fn delete(&mut self, pk: TableKey<Bytes>, old_value: Bytes) -> Result<()> {
         let key_len = std::mem::size_of::<Bytes>() + pk.len();
-        let OpConsistencyLevel::ConsistentOldValue(value_checker) = &self.op_consistency_level
+        let OpConsistencyLevel::ConsistentOldValue {
+            check_old_value: value_checker,
+            ..
+        } = &self.op_consistency_level
         else {
-            let delete_value = KeyOp::Delete(old_value);
-            self.kv_size.add(&pk, &delete_value);
-            let origin_value = self.buffer.insert(pk, delete_value);
-            self.sub_origin_size(origin_value, key_len);
+            let op = KeyOp::Delete(old_value);
+            self.kv_size.add(&pk, &op);
+            let old_op = self.buffer.insert(pk, op);
+            self.sub_old_op_size(old_op, key_len);
             return Ok(());
         };
         let entry = self.buffer.entry(pk);
         match entry {
             Entry::Vacant(e) => {
-                let delete_value = KeyOp::Delete(old_value);
-                self.kv_size.add(e.key(), &delete_value);
-                e.insert(delete_value);
+                let op = KeyOp::Delete(old_value);
+                self.kv_size.add(e.key(), &op);
+                e.insert(op);
                 Ok(())
             }
             Entry::Occupied(mut e) => {
-                let origin_value = e.get_mut();
-                self.kv_size.sub_val(origin_value);
-                match origin_value {
-                    KeyOp::Insert(original_value) => {
-                        if ENABLE_SANITY_CHECK && !value_checker(original_value, &old_value) {
+                let old_op = e.get_mut();
+                self.kv_size.sub_val(old_op);
+
+                match old_op {
+                    KeyOp::Insert(old_op_new_value) => {
+                        if sanity_check_enabled() && !value_checker(old_op_new_value, &old_value) {
                             return Err(Box::new(MemTableError::InconsistentOperation {
                                 key: e.key().clone(),
                                 prev: e.get().clone(),
@@ -206,27 +245,40 @@ impl MemTable {
 
                         self.kv_size.sub_size(key_len);
                         e.remove();
-
                         Ok(())
                     }
-                    KeyOp::Delete(_) => Err(MemTableError::InconsistentOperation {
-                        key: e.key().clone(),
-                        prev: e.get().clone(),
-                        new: KeyOp::Delete(old_value),
+                    KeyOp::Delete(_) => {
+                        let new_op = KeyOp::Delete(old_value);
+                        let err = MemTableError::InconsistentOperation {
+                            key: e.key().clone(),
+                            prev: e.get().clone(),
+                            new: new_op.clone(),
+                        };
+
+                        if sanity_check_enabled() {
+                            Err(err.into())
+                        } else {
+                            tracing::error!(
+                                error = %err.as_report(),
+                                "double delete, ignoring because sanity check is disabled"
+                            );
+                            self.kv_size.add_val(&new_op);
+                            e.insert(new_op);
+                            Ok(())
+                        }
                     }
-                    .into()),
-                    KeyOp::Update(value) => {
-                        let (original_old_value, original_new_value) = std::mem::take(value);
-                        if ENABLE_SANITY_CHECK && original_new_value != old_value {
+                    KeyOp::Update((old_op_old_value, old_op_new_value)) => {
+                        if sanity_check_enabled() && !value_checker(old_op_new_value, &old_value) {
                             return Err(Box::new(MemTableError::InconsistentOperation {
                                 key: e.key().clone(),
                                 prev: e.get().clone(),
                                 new: KeyOp::Delete(old_value),
                             }));
                         }
-                        let delete_value = KeyOp::Delete(original_old_value);
-                        self.kv_size.add_val(&delete_value);
-                        e.insert(delete_value);
+
+                        let new_op = KeyOp::Delete(std::mem::take(old_op_old_value));
+                        self.kv_size.add_val(&new_op);
+                        e.insert(new_op);
                         Ok(())
                     }
                 }
@@ -240,61 +292,79 @@ impl MemTable {
         old_value: Bytes,
         new_value: Bytes,
     ) -> Result<()> {
-        let OpConsistencyLevel::ConsistentOldValue(value_checker) = &self.op_consistency_level
+        let OpConsistencyLevel::ConsistentOldValue {
+            check_old_value: value_checker,
+            ..
+        } = &self.op_consistency_level
         else {
             let key_len = std::mem::size_of::<Bytes>() + pk.len();
-
-            let update_value = KeyOp::Update((old_value, new_value));
-            self.kv_size.add(&pk, &update_value);
-            let origin_value = self.buffer.insert(pk, update_value);
-            self.sub_origin_size(origin_value, key_len);
+            let op = KeyOp::Update((old_value, new_value));
+            self.kv_size.add(&pk, &op);
+            let old_op = self.buffer.insert(pk, op);
+            self.sub_old_op_size(old_op, key_len);
             return Ok(());
         };
         let entry = self.buffer.entry(pk);
         match entry {
             Entry::Vacant(e) => {
-                let update_value = KeyOp::Update((old_value, new_value));
-                self.kv_size.add(e.key(), &update_value);
-                e.insert(update_value);
+                let op = KeyOp::Update((old_value, new_value));
+                self.kv_size.add(e.key(), &op);
+                e.insert(op);
                 Ok(())
             }
             Entry::Occupied(mut e) => {
-                let origin_value = e.get_mut();
-                self.kv_size.sub_val(origin_value);
-                match origin_value {
-                    KeyOp::Insert(original_new_value) => {
-                        if ENABLE_SANITY_CHECK && !value_checker(original_new_value, &old_value) {
+                let old_op = e.get_mut();
+                self.kv_size.sub_val(old_op);
+
+                match old_op {
+                    KeyOp::Insert(old_op_new_value) => {
+                        if sanity_check_enabled() && !value_checker(old_op_new_value, &old_value) {
                             return Err(Box::new(MemTableError::InconsistentOperation {
                                 key: e.key().clone(),
                                 prev: e.get().clone(),
                                 new: KeyOp::Update((old_value, new_value)),
                             }));
                         }
-                        let new_key_op = KeyOp::Insert(new_value);
-                        self.kv_size.add_val(&new_key_op);
-                        e.insert(new_key_op);
+
+                        let new_op = KeyOp::Insert(new_value);
+                        self.kv_size.add_val(&new_op);
+                        e.insert(new_op);
                         Ok(())
                     }
-                    KeyOp::Update((origin_old_value, original_new_value)) => {
-                        if ENABLE_SANITY_CHECK && !value_checker(original_new_value, &old_value) {
+                    KeyOp::Update((old_op_old_value, old_op_new_value)) => {
+                        if sanity_check_enabled() && !value_checker(old_op_new_value, &old_value) {
                             return Err(Box::new(MemTableError::InconsistentOperation {
                                 key: e.key().clone(),
                                 prev: e.get().clone(),
                                 new: KeyOp::Update((old_value, new_value)),
                             }));
                         }
-                        let old_value = std::mem::take(origin_old_value);
-                        let new_key_op = KeyOp::Update((old_value, new_value));
-                        self.kv_size.add_val(&new_key_op);
-                        e.insert(new_key_op);
+
+                        let new_op = KeyOp::Update((std::mem::take(old_op_old_value), new_value));
+                        self.kv_size.add_val(&new_op);
+                        e.insert(new_op);
                         Ok(())
                     }
-                    KeyOp::Delete(_) => Err(MemTableError::InconsistentOperation {
-                        key: e.key().clone(),
-                        prev: e.get().clone(),
-                        new: KeyOp::Update((old_value, new_value)),
+                    KeyOp::Delete(_) => {
+                        let new_op = KeyOp::Update((old_value, new_value));
+                        let err = MemTableError::InconsistentOperation {
+                            key: e.key().clone(),
+                            prev: e.get().clone(),
+                            new: new_op.clone(),
+                        };
+
+                        if sanity_check_enabled() {
+                            Err(err.into())
+                        } else {
+                            tracing::error!(
+                                error = %err.as_report(),
+                                "update on deleted, ignoring because sanity check is disabled"
+                            );
+                            self.kv_size.add_val(&new_op);
+                            e.insert(new_op);
+                            Ok(())
+                        }
                     }
-                    .into()),
                 }
             }
         }
@@ -314,9 +384,19 @@ impl MemTable {
         self.buffer.range(key_range)
     }
 
-    fn sub_origin_size(&mut self, origin_value: Option<KeyOp>, key_len: usize) {
-        if let Some(origin_value) = origin_value {
-            self.kv_size.sub_val(&origin_value);
+    pub fn rev_iter<'a, R>(
+        &'a self,
+        key_range: R,
+    ) -> impl Iterator<Item = (&'a TableKey<Bytes>, &'a KeyOp)>
+    where
+        R: RangeBounds<TableKey<Bytes>> + 'a,
+    {
+        self.buffer.range(key_range).rev()
+    }
+
+    fn sub_old_op_size(&mut self, old_op: Option<KeyOp>, key_len: usize) {
+        if let Some(op) = old_op {
+            self.kv_size.sub_val(&op);
             self.kv_size.sub_size(key_len);
         }
     }
@@ -459,6 +539,7 @@ impl<S: StateStoreWrite + StateStoreRead> MemtableLocalStateStore<S> {
 
 impl<S: StateStoreWrite + StateStoreRead> LocalStateStore for MemtableLocalStateStore<S> {
     type Iter<'a> = impl StateStoreIter + 'a;
+    type RevIter<'a> = impl StateStoreIter + 'a;
 
     #[allow(clippy::unused_async)]
     async fn may_exist(
@@ -503,6 +584,26 @@ impl<S: StateStoreWrite + StateStoreRead> LocalStateStore for MemtableLocalState
         }
     }
 
+    #[allow(clippy::manual_async_fn)]
+    fn rev_iter(
+        &self,
+        key_range: TableKeyRange,
+        read_options: ReadOptions,
+    ) -> impl Future<Output = StorageResult<Self::RevIter<'_>>> + Send + '_ {
+        async move {
+            let iter = self
+                .inner
+                .rev_iter(key_range.clone(), self.epoch(), read_options)
+                .await?;
+            Ok(FromStreamStateStoreIter::new(Box::pin(merge_stream(
+                self.mem_table.rev_iter(key_range),
+                iter.into_stream(to_owned_item),
+                self.table_id,
+                self.epoch(),
+            ))))
+        }
+    }
+
     fn insert(
         &mut self,
         key: TableKey<Bytes>,
@@ -529,7 +630,7 @@ impl<S: StateStoreWrite + StateStoreRead> LocalStateStore for MemtableLocalState
                 // a workaround you may call disable the check by initializing the
                 // state store with `op_consistency_level=Inconsistent`.
                 KeyOp::Insert(value) => {
-                    if ENABLE_SANITY_CHECK {
+                    if sanity_check_enabled() {
                         do_insert_sanity_check(
                             &key,
                             &value,
@@ -544,7 +645,7 @@ impl<S: StateStoreWrite + StateStoreRead> LocalStateStore for MemtableLocalState
                     kv_pairs.push((key, StorageValue::new_put(value)));
                 }
                 KeyOp::Delete(old_value) => {
-                    if ENABLE_SANITY_CHECK {
+                    if sanity_check_enabled() {
                         do_delete_sanity_check(
                             &key,
                             &old_value,
@@ -559,7 +660,7 @@ impl<S: StateStoreWrite + StateStoreRead> LocalStateStore for MemtableLocalState
                     kv_pairs.push((key, StorageValue::new_delete()));
                 }
                 KeyOp::Update((old_value, new_value)) => {
-                    if ENABLE_SANITY_CHECK {
+                    if sanity_check_enabled() {
                         do_update_sanity_check(
                             &key,
                             &old_value,
@@ -678,14 +779,15 @@ mod tests {
 
     use crate::hummock::iterator::HummockIterator;
     use crate::hummock::value::HummockValue;
-    use crate::mem_table::{KeyOp, MemTable, MemTableHummockIterator};
+    use crate::mem_table::{KeyOp, MemTable, MemTableHummockIterator, MemTableHummockRevIterator};
     use crate::store::{OpConsistencyLevel, CHECK_BYTES_EQUAL};
 
     #[tokio::test]
     async fn test_mem_table_memory_size() {
-        let mut mem_table = MemTable::new(OpConsistencyLevel::ConsistentOldValue(
-            CHECK_BYTES_EQUAL.clone(),
-        ));
+        let mut mem_table = MemTable::new(OpConsistencyLevel::ConsistentOldValue {
+            check_old_value: CHECK_BYTES_EQUAL.clone(),
+            is_log_store: false,
+        });
         assert_eq!(mem_table.kv_size.size(), 0);
 
         mem_table
@@ -863,7 +965,7 @@ mod tests {
             TableKey(bytes.freeze())
         }
 
-        let ordered_test_data = (0..10000)
+        let mut ordered_test_data = (0..10000)
             .map(|i| {
                 let key_op = match rng.gen::<usize>() % 3 {
                     0 => KeyOp::Insert(Bytes::from("insert")),
@@ -878,9 +980,10 @@ mod tests {
         let mut test_data = ordered_test_data.clone();
 
         test_data.shuffle(&mut rng);
-        let mut mem_table = MemTable::new(OpConsistencyLevel::ConsistentOldValue(
-            CHECK_BYTES_EQUAL.clone(),
-        ));
+        let mut mem_table = MemTable::new(OpConsistencyLevel::ConsistentOldValue {
+            check_old_value: CHECK_BYTES_EQUAL.clone(),
+            is_log_store: false,
+        });
         for (key, op) in test_data {
             match op {
                 KeyOp::Insert(value) => {
@@ -896,8 +999,8 @@ mod tests {
         const TEST_TABLE_ID: TableId = TableId::new(233);
         const TEST_EPOCH: u64 = test_epoch(10);
 
-        async fn check_data(
-            iter: &mut MemTableHummockIterator<'_>,
+        async fn check_data<I: HummockIterator>(
+            iter: &mut I,
             test_data: &[(TableKey<Bytes>, KeyOp)],
         ) {
             let mut idx = 0;
@@ -908,7 +1011,14 @@ mod tests {
                 let (expected_key, expected_value) = test_data[idx].clone();
                 assert_eq!(key.epoch_with_gap, EpochWithGap::new_from_epoch(TEST_EPOCH));
                 assert_eq!(key.user_key.table_id, TEST_TABLE_ID);
-                assert_eq!(key.user_key.table_key.0, expected_key.0.as_ref());
+                assert_eq!(
+                    key.user_key.table_key.0,
+                    expected_key.0.as_ref(),
+                    "failed at {}, {:?} != {:?}",
+                    idx,
+                    String::from_utf8(key.user_key.table_key.key_part().to_vec()).unwrap(),
+                    String::from_utf8(expected_key.key_part().to_vec()).unwrap(),
+                );
                 match expected_value {
                     KeyOp::Insert(expected_value) | KeyOp::Update((_, expected_value)) => {
                         assert_eq!(value, HummockValue::Put(expected_value.as_ref()));
@@ -999,5 +1109,109 @@ mod tests {
         .await
         .unwrap();
         check_data(&mut iter, &[]).await;
+
+        // check reverse iterator
+        ordered_test_data.reverse();
+        drop(iter);
+        let mut iter = MemTableHummockRevIterator::new(
+            &mem_table.buffer,
+            EpochWithGap::new_from_epoch(TEST_EPOCH),
+            TEST_TABLE_ID,
+        );
+
+        // Test rewind
+        iter.rewind().await.unwrap();
+        check_data(&mut iter, &ordered_test_data).await;
+
+        // Test seek with a smaller table id
+        let smaller_table_id = TableId::new(TEST_TABLE_ID.table_id() - 1);
+        iter.seek(FullKey {
+            user_key: UserKey {
+                table_id: smaller_table_id,
+                table_key: TableKey(&get_key(ordered_test_data.len() + 10)),
+            },
+            epoch_with_gap: EpochWithGap::new_from_epoch(TEST_EPOCH),
+        })
+        .await
+        .unwrap();
+        check_data(&mut iter, &[]).await;
+
+        // Test seek with a greater table id
+        let greater_table_id = TableId::new(TEST_TABLE_ID.table_id() + 1);
+        iter.seek(FullKey {
+            user_key: UserKey {
+                table_id: greater_table_id,
+                table_key: TableKey(&get_key(0)),
+            },
+            epoch_with_gap: EpochWithGap::new_from_epoch(TEST_EPOCH),
+        })
+        .await
+        .unwrap();
+        check_data(&mut iter, &ordered_test_data).await;
+
+        // Test seek with a later epoch, the first key is skipped
+        let later_epoch = EpochWithGap::new_from_epoch(TEST_EPOCH.next_epoch());
+        let seek_idx = 500;
+        iter.seek(FullKey {
+            user_key: UserKey {
+                table_id: TEST_TABLE_ID,
+                table_key: TableKey(&get_key(seek_idx)),
+            },
+            epoch_with_gap: later_epoch,
+        })
+        .await
+        .unwrap();
+        let rev_seek_idx = ordered_test_data.len() - seek_idx - 1;
+        check_data(&mut iter, &ordered_test_data[rev_seek_idx + 1..]).await;
+
+        // Test seek with a earlier epoch, the first key is not skipped
+        let early_epoch = EpochWithGap::new_from_epoch(TEST_EPOCH.prev_epoch());
+        let seek_idx = 500;
+        iter.seek(FullKey {
+            user_key: UserKey {
+                table_id: TEST_TABLE_ID,
+                table_key: TableKey(&get_key(seek_idx)),
+            },
+            epoch_with_gap: early_epoch,
+        })
+        .await
+        .unwrap();
+        let rev_seek_idx = ordered_test_data.len() - seek_idx - 1;
+        check_data(&mut iter, &ordered_test_data[rev_seek_idx..]).await;
+
+        drop(iter);
+        mem_table.insert(get_key(10001), "value1".into()).unwrap();
+
+        let mut iter = MemTableHummockRevIterator::new(
+            &mem_table.buffer,
+            EpochWithGap::new_from_epoch(TEST_EPOCH),
+            TEST_TABLE_ID,
+        );
+        iter.seek(FullKey {
+            user_key: UserKey {
+                table_id: TEST_TABLE_ID,
+                table_key: TableKey(&get_key(10000)),
+            },
+            epoch_with_gap: early_epoch,
+        })
+        .await
+        .unwrap();
+        assert_eq!(iter.key().user_key.table_key, get_key(9999).to_ref());
+
+        let mut iter = MemTableHummockIterator::new(
+            &mem_table.buffer,
+            EpochWithGap::new_from_epoch(TEST_EPOCH),
+            TEST_TABLE_ID,
+        );
+        iter.seek(FullKey {
+            user_key: UserKey {
+                table_id: TEST_TABLE_ID,
+                table_key: TableKey(&get_key(10000)),
+            },
+            epoch_with_gap: later_epoch,
+        })
+        .await
+        .unwrap();
+        assert_eq!(iter.key().user_key.table_key, get_key(10001).to_ref());
     }
 }
diff --git a/src/storage/src/memory.rs b/src/storage/src/memory.rs
index 5e645aa7ea57f..a266fbc129cfd 100644
--- a/src/storage/src/memory.rs
+++ b/src/storage/src/memory.rs
@@ -40,6 +40,12 @@ pub trait RangeKv: Clone + Send + Sync + 'static {
         limit: Option<usize>,
     ) -> StorageResult<Vec<(BytesFullKey, Option<Bytes>)>>;
 
+    fn rev_range(
+        &self,
+        range: BytesFullKeyRange,
+        limit: Option<usize>,
+    ) -> StorageResult<Vec<(BytesFullKey, Option<Bytes>)>>;
+
     fn ingest_batch(
         &self,
         kv_pairs: impl Iterator<Item = (BytesFullKey, Option<Bytes>)>,
@@ -65,6 +71,21 @@ impl RangeKv for BTreeMapRangeKv {
             .collect())
     }
 
+    fn rev_range(
+        &self,
+        range: BytesFullKeyRange,
+        limit: Option<usize>,
+    ) -> StorageResult<Vec<(BytesFullKey, Option<Bytes>)>> {
+        let limit = limit.unwrap_or(usize::MAX);
+        Ok(self
+            .read()
+            .range(range)
+            .rev()
+            .take(limit)
+            .map(|(key, value)| (key.clone(), value.clone()))
+            .collect())
+    }
+
     fn ingest_batch(
         &self,
         kv_pairs: impl Iterator<Item = (BytesFullKey, Option<Bytes>)>,
@@ -148,6 +169,43 @@ pub mod sled {
             Ok(ret)
         }
 
+        fn rev_range(
+            &self,
+            range: BytesFullKeyRange,
+            limit: Option<usize>,
+        ) -> StorageResult<Vec<(BytesFullKey, Option<Bytes>)>> {
+            let (left, right) = range;
+            let full_key_ref_bound = (
+                left.as_ref().map(FullKey::to_ref),
+                right.as_ref().map(FullKey::to_ref),
+            );
+            let left_encoded = left.as_ref().map(|key| key.to_ref().encode_reverse_epoch());
+            let right_encoded = right
+                .as_ref()
+                .map(|key| key.to_ref().encode_reverse_epoch());
+            let limit = limit.unwrap_or(usize::MAX);
+            let mut ret = vec![];
+            for result in self
+                .inner
+                .range((left_encoded, right_encoded))
+                .rev()
+                .take(limit)
+            {
+                let (key, value) = result?;
+                let full_key = FullKey::decode_reverse_epoch(key.as_ref()).copy_into();
+                if !full_key_ref_bound.contains(&full_key.to_ref()) {
+                    continue;
+                }
+                let value = match value.as_ref() {
+                    [EMPTY] => None,
+                    [NON_EMPTY, rest @ ..] => Some(Bytes::from(Vec::from(rest))),
+                    _ => unreachable!("malformed value: {:?}", value),
+                };
+                ret.push((full_key, value))
+            }
+            Ok(ret)
+        }
+
         fn ingest_batch(
             &self,
             kv_pairs: impl Iterator<Item = (BytesFullKey, Option<Bytes>)>,
@@ -293,13 +351,15 @@ mod batched_iter {
         inner: R,
         range: BytesFullKeyRange,
         current: std::vec::IntoIter<(FullKey<Bytes>, Option<Bytes>)>,
+        rev: bool,
     }
 
     impl<R: RangeKv> Iter<R> {
-        pub fn new(inner: R, range: BytesFullKeyRange) -> Self {
+        pub fn new(inner: R, range: BytesFullKeyRange, rev: bool) -> Self {
             Self {
                 inner,
                 range,
+                rev,
                 current: Vec::new().into_iter(),
             }
         }
@@ -312,14 +372,20 @@ mod batched_iter {
         fn refill(&mut self) -> StorageResult<()> {
             assert!(self.current.is_empty());
 
-            let batch = self
-                .inner
-                .range(
+            let batch = if self.rev {
+                self.inner.rev_range(
                     (self.range.0.clone(), self.range.1.clone()),
                     Some(Self::BATCH_SIZE),
                 )?
-                .into_iter()
-                .collect_vec();
+            } else {
+                self.inner
+                    .range(
+                        (self.range.0.clone(), self.range.1.clone()),
+                        Some(Self::BATCH_SIZE),
+                    )?
+                    .into_iter()
+                    .collect_vec()
+            };
 
             if let Some((last_key, _)) = batch.last() {
                 let full_key = FullKey::new_with_gap_epoch(
@@ -327,7 +393,11 @@ mod batched_iter {
                     TableKey(last_key.user_key.table_key.0.clone()),
                     last_key.epoch_with_gap,
                 );
-                self.range.0 = Bound::Excluded(full_key);
+                if self.rev {
+                    self.range.1 = Bound::Excluded(full_key);
+                } else {
+                    self.range.0 = Bound::Excluded(full_key);
+                }
             }
             self.current = batch.into_iter();
             Ok(())
@@ -350,7 +420,6 @@ mod batched_iter {
     #[cfg(test)]
     mod tests {
         use rand::Rng;
-        use risingwave_hummock_sdk::key::FullKey;
 
         use super::*;
         use crate::memory::sled::SledRangeKv;
@@ -414,7 +483,7 @@ mod batched_iter {
 
                 let v1 = {
                     let mut v = vec![];
-                    let mut iter = Iter::new(map.clone(), range.clone());
+                    let mut iter = Iter::new(map.clone(), range.clone(), false);
                     while let Some((key, value)) = iter.next().unwrap() {
                         v.push((key, value));
                     }
@@ -540,6 +609,7 @@ impl<R: RangeKv> RangeKvStateStore<R> {
 impl<R: RangeKv> StateStoreRead for RangeKvStateStore<R> {
     type ChangeLogIter = RangeKvStateStoreChangeLogIter<R>;
     type Iter = RangeKvStateStoreIter<R>;
+    type RevIter = RangeKvStateStoreIter<R>;
 
     #[allow(clippy::unused_async)]
     async fn get(
@@ -570,6 +640,25 @@ impl<R: RangeKv> StateStoreRead for RangeKvStateStore<R> {
             batched_iter::Iter::new(
                 self.inner.clone(),
                 to_full_key_range(read_options.table_id, key_range),
+                false,
+            ),
+            epoch,
+            true,
+        ))
+    }
+
+    #[allow(clippy::unused_async)]
+    async fn rev_iter(
+        &self,
+        key_range: TableKeyRange,
+        epoch: u64,
+        read_options: ReadOptions,
+    ) -> StorageResult<Self::RevIter> {
+        Ok(RangeKvStateStoreIter::new(
+            batched_iter::Iter::new(
+                self.inner.clone(),
+                to_full_key_range(read_options.table_id, key_range),
+                true,
             ),
             epoch,
             true,
@@ -586,6 +675,7 @@ impl<R: RangeKv> StateStoreRead for RangeKvStateStore<R> {
             batched_iter::Iter::new(
                 self.inner.clone(),
                 to_full_key_range(options.table_id, key_range.clone()),
+                false,
             ),
             max_epoch,
             true,
@@ -594,6 +684,7 @@ impl<R: RangeKv> StateStoreRead for RangeKvStateStore<R> {
             batched_iter::Iter::new(
                 self.inner.clone(),
                 to_full_key_range(options.table_id, key_range),
+                false,
             ),
             min_epoch,
             false,
@@ -654,10 +745,13 @@ impl<R: RangeKv> StateStore for RangeKvStateStore<R> {
     }
 
     #[allow(clippy::unused_async)]
-    async fn sync(&self, _epoch: u64) -> StorageResult<SyncResult> {
-        self.inner.flush()?;
+    fn sync(&self, _epoch: u64) -> impl SyncFuture {
+        let result = self.inner.flush();
         // memory backend doesn't need to push to S3, so this is a no-op
-        Ok(SyncResult::default())
+        async move {
+            result?;
+            Ok(SyncResult::default())
+        }
     }
 
     fn seal_epoch(&self, _epoch: u64, _is_checkpoint: bool) {}
diff --git a/src/storage/src/monitor/hummock_state_store_metrics.rs b/src/storage/src/monitor/hummock_state_store_metrics.rs
index 16b54777233e4..ff60735f0a5e4 100644
--- a/src/storage/src/monitor/hummock_state_store_metrics.rs
+++ b/src/storage/src/monitor/hummock_state_store_metrics.rs
@@ -75,15 +75,17 @@ pub struct HummockStateStoreMetrics {
     // uploading task
     pub uploader_uploading_task_size: GenericGauge<AtomicU64>,
     pub uploader_uploading_task_count: IntGauge,
+    pub uploader_imm_size: GenericGauge<AtomicU64>,
     pub uploader_upload_task_latency: Histogram,
     pub uploader_syncing_epoch_count: IntGauge,
     pub uploader_wait_poll_latency: Histogram,
 
     // memory
     pub mem_table_spill_counts: RelabeledCounterVec,
+    pub old_value_size: IntGauge,
 
     // block statistics
-    pub block_efficiency_histogram: RelabeledHistogramVec,
+    pub block_efficiency_histogram: Histogram,
 
     pub event_handler_pending_event: IntGauge,
     pub event_handler_latency: HistogramVec,
@@ -323,6 +325,15 @@ impl HummockStateStoreMetrics {
         )
         .unwrap();
 
+        let uploader_imm_size = GenericGauge::new(
+            "state_store_uploader_imm_size",
+            "Total size of imms tracked by uploader",
+        )
+        .unwrap();
+        registry
+            .register(Box::new(uploader_imm_size.clone()))
+            .unwrap();
+
         let opts = histogram_opts!(
             "state_store_uploader_upload_task_latency",
             "Latency of uploader uploading tasks",
@@ -404,18 +415,19 @@ impl HummockStateStoreMetrics {
             metric_level,
         );
 
+        let old_value_size = register_int_gauge_with_registry!(
+            "state_store_old_value_size",
+            "The size of old value",
+            registry
+        )
+        .unwrap();
+
         let opts = histogram_opts!(
             "block_efficiency_histogram",
             "Access ratio of in-memory block.",
             exponential_buckets(0.001, 2.0, 11).unwrap(),
         );
-        let block_efficiency_histogram =
-            register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
-        let block_efficiency_histogram = RelabeledHistogramVec::with_metric_level(
-            MetricLevel::Info,
-            block_efficiency_histogram,
-            metric_level,
-        );
+        let block_efficiency_histogram = register_histogram_with_registry!(opts, registry).unwrap();
 
         let event_handler_pending_event = register_int_gauge_with_registry!(
             "state_store_event_handler_pending_event",
@@ -458,10 +470,12 @@ impl HummockStateStoreMetrics {
             spill_task_size_from_unsealed: spill_task_size.with_label_values(&["unsealed"]),
             uploader_uploading_task_size,
             uploader_uploading_task_count,
+            uploader_imm_size,
             uploader_upload_task_latency,
             uploader_syncing_epoch_count,
             uploader_wait_poll_latency,
             mem_table_spill_counts,
+            old_value_size,
 
             block_efficiency_histogram,
             event_handler_pending_event,
@@ -478,6 +492,7 @@ pub trait MemoryCollector: Sync + Send {
     fn get_meta_memory_usage(&self) -> u64;
     fn get_data_memory_usage(&self) -> u64;
     fn get_uploading_memory_usage(&self) -> u64;
+    fn get_prefetch_memory_usage(&self) -> usize;
     fn get_meta_cache_memory_usage_ratio(&self) -> f64;
     fn get_block_cache_memory_usage_ratio(&self) -> f64;
     fn get_shared_buffer_usage_ratio(&self) -> f64;
@@ -490,6 +505,7 @@ struct StateStoreCollector {
     block_cache_size: IntGauge,
     meta_cache_size: IntGauge,
     uploading_memory_size: IntGauge,
+    prefetch_memory_size: IntGauge,
     meta_cache_usage_ratio: Gauge,
     block_cache_usage_ratio: Gauge,
     uploading_memory_usage_ratio: Gauge,
@@ -541,12 +557,20 @@ impl StateStoreCollector {
         .unwrap();
         descs.extend(uploading_memory_usage_ratio.desc().into_iter().cloned());
 
+        let prefetch_memory_size = IntGauge::with_opts(Opts::new(
+            "state_store_prefetch_memory_size",
+            "the size of prefetch memory usage",
+        ))
+        .unwrap();
+        descs.extend(prefetch_memory_size.desc().into_iter().cloned());
+
         Self {
             memory_collector,
             descs,
             block_cache_size,
             meta_cache_size,
             uploading_memory_size,
+            prefetch_memory_size,
             meta_cache_usage_ratio,
             block_cache_usage_ratio,
 
@@ -567,6 +591,8 @@ impl Collector for StateStoreCollector {
             .set(self.memory_collector.get_meta_memory_usage() as i64);
         self.uploading_memory_size
             .set(self.memory_collector.get_uploading_memory_usage() as i64);
+        self.prefetch_memory_size
+            .set(self.memory_collector.get_prefetch_memory_usage() as i64);
         self.meta_cache_usage_ratio
             .set(self.memory_collector.get_meta_cache_memory_usage_ratio());
         self.block_cache_usage_ratio
diff --git a/src/storage/src/monitor/local_metrics.rs b/src/storage/src/monitor/local_metrics.rs
index 910b4f2939383..c218e484265bf 100644
--- a/src/storage/src/monitor/local_metrics.rs
+++ b/src/storage/src/monitor/local_metrics.rs
@@ -22,6 +22,7 @@ use std::sync::Arc;
 use prometheus::local::{LocalHistogram, LocalIntCounter};
 use risingwave_common::catalog::TableId;
 use risingwave_common::metrics::LabelGuardedLocalIntCounter;
+use risingwave_hummock_sdk::table_stats::TableStatsMap;
 
 use super::HummockStateStoreMetrics;
 use crate::monitor::CompactorMetrics;
@@ -63,6 +64,11 @@ pub struct StoreLocalStatistic {
     reported: AtomicBool,
     #[cfg(all(debug_assertions, not(any(madsim, test, feature = "test"))))]
     added: AtomicBool,
+
+    /// The stats of key skipped by watermark for each table.
+    /// Used by `SkipWatermarkIterator`.
+    /// Generalize it in the future if there're other iterators that'll also drop keys.
+    pub skipped_by_watermark_table_stats: TableStatsMap,
 }
 
 impl StoreLocalStatistic {
diff --git a/src/storage/src/monitor/monitored_storage_metrics.rs b/src/storage/src/monitor/monitored_storage_metrics.rs
index 15f58cb57feb3..bbf904d38381a 100644
--- a/src/storage/src/monitor/monitored_storage_metrics.rs
+++ b/src/storage/src/monitor/monitored_storage_metrics.rs
@@ -17,34 +17,46 @@ use std::collections::HashMap;
 use std::sync::{Arc, OnceLock};
 use std::time::{Duration, Instant};
 
-use prometheus::core::{AtomicU64, GenericLocalCounter};
-use prometheus::local::LocalHistogram;
 use prometheus::{
     exponential_buckets, histogram_opts, linear_buckets, register_histogram_vec_with_registry,
-    register_histogram_with_registry, register_int_counter_vec_with_registry, Histogram, Registry,
+    register_histogram_with_registry, Histogram, Registry,
 };
 use risingwave_common::config::MetricLevel;
 use risingwave_common::metrics::{
-    LabelGuardedMetric, RelabeledCounterVec, RelabeledGuardedHistogramVec, RelabeledHistogramVec,
+    LabelGuardedIntCounterVec, LabelGuardedIntGauge, LabelGuardedLocalHistogram,
+    LabelGuardedLocalIntCounter, RelabeledGuardedHistogramVec, RelabeledGuardedIntCounterVec,
+    RelabeledGuardedIntGaugeVec, RelabeledHistogramVec,
 };
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
-use risingwave_common::register_guarded_histogram_vec_with_registry;
+use risingwave_common::{
+    register_guarded_histogram_vec_with_registry, register_guarded_int_counter_vec_with_registry,
+    register_guarded_int_gauge_vec_with_registry,
+};
+
+use crate::store::{
+    ChangeLogValue, IterItem, StateStoreIterItem, StateStoreIterItemRef, StateStoreReadLogItem,
+    StateStoreReadLogItemRef,
+};
 
 /// [`MonitoredStorageMetrics`] stores the performance and IO metrics of Storage.
 #[derive(Debug, Clone)]
 pub struct MonitoredStorageMetrics {
     pub get_duration: RelabeledGuardedHistogramVec<1>,
-    pub get_key_size: RelabeledHistogramVec,
-    pub get_value_size: RelabeledHistogramVec,
-
-    pub iter_size: RelabeledHistogramVec,
-    pub iter_item: RelabeledHistogramVec,
-    pub iter_init_duration: RelabeledGuardedHistogramVec<1>,
-    pub iter_scan_duration: RelabeledGuardedHistogramVec<1>,
+    pub get_key_size: RelabeledGuardedHistogramVec<1>,
+    pub get_value_size: RelabeledGuardedHistogramVec<1>,
+
+    // [table_id, iter_type: {"iter", "iter_log"}]
+    pub iter_size: RelabeledGuardedHistogramVec<2>,
+    pub iter_item: RelabeledGuardedHistogramVec<2>,
+    pub iter_init_duration: RelabeledGuardedHistogramVec<2>,
+    pub iter_scan_duration: RelabeledGuardedHistogramVec<2>,
+    pub iter_counts: RelabeledGuardedIntCounterVec<2>,
+    pub iter_in_progress_counts: RelabeledGuardedIntGaugeVec<2>,
+
+    // [table_id, op_type]
+    pub iter_log_op_type_counts: LabelGuardedIntCounterVec<2>,
     pub may_exist_duration: RelabeledHistogramVec,
 
-    pub iter_in_process_counts: RelabeledCounterVec,
-
     pub sync_duration: Histogram,
     pub sync_size: Histogram,
 }
@@ -70,8 +82,8 @@ impl MonitoredStorageMetrics {
             size_buckets.clone()
         );
         let get_key_size =
-            register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
-        let get_key_size = RelabeledHistogramVec::with_metric_level(
+            register_guarded_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let get_key_size = RelabeledGuardedHistogramVec::with_metric_level(
             MetricLevel::Debug,
             get_key_size,
             metric_level,
@@ -83,8 +95,8 @@ impl MonitoredStorageMetrics {
             size_buckets.clone()
         );
         let get_value_size =
-            register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
-        let get_value_size = RelabeledHistogramVec::with_metric_level(
+            register_guarded_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+        let get_value_size = RelabeledGuardedHistogramVec::with_metric_level(
             MetricLevel::Debug,
             get_value_size,
             metric_level,
@@ -124,32 +136,53 @@ impl MonitoredStorageMetrics {
             "Total bytes gotten from state store scan(), for calculating read throughput",
             size_buckets.clone()
         );
-        let iter_size =
-            register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
-        let iter_size =
-            RelabeledHistogramVec::with_metric_level(MetricLevel::Debug, iter_size, metric_level);
+        let iter_size = register_guarded_histogram_vec_with_registry!(
+            opts,
+            &["table_id", "iter_type"],
+            registry
+        )
+        .unwrap();
+        let iter_size = RelabeledGuardedHistogramVec::with_metric_level_relabel_n(
+            MetricLevel::Debug,
+            iter_size,
+            metric_level,
+            1,
+        );
 
         let opts = histogram_opts!(
             "state_store_iter_item",
             "Total bytes gotten from state store scan(), for calculating read throughput",
             size_buckets.clone(),
         );
-        let iter_item =
-            register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
-        let iter_item =
-            RelabeledHistogramVec::with_metric_level(MetricLevel::Debug, iter_item, metric_level);
+        let iter_item = register_guarded_histogram_vec_with_registry!(
+            opts,
+            &["table_id", "iter_type"],
+            registry
+        )
+        .unwrap();
+        let iter_item = RelabeledGuardedHistogramVec::with_metric_level_relabel_n(
+            MetricLevel::Debug,
+            iter_item,
+            metric_level,
+            1,
+        );
 
         let opts = histogram_opts!(
             "state_store_iter_init_duration",
             "Histogram of the time spent on iterator initialization.",
             state_store_read_time_buckets.clone(),
         );
-        let iter_init_duration =
-            register_guarded_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
-        let iter_init_duration = RelabeledGuardedHistogramVec::with_metric_level(
+        let iter_init_duration = register_guarded_histogram_vec_with_registry!(
+            opts,
+            &["table_id", "iter_type"],
+            registry
+        )
+        .unwrap();
+        let iter_init_duration = RelabeledGuardedHistogramVec::with_metric_level_relabel_n(
             MetricLevel::Critical,
             iter_init_duration,
             metric_level,
+            1,
         );
 
         let opts = histogram_opts!(
@@ -157,27 +190,55 @@ impl MonitoredStorageMetrics {
             "Histogram of the time spent on iterator scanning.",
             state_store_read_time_buckets.clone(),
         );
-        let iter_scan_duration =
-            register_guarded_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
-        let iter_scan_duration = RelabeledGuardedHistogramVec::with_metric_level(
+        let iter_scan_duration = register_guarded_histogram_vec_with_registry!(
+            opts,
+            &["table_id", "iter_type"],
+            registry
+        )
+        .unwrap();
+        let iter_scan_duration = RelabeledGuardedHistogramVec::with_metric_level_relabel_n(
             MetricLevel::Critical,
             iter_scan_duration,
             metric_level,
+            1,
         );
 
-        let iter_in_process_counts = register_int_counter_vec_with_registry!(
-            "state_store_iter_in_process_counts",
-            "Total number of iter_in_process that have been issued to state store",
-            &["table_id"],
+        let iter_counts = register_guarded_int_counter_vec_with_registry!(
+            "state_store_iter_counts",
+            "Total number of iter that have been issued to state store",
+            &["table_id", "iter_type"],
+            registry
+        )
+        .unwrap();
+        let iter_counts = RelabeledGuardedIntCounterVec::with_metric_level_relabel_n(
+            MetricLevel::Debug,
+            iter_counts,
+            metric_level,
+            1,
+        );
+
+        let iter_in_progress_counts = register_guarded_int_gauge_vec_with_registry!(
+            "state_store_iter_in_progress_counts",
+            "Total number of existing iter",
+            &["table_id", "iter_type"],
             registry
         )
         .unwrap();
-        let iter_in_process_counts = RelabeledCounterVec::with_metric_level(
+        let iter_in_progress_counts = RelabeledGuardedIntGaugeVec::with_metric_level_relabel_n(
             MetricLevel::Debug,
-            iter_in_process_counts,
+            iter_in_progress_counts,
             metric_level,
+            1,
         );
 
+        let iter_log_op_type_counts = register_guarded_int_counter_vec_with_registry!(
+            "state_store_iter_log_op_type_counts",
+            "Counter of each op type in iter_log",
+            &["table_id", "op_type"],
+            registry
+        )
+        .unwrap();
+
         let opts = histogram_opts!(
             "state_store_may_exist_duration",
             "Histogram of may exist time that have been issued to state store",
@@ -213,8 +274,10 @@ impl MonitoredStorageMetrics {
             iter_item,
             iter_init_duration,
             iter_scan_duration,
+            iter_counts,
+            iter_in_progress_counts,
+            iter_log_op_type_counts,
             may_exist_duration,
-            iter_in_process_counts,
             sync_duration,
             sync_size,
         }
@@ -224,22 +287,77 @@ impl MonitoredStorageMetrics {
         global_storage_metrics(MetricLevel::Disabled)
     }
 
-    pub fn local_metrics(&self, table_label: &str) -> LocalStorageMetrics {
+    fn local_iter_metrics(&self, table_label: &str) -> LocalIterMetrics {
+        let inner = self.new_local_iter_metrics_inner(table_label, "iter");
+        LocalIterMetrics {
+            inner,
+            report_count: 0,
+        }
+    }
+
+    fn new_local_iter_metrics_inner(
+        &self,
+        table_label: &str,
+        iter_type: &str,
+    ) -> LocalIterMetricsInner {
         let iter_init_duration = self
             .iter_init_duration
-            .with_label_values(&[table_label])
+            .with_label_values(&[table_label, iter_type])
             .local();
-        let iter_in_process_counts = self
-            .iter_in_process_counts
-            .with_label_values(&[table_label])
+        let iter_counts = self
+            .iter_counts
+            .with_label_values(&[table_label, iter_type])
             .local();
         let iter_scan_duration = self
             .iter_scan_duration
-            .with_label_values(&[table_label])
+            .with_label_values(&[table_label, iter_type])
+            .local();
+        let iter_item = self
+            .iter_item
+            .with_label_values(&[table_label, iter_type])
+            .local();
+        let iter_size = self
+            .iter_size
+            .with_label_values(&[table_label, iter_type])
+            .local();
+        let iter_in_progress_counts = self
+            .iter_in_progress_counts
+            .with_label_values(&[table_label, iter_type]);
+
+        LocalIterMetricsInner {
+            iter_init_duration,
+            iter_scan_duration,
+            iter_counts,
+            iter_item,
+            iter_size,
+            iter_in_progress_counts,
+        }
+    }
+
+    fn local_iter_log_metrics(&self, table_label: &str) -> LocalIterLogMetrics {
+        let iter_metrics = self.new_local_iter_metrics_inner(table_label, "iter_log");
+        let insert_count = self
+            .iter_log_op_type_counts
+            .with_guarded_label_values(&[table_label, "INSERT"])
+            .local();
+        let update_count = self
+            .iter_log_op_type_counts
+            .with_guarded_label_values(&[table_label, "UPDATE"])
             .local();
-        let iter_item = self.iter_item.with_label_values(&[table_label]).local();
-        let iter_size = self.iter_size.with_label_values(&[table_label]).local();
+        let delete_count = self
+            .iter_log_op_type_counts
+            .with_guarded_label_values(&[table_label, "DELETE"])
+            .local();
+        LocalIterLogMetrics {
+            iter_metrics,
+            insert_count,
+            update_count,
+            delete_count,
+            report_count: 0,
+        }
+    }
 
+    fn local_get_metrics(&self, table_label: &str) -> LocalGetMetrics {
         let get_duration = self.get_duration.with_label_values(&[table_label]).local();
         let get_key_size = self.get_key_size.with_label_values(&[table_label]).local();
         let get_value_size = self
@@ -247,12 +365,7 @@ impl MonitoredStorageMetrics {
             .with_label_values(&[table_label])
             .local();
 
-        LocalStorageMetrics {
-            iter_init_duration,
-            iter_scan_duration,
-            iter_in_process_counts,
-            iter_item,
-            iter_size,
+        LocalGetMetrics {
             get_duration,
             get_key_size,
             get_value_size,
@@ -261,28 +374,51 @@ impl MonitoredStorageMetrics {
     }
 }
 
-pub struct LocalStorageMetrics {
-    iter_init_duration: LabelGuardedMetric<LocalHistogram, 1>,
-    iter_scan_duration: LabelGuardedMetric<LocalHistogram, 1>,
-    iter_in_process_counts: GenericLocalCounter<AtomicU64>,
-    iter_item: LocalHistogram,
-    iter_size: LocalHistogram,
+struct LocalIterMetricsInner {
+    iter_init_duration: LabelGuardedLocalHistogram<2>,
+    iter_scan_duration: LabelGuardedLocalHistogram<2>,
+    iter_counts: LabelGuardedLocalIntCounter<2>,
+    iter_item: LabelGuardedLocalHistogram<2>,
+    iter_size: LabelGuardedLocalHistogram<2>,
+    iter_in_progress_counts: LabelGuardedIntGauge<2>,
+}
 
-    get_duration: LabelGuardedMetric<LocalHistogram, 1>,
-    get_key_size: LocalHistogram,
-    get_value_size: LocalHistogram,
+struct LocalIterMetrics {
+    inner: LocalIterMetricsInner,
     report_count: usize,
 }
 
-impl LocalStorageMetrics {
-    pub fn may_flush(&mut self) {
+impl LocalIterMetrics {
+    fn may_flush(&mut self) {
+        self.report_count += 1;
+        if self.report_count > MAX_FLUSH_TIMES {
+            self.inner.flush();
+            self.report_count = 0;
+        }
+    }
+}
+
+impl LocalIterMetricsInner {
+    fn flush(&mut self) {
+        self.iter_scan_duration.flush();
+        self.iter_init_duration.flush();
+        self.iter_counts.flush();
+        self.iter_item.flush();
+        self.iter_size.flush();
+    }
+}
+
+struct LocalGetMetrics {
+    get_duration: LabelGuardedLocalHistogram<1>,
+    get_key_size: LabelGuardedLocalHistogram<1>,
+    get_value_size: LabelGuardedLocalHistogram<1>,
+    report_count: usize,
+}
+
+impl LocalGetMetrics {
+    fn may_flush(&mut self) {
         self.report_count += 1;
         if self.report_count > MAX_FLUSH_TIMES {
-            self.iter_scan_duration.flush();
-            self.iter_init_duration.flush();
-            self.iter_in_process_counts.flush();
-            self.iter_item.flush();
-            self.iter_size.flush();
             self.get_duration.flush();
             self.get_key_size.flush();
             self.get_value_size.flush();
@@ -291,40 +427,115 @@ impl LocalStorageMetrics {
     }
 }
 
-pub struct MonitoredStateStoreIterStats {
+struct LocalIterLogMetrics {
+    iter_metrics: LocalIterMetricsInner,
+    insert_count: LabelGuardedLocalIntCounter<2>,
+    update_count: LabelGuardedLocalIntCounter<2>,
+    delete_count: LabelGuardedLocalIntCounter<2>,
+    report_count: usize,
+}
+
+impl LocalIterLogMetrics {
+    fn may_flush(&mut self) {
+        self.report_count += 1;
+        if self.report_count > MAX_FLUSH_TIMES {
+            self.iter_metrics.flush();
+            self.insert_count.flush();
+            self.update_count.flush();
+            self.delete_count.flush();
+            self.report_count = 0;
+        }
+    }
+}
+
+pub(crate) trait StateStoreIterStatsTrait: Send {
+    type Item: IterItem;
+    fn new(table_id: u32, metrics: &MonitoredStorageMetrics, iter_init_duration: Duration) -> Self;
+    fn observe(&mut self, item: <Self::Item as IterItem>::ItemRef<'_>);
+    fn report(&mut self, table_id: u32, metrics: &MonitoredStorageMetrics);
+}
+
+const MAX_FLUSH_TIMES: usize = 64;
+
+struct StateStoreIterStatsInner {
     pub iter_init_duration: Duration,
     pub iter_scan_time: Instant,
     pub total_items: usize,
     pub total_size: usize,
-    pub table_id: u32,
-    pub metrics: Arc<MonitoredStorageMetrics>,
 }
 
-thread_local!(static LOCAL_METRICS: RefCell<HashMap<u32, LocalStorageMetrics>> = RefCell::new(HashMap::default()));
-const MAX_FLUSH_TIMES: usize = 64;
-
-impl MonitoredStateStoreIterStats {
-    pub fn new(
-        table_id: u32,
-        iter_init_duration: Duration,
-        metrics: Arc<MonitoredStorageMetrics>,
-    ) -> Self {
+impl StateStoreIterStatsInner {
+    fn new(iter_init_duration: Duration) -> Self {
         Self {
             iter_init_duration,
             iter_scan_time: Instant::now(),
             total_items: 0,
             total_size: 0,
-            table_id,
-            metrics,
         }
     }
+}
 
-    pub fn report(&self) {
-        LOCAL_METRICS.with_borrow_mut(|local_metrics| {
-            let table_metrics = local_metrics.entry(self.table_id).or_insert_with(|| {
-                let table_label = self.table_id.to_string();
-                self.metrics.local_metrics(&table_label)
+pub(crate) struct MonitoredStateStoreIterStats<S: StateStoreIterStatsTrait> {
+    pub inner: S,
+    pub table_id: u32,
+    pub metrics: Arc<MonitoredStorageMetrics>,
+}
+
+impl<S: StateStoreIterStatsTrait> Drop for MonitoredStateStoreIterStats<S> {
+    fn drop(&mut self) {
+        self.inner.report(self.table_id, &self.metrics)
+    }
+}
+
+pub(crate) struct StateStoreIterStats {
+    inner: StateStoreIterStatsInner,
+}
+
+impl StateStoreIterStats {
+    fn for_table_metrics(
+        table_id: u32,
+        global_metrics: &MonitoredStorageMetrics,
+        f: impl FnOnce(&mut LocalIterMetrics),
+    ) {
+        thread_local!(static LOCAL_ITER_METRICS: RefCell<HashMap<u32, LocalIterMetrics>> = RefCell::new(HashMap::default()));
+        LOCAL_ITER_METRICS.with_borrow_mut(|local_metrics| {
+            let table_metrics = local_metrics.entry(table_id).or_insert_with(|| {
+                let table_label = table_id.to_string();
+                global_metrics.local_iter_metrics(&table_label)
             });
+            f(table_metrics)
+        });
+    }
+}
+
+impl StateStoreIterStatsTrait for StateStoreIterStats {
+    type Item = StateStoreIterItem;
+
+    fn new(table_id: u32, metrics: &MonitoredStorageMetrics, iter_init_duration: Duration) -> Self {
+        Self::for_table_metrics(table_id, metrics, |metrics| {
+            metrics.inner.iter_in_progress_counts.inc();
+        });
+        Self {
+            inner: StateStoreIterStatsInner::new(iter_init_duration),
+        }
+    }
+
+    fn observe(&mut self, (key, value): StateStoreIterItemRef<'_>) {
+        self.inner.total_items += 1;
+        self.inner.total_size += key.encoded_len() + value.len();
+    }
+
+    fn report(&mut self, table_id: u32, metrics: &MonitoredStorageMetrics) {
+        Self::for_table_metrics(table_id, metrics, |table_metrics| {
+            self.inner.apply_to_local(&mut table_metrics.inner);
+            table_metrics.may_flush();
+        });
+    }
+}
+
+impl StateStoreIterStatsInner {
+    fn apply_to_local(&self, table_metrics: &mut LocalIterMetricsInner) {
+        {
             let iter_scan_duration = self.iter_scan_time.elapsed();
             table_metrics
                 .iter_scan_duration
@@ -332,21 +543,87 @@ impl MonitoredStateStoreIterStats {
             table_metrics
                 .iter_init_duration
                 .observe(self.iter_init_duration.as_secs_f64());
-            table_metrics.iter_in_process_counts.inc();
+            table_metrics.iter_counts.inc();
             table_metrics.iter_item.observe(self.total_items as f64);
             table_metrics.iter_size.observe(self.total_size as f64);
-            table_metrics.may_flush();
+            table_metrics.iter_in_progress_counts.dec();
+        }
+    }
+}
+
+pub(crate) struct StateStoreIterLogStats {
+    inner: StateStoreIterStatsInner,
+    insert_count: u64,
+    update_count: u64,
+    delete_count: u64,
+}
+
+impl StateStoreIterLogStats {
+    fn for_table_metrics(
+        table_id: u32,
+        global_metrics: &MonitoredStorageMetrics,
+        f: impl FnOnce(&mut LocalIterLogMetrics),
+    ) {
+        thread_local!(static LOCAL_ITER_LOG_METRICS: RefCell<HashMap<u32, LocalIterLogMetrics>> = RefCell::new(HashMap::default()));
+        LOCAL_ITER_LOG_METRICS.with_borrow_mut(|local_metrics| {
+            let table_metrics = local_metrics.entry(table_id).or_insert_with(|| {
+                let table_label = table_id.to_string();
+                global_metrics.local_iter_log_metrics(&table_label)
+            });
+            f(table_metrics)
         });
     }
 }
 
-impl Drop for MonitoredStateStoreIterStats {
-    fn drop(&mut self) {
-        self.report();
+impl StateStoreIterStatsTrait for StateStoreIterLogStats {
+    type Item = StateStoreReadLogItem;
+
+    fn new(table_id: u32, metrics: &MonitoredStorageMetrics, iter_init_duration: Duration) -> Self {
+        Self::for_table_metrics(table_id, metrics, |metrics| {
+            metrics.iter_metrics.iter_in_progress_counts.inc();
+        });
+        Self {
+            inner: StateStoreIterStatsInner::new(iter_init_duration),
+            insert_count: 0,
+            update_count: 0,
+            delete_count: 0,
+        }
+    }
+
+    fn observe(&mut self, (key, change_log): StateStoreReadLogItemRef<'_>) {
+        self.inner.total_items += 1;
+        let value_len = match change_log {
+            ChangeLogValue::Insert(value) => {
+                self.insert_count += 1;
+                value.len()
+            }
+            ChangeLogValue::Update {
+                old_value,
+                new_value,
+            } => {
+                self.update_count += 1;
+                old_value.len() + new_value.len()
+            }
+            ChangeLogValue::Delete(value) => {
+                self.delete_count += 1;
+                value.len()
+            }
+        };
+        self.inner.total_size += key.len() + value_len;
+    }
+
+    fn report(&mut self, table_id: u32, metrics: &MonitoredStorageMetrics) {
+        Self::for_table_metrics(table_id, metrics, |table_metrics| {
+            self.inner.apply_to_local(&mut table_metrics.iter_metrics);
+            table_metrics.insert_count.inc_by(self.insert_count);
+            table_metrics.update_count.inc_by(self.update_count);
+            table_metrics.delete_count.inc_by(self.delete_count);
+            table_metrics.may_flush();
+        });
     }
 }
 
-pub struct MonitoredStateStoreGetStats {
+pub(crate) struct MonitoredStateStoreGetStats {
     pub get_duration: Instant,
     pub get_key_size: usize,
     pub get_value_size: usize,
@@ -355,7 +632,7 @@ pub struct MonitoredStateStoreGetStats {
 }
 
 impl MonitoredStateStoreGetStats {
-    pub fn new(table_id: u32, metrics: Arc<MonitoredStorageMetrics>) -> Self {
+    pub(crate) fn new(table_id: u32, metrics: Arc<MonitoredStorageMetrics>) -> Self {
         Self {
             get_duration: Instant::now(),
             get_key_size: 0,
@@ -365,11 +642,12 @@ impl MonitoredStateStoreGetStats {
         }
     }
 
-    pub fn report(&self) {
-        LOCAL_METRICS.with_borrow_mut(|local_metrics| {
+    pub(crate) fn report(&self) {
+        thread_local!(static LOCAL_GET_METRICS: RefCell<HashMap<u32, LocalGetMetrics>> = RefCell::new(HashMap::default()));
+        LOCAL_GET_METRICS.with_borrow_mut(|local_metrics| {
             let table_metrics = local_metrics.entry(self.table_id).or_insert_with(|| {
                 let table_label = self.table_id.to_string();
-                self.metrics.local_metrics(&table_label)
+                self.metrics.local_get_metrics(&table_label)
             });
             let get_duration = self.get_duration.elapsed();
             table_metrics
diff --git a/src/storage/src/monitor/monitored_store.rs b/src/storage/src/monitor/monitored_store.rs
index 79f5bf3bdcf1b..a571d47b799a2 100644
--- a/src/storage/src/monitor/monitored_store.rs
+++ b/src/storage/src/monitor/monitored_store.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::marker::PhantomData;
 use std::sync::Arc;
 
 use await_tree::InstrumentAwait;
@@ -20,7 +21,7 @@ use futures::{Future, TryFutureExt};
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::TableId;
 use risingwave_hummock_sdk::key::{TableKey, TableKeyRange};
-use risingwave_hummock_sdk::{HummockReadEpoch, SyncResult};
+use risingwave_hummock_sdk::HummockReadEpoch;
 use thiserror_ext::AsReport;
 use tokio::time::Instant;
 use tracing::{error, Instrument};
@@ -31,6 +32,8 @@ use super::{MonitoredStateStoreGetStats, MonitoredStateStoreIterStats, Monitored
 use crate::error::StorageResult;
 use crate::hummock::sstable_store::SstableStoreRef;
 use crate::hummock::{HummockStorage, SstableObjectIdManagerRef};
+use crate::monitor::monitored_storage_metrics::StateStoreIterStats;
+use crate::monitor::{StateStoreIterLogStats, StateStoreIterStatsTrait};
 use crate::store::*;
 /// A state store wrapper for monitoring metrics.
 #[derive(Clone)]
@@ -74,21 +77,21 @@ impl<S> MonitoredStateStore<S> {
     }
 }
 
-/// A util function to break the type connection between two opaque return types defined by `impl`.
-pub(crate) fn identity(input: impl StateStoreIter) -> impl StateStoreIter {
-    input
-}
-
 // Note: it is important to define the `MonitoredStateStoreIter` type alias, as it marks that
 // the return type of `monitored_iter` only captures the lifetime `'s` and has nothing to do with
 // `'a`. If we simply use `impl StateStoreIter + 's`, the rust compiler will also capture
 // the lifetime `'a` in the scope defined in the scope.
 impl<S> MonitoredStateStore<S> {
-    async fn monitored_iter<'a, St: StateStoreIter + 'a>(
+    async fn monitored_iter<
+        'a,
+        Item: IterItem,
+        I: StateStoreIter<Item> + 'a,
+        Stat: StateStoreIterStatsTrait<Item = Item>,
+    >(
         &'a self,
         table_id: TableId,
-        iter_stream_future: impl Future<Output = StorageResult<St>> + 'a,
-    ) -> StorageResult<MonitoredStateStoreIter<St>> {
+        iter_stream_future: impl Future<Output = StorageResult<I>> + 'a,
+    ) -> StorageResult<MonitoredStateStoreIter<Item, I, Stat>> {
         // start time takes iterator build time into account
         // wait for iterator creation (e.g. seek)
         let start_time = Instant::now();
@@ -100,11 +103,12 @@ impl<S> MonitoredStateStore<S> {
         // create a monitored iterator to collect metrics
         let monitored = MonitoredStateStoreIter {
             inner: iter_stream,
-            stats: MonitoredStateStoreIterStats::new(
-                table_id.table_id,
-                iter_init_duration,
-                self.storage_metrics.clone(),
-            ),
+            stats: MonitoredStateStoreIterStats {
+                inner: Stat::new(table_id.table_id, &self.storage_metrics, iter_init_duration),
+                table_id: table_id.table_id,
+                metrics: self.storage_metrics.clone(),
+            },
+            _phantom: PhantomData,
         };
         Ok(monitored)
     }
@@ -146,6 +150,7 @@ impl<S> MonitoredStateStore<S> {
 impl<S: StateStoreRead> StateStoreRead for MonitoredStateStore<S> {
     type ChangeLogIter = impl StateStoreReadChangeLogIter;
     type Iter = impl StateStoreReadIter;
+    type RevIter = impl StateStoreReadIter;
 
     fn get(
         &self,
@@ -164,11 +169,22 @@ impl<S: StateStoreRead> StateStoreRead for MonitoredStateStore<S> {
         epoch: u64,
         read_options: ReadOptions,
     ) -> impl Future<Output = StorageResult<Self::Iter>> + '_ {
-        self.monitored_iter(
+        self.monitored_iter::<'_, _, _, StateStoreIterStats>(
             read_options.table_id,
             self.inner.iter(key_range, epoch, read_options),
         )
-        .map_ok(identity)
+    }
+
+    fn rev_iter(
+        &self,
+        key_range: TableKeyRange,
+        epoch: u64,
+        read_options: ReadOptions,
+    ) -> impl Future<Output = StorageResult<Self::RevIter>> + '_ {
+        self.monitored_iter::<'_, _, _, StateStoreIterStats>(
+            read_options.table_id,
+            self.inner.rev_iter(key_range, epoch, read_options),
+        )
     }
 
     fn iter_log(
@@ -177,12 +193,16 @@ impl<S: StateStoreRead> StateStoreRead for MonitoredStateStore<S> {
         key_range: TableKeyRange,
         options: ReadLogOptions,
     ) -> impl Future<Output = StorageResult<Self::ChangeLogIter>> + Send + '_ {
-        self.inner.iter_log(epoch_range, key_range, options)
+        self.monitored_iter::<'_, _, _, StateStoreIterLogStats>(
+            options.table_id,
+            self.inner.iter_log(epoch_range, key_range, options),
+        )
     }
 }
 
 impl<S: LocalStateStore> LocalStateStore for MonitoredStateStore<S> {
     type Iter<'a> = impl StateStoreIter + 'a;
+    type RevIter<'a> = impl StateStoreIter + 'a;
 
     async fn may_exist(
         &self,
@@ -221,9 +241,22 @@ impl<S: LocalStateStore> LocalStateStore for MonitoredStateStore<S> {
         read_options: ReadOptions,
     ) -> impl Future<Output = StorageResult<Self::Iter<'_>>> + Send + '_ {
         let table_id = read_options.table_id;
-        // TODO: may collect the metrics as local
-        self.monitored_iter(table_id, self.inner.iter(key_range, read_options))
-            .map_ok(identity)
+        self.monitored_iter::<'_, _, _, StateStoreIterStats>(
+            table_id,
+            self.inner.iter(key_range, read_options),
+        )
+    }
+
+    fn rev_iter(
+        &self,
+        key_range: TableKeyRange,
+        read_options: ReadOptions,
+    ) -> impl Future<Output = StorageResult<Self::RevIter<'_>>> + Send + '_ {
+        let table_id = read_options.table_id;
+        self.monitored_iter::<'_, _, _, StateStoreIterStats>(
+            table_id,
+            self.inner.rev_iter(key_range, read_options),
+        )
     }
 
     fn insert(
@@ -286,23 +319,20 @@ impl<S: StateStore> StateStore for MonitoredStateStore<S> {
             .inspect_err(|e| error!(error = %e.as_report(), "Failed in wait_epoch"))
     }
 
-    async fn sync(&self, epoch: u64) -> StorageResult<SyncResult> {
-        // TODO: this metrics may not be accurate if we start syncing after `seal_epoch`. We may
-        // move this metrics to inside uploader
+    fn sync(&self, epoch: u64) -> impl SyncFuture {
+        let future = self.inner.sync(epoch).instrument_await("store_sync");
         let timer = self.storage_metrics.sync_duration.start_timer();
-        let sync_result = self
-            .inner
-            .sync(epoch)
-            .instrument_await("store_sync")
-            .await
-            .inspect_err(|e| error!(error = %e.as_report(), "Failed in sync"))?;
-        timer.observe_duration();
-        if sync_result.sync_size != 0 {
-            self.storage_metrics
-                .sync_size
-                .observe(sync_result.sync_size as _);
+        let sync_size = self.storage_metrics.sync_size.clone();
+        async move {
+            let sync_result = future
+                .await
+                .inspect_err(|e| error!(error = %e.as_report(), "Failed in sync"))?;
+            timer.observe_duration();
+            if sync_result.sync_size != 0 {
+                sync_size.observe(sync_result.sync_size as _);
+            }
+            Ok(sync_result)
         }
-        Ok(sync_result)
     }
 
     fn seal_epoch(&self, epoch: u64, is_checkpoint: bool) {
@@ -348,23 +378,29 @@ impl MonitoredStateStore<HummockStorage> {
 }
 
 /// A state store iterator wrapper for monitoring metrics.
-pub struct MonitoredStateStoreIter<S> {
-    inner: S,
-    stats: MonitoredStateStoreIterStats,
+pub(crate) struct MonitoredStateStoreIter<
+    Item: IterItem,
+    I,
+    S: StateStoreIterStatsTrait<Item = Item>,
+> {
+    inner: I,
+    stats: MonitoredStateStoreIterStats<S>,
+    _phantom: PhantomData<Item>,
 }
 
-impl<S: StateStoreIter> StateStoreIter for MonitoredStateStoreIter<S> {
-    async fn try_next(&mut self) -> StorageResult<Option<StateStoreIterItemRef<'_>>> {
-        if let Some((key, value)) = self
+impl<Item: IterItem, I: StateStoreIter<Item>, S: StateStoreIterStatsTrait<Item = Item>>
+    StateStoreIter<Item> for MonitoredStateStoreIter<Item, I, S>
+{
+    async fn try_next(&mut self) -> StorageResult<Option<Item::ItemRef<'_>>> {
+        if let Some(item) = self
             .inner
             .try_next()
             .instrument(tracing::trace_span!("store_iter_try_next"))
             .await
             .inspect_err(|e| error!(error = %e.as_report(), "Failed in next"))?
         {
-            self.stats.total_items += 1;
-            self.stats.total_size += key.encoded_len() + value.len();
-            Ok(Some((key, value)))
+            self.stats.inner.observe(item);
+            Ok(Some(item))
         } else {
             Ok(None)
         }
diff --git a/src/storage/src/monitor/traced_store.rs b/src/storage/src/monitor/traced_store.rs
index 47c0de67729b6..f22c854808114 100644
--- a/src/storage/src/monitor/traced_store.rs
+++ b/src/storage/src/monitor/traced_store.rs
@@ -11,25 +11,24 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 use std::sync::Arc;
 
 use bytes::Bytes;
-use futures::{Future, TryFutureExt};
+use futures::{Future, FutureExt};
 use risingwave_common::buffer::Bitmap;
 use risingwave_hummock_sdk::key::{TableKey, TableKeyRange};
-use risingwave_hummock_sdk::{HummockReadEpoch, SyncResult};
+use risingwave_hummock_sdk::HummockReadEpoch;
 use risingwave_hummock_trace::{
     init_collector, should_use_trace, ConcurrentId, MayTraceSpan, OperationResult, StorageType,
     TraceResult, TraceSpan, TracedBytes, TracedSealCurrentEpochOptions, LOCAL_ID,
 };
 use thiserror_ext::AsReport;
 
-use super::identity;
 use crate::error::StorageResult;
 use crate::hummock::sstable_store::SstableStoreRef;
 use crate::hummock::{HummockStorage, SstableObjectIdManagerRef};
 use crate::store::*;
-use crate::StateStore;
 
 #[derive(Clone)]
 pub struct TracedStateStore<S> {
@@ -107,6 +106,7 @@ impl<S> TracedStateStore<S> {
 
 impl<S: LocalStateStore> LocalStateStore for TracedStateStore<S> {
     type Iter<'a> = impl StateStoreIter + 'a;
+    type RevIter<'a> = impl StateStoreIter + 'a;
 
     fn may_exist(
         &self,
@@ -143,7 +143,22 @@ impl<S: LocalStateStore> LocalStateStore for TracedStateStore<S> {
             self.storage_type,
         );
         self.traced_iter(self.inner.iter(key_range, read_options), span)
-            .map_ok(identity)
+    }
+
+    fn rev_iter(
+        &self,
+        key_range: TableKeyRange,
+        read_options: ReadOptions,
+    ) -> impl Future<Output = StorageResult<Self::RevIter<'_>>> + Send + '_ {
+        let (l, r) = key_range.clone();
+        let bytes_key_range = (l.map(|l| l.0), r.map(|r| r.0));
+        let span = TraceSpan::new_iter_span(
+            bytes_key_range,
+            None,
+            read_options.clone().into(),
+            self.storage_type,
+        );
+        self.traced_iter(self.inner.rev_iter(key_range, read_options), span)
     }
 
     fn insert(
@@ -238,15 +253,17 @@ impl<S: StateStore> StateStore for TracedStateStore<S> {
         res
     }
 
-    async fn sync(&self, epoch: u64) -> StorageResult<SyncResult> {
+    fn sync(&self, epoch: u64) -> impl SyncFuture {
         let span: MayTraceSpan = TraceSpan::new_sync_span(epoch, self.storage_type);
 
-        let sync_result = self.inner.sync(epoch).await;
+        let future = self.inner.sync(epoch);
 
-        span.may_send_result(OperationResult::Sync(
-            sync_result.as_ref().map(|res| res.sync_size).into(),
-        ));
-        sync_result
+        future.map(move |sync_result| {
+            span.may_send_result(OperationResult::Sync(
+                sync_result.as_ref().map(|res| res.sync_size).into(),
+            ));
+            sync_result
+        })
     }
 
     fn seal_epoch(&self, epoch: u64, is_checkpoint: bool) {
@@ -276,6 +293,7 @@ impl<S: StateStore> StateStore for TracedStateStore<S> {
 impl<S: StateStoreRead> StateStoreRead for TracedStateStore<S> {
     type ChangeLogIter = impl StateStoreReadChangeLogIter;
     type Iter = impl StateStoreReadIter;
+    type RevIter = impl StateStoreReadIter;
 
     fn get(
         &self,
@@ -306,7 +324,23 @@ impl<S: StateStoreRead> StateStoreRead for TracedStateStore<S> {
             self.storage_type,
         );
         self.traced_iter(self.inner.iter(key_range, epoch, read_options), span)
-            .map_ok(identity)
+    }
+
+    fn rev_iter(
+        &self,
+        key_range: TableKeyRange,
+        epoch: u64,
+        read_options: ReadOptions,
+    ) -> impl Future<Output = StorageResult<Self::RevIter>> + '_ {
+        let (l, r) = key_range.clone();
+        let bytes_key_range = (l.map(|l| l.0), r.map(|r| r.0));
+        let span = TraceSpan::new_iter_span(
+            bytes_key_range,
+            Some(epoch),
+            read_options.clone().into(),
+            self.storage_type,
+        );
+        self.traced_iter(self.inner.rev_iter(key_range, epoch, read_options), span)
     }
 
     fn iter_log(
diff --git a/src/storage/src/opts.rs b/src/storage/src/opts.rs
index 7d7452f229b43..143b6bba37981 100644
--- a/src/storage/src/opts.rs
+++ b/src/storage/src/opts.rs
@@ -24,6 +24,8 @@ pub struct StorageOpts {
     pub parallel_compact_size_mb: u32,
     /// Target size of the Sstable.
     pub sstable_size_mb: u32,
+    /// Minimal target size of the Sstable to store data of different state-table in independent files as soon as possible.
+    pub min_sstable_size_mb: u32,
     /// Size of each block in bytes in SST.
     pub block_size_kb: u32,
     /// False positive probability of bloom filter.
@@ -39,8 +41,9 @@ pub struct StorageOpts {
     /// The shared buffer will start flushing data to object when the ratio of memory usage to the
     /// shared buffer capacity exceed such ratio.
     pub shared_buffer_flush_ratio: f32,
-    /// The threshold for the number of immutable memtables to merge to a new imm.
-    pub imm_merge_threshold: usize,
+    /// The minimum total flush size of shared buffer spill. When a shared buffer spill is trigger,
+    /// the total flush size across multiple epochs should be at least higher than this size.
+    pub shared_buffer_min_batch_flush_size_mb: usize,
     /// Remote directory for storing data and metadata objects.
     pub data_directory: String,
     /// Whether to enable write conflict detection
@@ -74,8 +77,6 @@ pub struct StorageOpts {
     pub sstable_id_remote_fetch_number: u32,
     /// Whether to enable streaming upload for sstable.
     pub min_sst_size_for_streaming_upload: u64,
-    /// Max sub compaction task numbers
-    pub max_sub_compaction: u32,
     pub max_concurrent_compaction_task_number: u64,
     pub max_version_pinning_duration_sec: u64,
     pub compactor_iter_max_io_retry_times: usize,
@@ -83,15 +84,11 @@ pub struct StorageOpts {
     pub data_file_cache_dir: String,
     pub data_file_cache_capacity_mb: usize,
     pub data_file_cache_file_capacity_mb: usize,
-    pub data_file_cache_device_align: usize,
-    pub data_file_cache_device_io_size: usize,
     pub data_file_cache_flushers: usize,
     pub data_file_cache_reclaimers: usize,
     pub data_file_cache_recover_concurrency: usize,
-    pub data_file_cache_lfu_window_to_cache_size_ratio: usize,
-    pub data_file_cache_lfu_tiny_lru_capacity_ratio: f64,
     pub data_file_cache_insert_rate_limit_mb: usize,
-    pub data_file_cache_catalog_bits: usize,
+    pub data_file_cache_indexer_shards: usize,
     pub data_file_cache_compression: String,
 
     pub cache_refill_data_refill_levels: Vec<u32>,
@@ -105,15 +102,11 @@ pub struct StorageOpts {
     pub meta_file_cache_dir: String,
     pub meta_file_cache_capacity_mb: usize,
     pub meta_file_cache_file_capacity_mb: usize,
-    pub meta_file_cache_device_align: usize,
-    pub meta_file_cache_device_io_size: usize,
     pub meta_file_cache_flushers: usize,
     pub meta_file_cache_reclaimers: usize,
     pub meta_file_cache_recover_concurrency: usize,
-    pub meta_file_cache_lfu_window_to_cache_size_ratio: usize,
-    pub meta_file_cache_lfu_tiny_lru_capacity_ratio: f64,
     pub meta_file_cache_insert_rate_limit_mb: usize,
-    pub meta_file_cache_catalog_bits: usize,
+    pub meta_file_cache_indexer_shards: usize,
     pub meta_file_cache_compression: String,
 
     /// The storage url for storing backups.
@@ -122,14 +115,6 @@ pub struct StorageOpts {
     pub backup_storage_directory: String,
     /// max time which wait for preload. 0 represent do not do any preload.
     pub max_preload_wait_time_mill: u64,
-    /// object store streaming read timeout.
-    pub object_store_streaming_read_timeout_ms: u64,
-    /// object store streaming upload timeout.
-    pub object_store_streaming_upload_timeout_ms: u64,
-    /// object store upload timeout.
-    pub object_store_upload_timeout_ms: u64,
-    /// object store read timeout.
-    pub object_store_read_timeout_ms: u64,
 
     pub compactor_max_sst_key_count: u64,
     pub compactor_max_task_multiplier: f32,
@@ -143,6 +128,8 @@ pub struct StorageOpts {
 
     pub mem_table_spill_threshold: usize,
 
+    pub compactor_concurrent_uploading_sst_count: Option<usize>,
+
     pub object_store_config: ObjectStoreConfig,
 }
 
@@ -160,6 +147,7 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt
         Self {
             parallel_compact_size_mb: p.parallel_compact_size_mb(),
             sstable_size_mb: p.sstable_size_mb(),
+            min_sstable_size_mb: c.storage.min_sstable_size_mb,
             block_size_kb: p.block_size_kb(),
             bloom_false_positive: p.bloom_false_positive(),
             share_buffers_sync_parallelism: c.storage.share_buffers_sync_parallelism,
@@ -168,7 +156,7 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt
                 .share_buffer_compaction_worker_threads_number,
             shared_buffer_capacity_mb: s.shared_buffer_capacity_mb,
             shared_buffer_flush_ratio: c.storage.shared_buffer_flush_ratio,
-            imm_merge_threshold: c.storage.imm_merge_threshold,
+            shared_buffer_min_batch_flush_size_mb: c.storage.shared_buffer_min_batch_flush_size_mb,
             data_directory: p.data_directory().to_string(),
             write_conflict_detection_enabled: c.storage.write_conflict_detection_enabled,
             block_cache_capacity_mb: s.block_cache_capacity_mb,
@@ -184,46 +172,25 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt
             compactor_memory_limit_mb: s.compactor_memory_limit_mb,
             sstable_id_remote_fetch_number: c.storage.sstable_id_remote_fetch_number,
             min_sst_size_for_streaming_upload: c.storage.min_sst_size_for_streaming_upload,
-            max_sub_compaction: c.storage.max_sub_compaction,
             max_concurrent_compaction_task_number: c.storage.max_concurrent_compaction_task_number,
             max_version_pinning_duration_sec: c.storage.max_version_pinning_duration_sec,
             data_file_cache_dir: c.storage.data_file_cache.dir.clone(),
             data_file_cache_capacity_mb: c.storage.data_file_cache.capacity_mb,
             data_file_cache_file_capacity_mb: c.storage.data_file_cache.file_capacity_mb,
-            data_file_cache_device_align: c.storage.data_file_cache.device_align,
-            data_file_cache_device_io_size: c.storage.data_file_cache.device_io_size,
             data_file_cache_flushers: c.storage.data_file_cache.flushers,
             data_file_cache_reclaimers: c.storage.data_file_cache.reclaimers,
             data_file_cache_recover_concurrency: c.storage.data_file_cache.recover_concurrency,
-            data_file_cache_lfu_window_to_cache_size_ratio: c
-                .storage
-                .data_file_cache
-                .lfu_window_to_cache_size_ratio,
-            data_file_cache_lfu_tiny_lru_capacity_ratio: c
-                .storage
-                .data_file_cache
-                .lfu_tiny_lru_capacity_ratio,
             data_file_cache_insert_rate_limit_mb: c.storage.data_file_cache.insert_rate_limit_mb,
-            data_file_cache_catalog_bits: c.storage.data_file_cache.catalog_bits,
+            data_file_cache_indexer_shards: c.storage.data_file_cache.indexer_shards,
             data_file_cache_compression: c.storage.data_file_cache.compression.clone(),
             meta_file_cache_dir: c.storage.meta_file_cache.dir.clone(),
             meta_file_cache_capacity_mb: c.storage.meta_file_cache.capacity_mb,
             meta_file_cache_file_capacity_mb: c.storage.meta_file_cache.file_capacity_mb,
-            meta_file_cache_device_align: c.storage.meta_file_cache.device_align,
-            meta_file_cache_device_io_size: c.storage.meta_file_cache.device_io_size,
             meta_file_cache_flushers: c.storage.meta_file_cache.flushers,
             meta_file_cache_reclaimers: c.storage.meta_file_cache.reclaimers,
             meta_file_cache_recover_concurrency: c.storage.meta_file_cache.recover_concurrency,
-            meta_file_cache_lfu_window_to_cache_size_ratio: c
-                .storage
-                .meta_file_cache
-                .lfu_window_to_cache_size_ratio,
-            meta_file_cache_lfu_tiny_lru_capacity_ratio: c
-                .storage
-                .meta_file_cache
-                .lfu_tiny_lru_capacity_ratio,
             meta_file_cache_insert_rate_limit_mb: c.storage.meta_file_cache.insert_rate_limit_mb,
-            meta_file_cache_catalog_bits: c.storage.meta_file_cache.catalog_bits,
+            meta_file_cache_indexer_shards: c.storage.meta_file_cache.indexer_shards,
             meta_file_cache_compression: c.storage.meta_file_cache.compression.clone(),
             cache_refill_data_refill_levels: c.storage.cache_refill.data_refill_levels.clone(),
             cache_refill_timeout_ms: c.storage.cache_refill.timeout_ms,
@@ -236,17 +203,8 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt
             cache_refill_unit: c.storage.cache_refill.unit,
             cache_refill_threshold: c.storage.cache_refill.threshold,
             max_preload_wait_time_mill: c.storage.max_preload_wait_time_mill,
-            object_store_streaming_read_timeout_ms: c
-                .storage
-                .object_store
-                .object_store_streaming_read_timeout_ms,
             compact_iter_recreate_timeout_ms: c.storage.compact_iter_recreate_timeout_ms,
-            object_store_streaming_upload_timeout_ms: c
-                .storage
-                .object_store
-                .object_store_streaming_upload_timeout_ms,
-            object_store_read_timeout_ms: c.storage.object_store.object_store_read_timeout_ms,
-            object_store_upload_timeout_ms: c.storage.object_store.object_store_upload_timeout_ms,
+
             max_preload_io_retry_times: c.storage.max_preload_io_retry_times,
             backup_storage_url: p.backup_storage_url().to_string(),
             backup_storage_directory: p.backup_storage_directory().to_string(),
@@ -262,6 +220,9 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt
                 .compactor_fast_max_compact_delete_ratio,
             compactor_fast_max_compact_task_size: c.storage.compactor_fast_max_compact_task_size,
             compactor_iter_max_io_retry_times: c.storage.compactor_iter_max_io_retry_times,
+            compactor_concurrent_uploading_sst_count: c
+                .storage
+                .compactor_concurrent_uploading_sst_count,
         }
     }
 }
diff --git a/src/storage/src/panic_store.rs b/src/storage/src/panic_store.rs
index 7c9645697caa1..9f3a428442040 100644
--- a/src/storage/src/panic_store.rs
+++ b/src/storage/src/panic_store.rs
@@ -19,7 +19,7 @@ use std::sync::Arc;
 use bytes::Bytes;
 use risingwave_common::buffer::Bitmap;
 use risingwave_hummock_sdk::key::{TableKey, TableKeyRange};
-use risingwave_hummock_sdk::{HummockReadEpoch, SyncResult};
+use risingwave_hummock_sdk::HummockReadEpoch;
 
 use crate::error::StorageResult;
 use crate::storage_value::StorageValue;
@@ -33,6 +33,7 @@ pub struct PanicStateStore;
 impl StateStoreRead for PanicStateStore {
     type ChangeLogIter = PanicStateStoreIter<StateStoreReadLogItem>;
     type Iter = PanicStateStoreIter<StateStoreIterItem>;
+    type RevIter = PanicStateStoreIter<StateStoreIterItem>;
 
     #[allow(clippy::unused_async)]
     async fn get(
@@ -54,6 +55,16 @@ impl StateStoreRead for PanicStateStore {
         panic!("should not read from the state store!");
     }
 
+    #[allow(clippy::unused_async)]
+    async fn rev_iter(
+        &self,
+        _key_range: TableKeyRange,
+        _epoch: u64,
+        _read_options: ReadOptions,
+    ) -> StorageResult<Self::RevIter> {
+        panic!("should not read from the state store!");
+    }
+
     async fn iter_log(
         &self,
         _epoch_range: (u64, u64),
@@ -77,6 +88,7 @@ impl StateStoreWrite for PanicStateStore {
 
 impl LocalStateStore for PanicStateStore {
     type Iter<'a> = PanicStateStoreIter<StateStoreIterItem>;
+    type RevIter<'a> = PanicStateStoreIter<StateStoreIterItem>;
 
     #[allow(clippy::unused_async)]
     async fn may_exist(
@@ -105,6 +117,15 @@ impl LocalStateStore for PanicStateStore {
         panic!("should not operate on the panic state store!");
     }
 
+    #[allow(clippy::unused_async)]
+    async fn rev_iter(
+        &self,
+        _key_range: TableKeyRange,
+        _read_options: ReadOptions,
+    ) -> StorageResult<Self::RevIter<'_>> {
+        panic!("should not operate on the panic state store!");
+    }
+
     fn insert(
         &mut self,
         _key: TableKey<Bytes>,
@@ -159,8 +180,8 @@ impl StateStore for PanicStateStore {
     }
 
     #[allow(clippy::unused_async)]
-    async fn sync(&self, _epoch: u64) -> StorageResult<SyncResult> {
-        panic!("should not await sync epoch from the panic state store!");
+    fn sync(&self, _epoch: u64) -> impl SyncFuture {
+        async { panic!("should not await sync epoch from the panic state store!") }
     }
 
     fn seal_epoch(&self, _epoch: u64, _is_checkpoint: bool) {
diff --git a/src/storage/src/row_serde/value_serde.rs b/src/storage/src/row_serde/value_serde.rs
index bde7091597605..c4d4ef8b808f7 100644
--- a/src/storage/src/row_serde/value_serde.rs
+++ b/src/storage/src/row_serde/value_serde.rs
@@ -145,7 +145,6 @@ mod tests {
     use std::collections::HashSet;
 
     use risingwave_common::catalog::ColumnId;
-    use risingwave_common::row::OwnedRow;
     use risingwave_common::types::ScalarImpl::*;
     use risingwave_common::util::value_encoding::column_aware_row_encoding;
     use risingwave_common::util::value_encoding::column_aware_row_encoding::try_drop_invalid_columns;
diff --git a/src/storage/src/storage_failpoints/test_iterator.rs b/src/storage/src/storage_failpoints/test_iterator.rs
index 463c20ed469de..2a9fb64744371 100644
--- a/src/storage/src/storage_failpoints/test_iterator.rs
+++ b/src/storage/src/storage_failpoints/test_iterator.rs
@@ -41,7 +41,7 @@ use crate::monitor::StoreLocalStatistic;
 async fn test_failpoints_concat_read_err() {
     fail::cfg("disable_block_cache", "return").unwrap();
     let mem_read_err = "mem_read_err";
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     let table0 = gen_iterator_test_sstable_info(
         0,
         default_builder_opt_for_test(),
@@ -101,7 +101,7 @@ async fn test_failpoints_concat_read_err() {
 async fn test_failpoints_backward_concat_read_err() {
     fail::cfg("disable_block_cache", "return").unwrap();
     let mem_read_err = "mem_read_err";
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     let table0 = gen_iterator_test_sstable_info(
         0,
         default_builder_opt_for_test(),
@@ -157,7 +157,7 @@ async fn test_failpoints_backward_concat_read_err() {
 async fn test_failpoints_merge_invalid_key() {
     fail::cfg("disable_block_cache", "return").unwrap();
     let mem_read_err = "mem_read_err";
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     let table0 = gen_iterator_test_sstable_base(
         0,
         default_builder_opt_for_test(),
@@ -205,7 +205,7 @@ async fn test_failpoints_merge_invalid_key() {
 async fn test_failpoints_backward_merge_invalid_key() {
     fail::cfg("disable_block_cache", "return").unwrap();
     let mem_read_err = "mem_read_err";
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     let table0 = gen_iterator_test_sstable_base(
         0,
         default_builder_opt_for_test(),
@@ -249,7 +249,7 @@ async fn test_failpoints_backward_merge_invalid_key() {
 async fn test_failpoints_user_read_err() {
     fail::cfg("disable_block_cache", "return").unwrap();
     let mem_read_err = "mem_read_err";
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     let table0 = gen_iterator_test_sstable_base(
         0,
         default_builder_opt_for_test(),
@@ -309,7 +309,7 @@ async fn test_failpoints_user_read_err() {
 async fn test_failpoints_backward_user_read_err() {
     fail::cfg("disable_block_cache", "return").unwrap();
     let mem_read_err = "mem_read_err";
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     let table0 = gen_iterator_test_sstable_base(
         0,
         default_builder_opt_for_test(),
@@ -341,7 +341,7 @@ async fn test_failpoints_backward_user_read_err() {
         i -= 1;
         let key = ui.key();
         let val = ui.value();
-        assert_eq!(key, &iterator_test_bytes_key_of(i));
+        assert_eq!(key, iterator_test_bytes_key_of(i).to_ref());
         assert_eq!(val, iterator_test_value_of(i).as_slice());
         let result = ui.next().await;
         if result.is_err() {
@@ -362,7 +362,7 @@ async fn test_failpoints_compactor_iterator_recreate() {
     let get_stream_err = "get_stream_err";
     let stream_read_err = "stream_read_err";
     let create_stream_err = "create_stream_err";
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     // when upload data is successful, but upload meta is fail and delete is fail
     let has_create = Arc::new(AtomicBool::new(false));
     fail::cfg_callback(get_stream_err, move || {
diff --git a/src/storage/src/storage_failpoints/test_sstable.rs b/src/storage/src/storage_failpoints/test_sstable.rs
index 3361bdf55cde5..c89791767b4d0 100644
--- a/src/storage/src/storage_failpoints/test_sstable.rs
+++ b/src/storage/src/storage_failpoints/test_sstable.rs
@@ -35,7 +35,7 @@ use crate::monitor::StoreLocalStatistic;
 async fn test_failpoints_table_read() {
     let mem_read_err_fp = "mem_read_err";
     // build remote table
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
 
     // We should close buffer, so that table iterator must read in object_stores
     let kv_iter =
@@ -84,7 +84,7 @@ async fn test_failpoints_vacuum_and_metadata() {
     let data_upload_err = "data_upload_err";
     let mem_upload_err = "mem_upload_err";
     let mem_delete_err = "mem_delete_err";
-    let sstable_store = mock_sstable_store();
+    let sstable_store = mock_sstable_store().await;
     // when upload data is successful, but upload meta is fail and delete is fail
 
     fail::cfg_callback(data_upload_err, move || {
diff --git a/src/storage/src/store.rs b/src/storage/src/store.rs
index 95705d154af6c..beb19a0a98c5d 100644
--- a/src/storage/src/store.rs
+++ b/src/storage/src/store.rs
@@ -44,7 +44,7 @@ use crate::storage_value::StorageValue;
 pub trait StaticSendSync = Send + Sync + 'static;
 
 pub trait IterItem: Send + 'static {
-    type ItemRef<'a>: Send + 'a;
+    type ItemRef<'a>: Send + Copy + 'a;
 }
 
 impl IterItem for StateStoreIterItem {
@@ -66,7 +66,8 @@ pub fn to_owned_item((key, value): StateStoreIterItemRef<'_>) -> StorageResult<S
 }
 
 pub trait StateStoreIterExt<T: IterItem = StateStoreIterItem>: StateStoreIter<T> + Sized {
-    type ItemStream<O: Send, F: Send>: Stream<Item = StorageResult<O>> + Send;
+    type ItemStream<O: Send, F: Send + for<'a> Fn(T::ItemRef<'a>) -> StorageResult<O>>: Stream<Item = StorageResult<O>>
+        + Send;
 
     fn into_stream<O: Send, F: for<'a> Fn(T::ItemRef<'a>) -> StorageResult<O> + Send>(
         self,
@@ -151,7 +152,8 @@ impl<T: IterItem, I: StateStoreIter<T>> FusedStateStoreIter<I, T> {
 }
 
 impl<T: IterItem, I: StateStoreIter<T>> StateStoreIterExt<T> for I {
-    type ItemStream<O: Send, F: Send> = impl Stream<Item = StorageResult<O>> + Send;
+    type ItemStream<O: Send, F: Send + for<'a> Fn(T::ItemRef<'a>) -> StorageResult<O>> =
+        impl Stream<Item = StorageResult<O>> + Send;
 
     fn into_stream<O: Send, F: for<'a> Fn(T::ItemRef<'a>) -> StorageResult<O> + Send>(
         self,
@@ -215,6 +217,7 @@ pub trait StateStoreReadChangeLogIter = StateStoreIter<StateStoreReadLogItem> +
 
 pub trait StateStoreRead: StaticSendSync {
     type Iter: StateStoreReadIter;
+    type RevIter: StateStoreReadIter;
     type ChangeLogIter: StateStoreReadChangeLogIter;
 
     /// Point gets a value from the state store.
@@ -238,6 +241,13 @@ pub trait StateStoreRead: StaticSendSync {
         read_options: ReadOptions,
     ) -> impl Future<Output = StorageResult<Self::Iter>> + Send + '_;
 
+    fn rev_iter(
+        &self,
+        key_range: TableKeyRange,
+        epoch: u64,
+        read_options: ReadOptions,
+    ) -> impl Future<Output = StorageResult<Self::RevIter>> + Send + '_;
+
     fn iter_log(
         &self,
         epoch_range: (u64, u64),
@@ -308,6 +318,8 @@ pub trait StateStoreWrite: StaticSendSync {
     ) -> StorageResult<usize>;
 }
 
+pub trait SyncFuture = Future<Output = StorageResult<SyncResult>> + Send + 'static;
+
 pub trait StateStore: StateStoreRead + StaticSendSync + Clone {
     type Local: LocalStateStore;
 
@@ -318,7 +330,7 @@ pub trait StateStore: StateStoreRead + StaticSendSync + Clone {
         epoch: HummockReadEpoch,
     ) -> impl Future<Output = StorageResult<()>> + Send + '_;
 
-    fn sync(&self, epoch: u64) -> impl Future<Output = StorageResult<SyncResult>> + Send + '_;
+    fn sync(&self, epoch: u64) -> impl SyncFuture;
 
     /// update max current epoch in storage.
     fn seal_epoch(&self, epoch: u64, is_checkpoint: bool);
@@ -343,6 +355,7 @@ pub trait StateStore: StateStoreRead + StaticSendSync + Clone {
 /// table.
 pub trait LocalStateStore: StaticSendSync {
     type Iter<'a>: StateStoreIter + 'a;
+    type RevIter<'a>: StateStoreIter + 'a;
 
     /// Point gets a value from the state store.
     /// The result is based on the latest written snapshot.
@@ -363,6 +376,12 @@ pub trait LocalStateStore: StaticSendSync {
         read_options: ReadOptions,
     ) -> impl Future<Output = StorageResult<Self::Iter<'_>>> + Send + '_;
 
+    fn rev_iter(
+        &self,
+        key_range: TableKeyRange,
+        read_options: ReadOptions,
+    ) -> impl Future<Output = StorageResult<Self::RevIter<'_>>> + Send + '_;
+
     /// Inserts a key-value entry associated with a given `epoch` into the state store.
     fn insert(
         &mut self,
@@ -547,16 +566,21 @@ pub static CHECK_BYTES_EQUAL: LazyLock<Arc<dyn CheckOldValueEquality>> =
 pub enum OpConsistencyLevel {
     #[default]
     Inconsistent,
-    ConsistentOldValue(Arc<dyn CheckOldValueEquality>),
+    ConsistentOldValue {
+        check_old_value: Arc<dyn CheckOldValueEquality>,
+        /// whether should store the old value
+        is_log_store: bool,
+    },
 }
 
 impl Debug for OpConsistencyLevel {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
             OpConsistencyLevel::Inconsistent => f.write_str("OpConsistencyLevel::Inconsistent"),
-            OpConsistencyLevel::ConsistentOldValue(_) => {
-                f.write_str("OpConsistencyLevel::ConsistentOldValue")
-            }
+            OpConsistencyLevel::ConsistentOldValue { is_log_store, .. } => f
+                .debug_struct("OpConsistencyLevel::ConsistentOldValue")
+                .field("is_log_store", is_log_store)
+                .finish(),
         }
     }
 }
@@ -569,8 +593,23 @@ impl PartialEq<Self> for OpConsistencyLevel {
                 OpConsistencyLevel::Inconsistent,
                 OpConsistencyLevel::Inconsistent
             ) | (
-                OpConsistencyLevel::ConsistentOldValue(_),
-                OpConsistencyLevel::ConsistentOldValue(_),
+                OpConsistencyLevel::ConsistentOldValue {
+                    is_log_store: true,
+                    ..
+                },
+                OpConsistencyLevel::ConsistentOldValue {
+                    is_log_store: true,
+                    ..
+                },
+            ) | (
+                OpConsistencyLevel::ConsistentOldValue {
+                    is_log_store: false,
+                    ..
+                },
+                OpConsistencyLevel::ConsistentOldValue {
+                    is_log_store: false,
+                    ..
+                },
             )
         )
     }
@@ -591,10 +630,10 @@ pub struct NewLocalOptions {
     /// Whether the operation is consistent. The term `consistent` requires the following:
     ///
     /// 1. A key cannot be inserted or deleted for more than once, i.e. inserting to an existing
-    /// key or deleting an non-existing key is not allowed.
+    ///    key or deleting an non-existing key is not allowed.
     ///
     /// 2. The old value passed from
-    /// `update` and `delete` should match the original stored value.
+    ///    `update` and `delete` should match the original stored value.
     pub op_consistency_level: OpConsistencyLevel,
     pub table_option: TableOption,
 
@@ -613,7 +652,11 @@ impl From<TracedNewLocalOptions> for NewLocalOptions {
             op_consistency_level: match value.op_consistency_level {
                 TracedOpConsistencyLevel::Inconsistent => OpConsistencyLevel::Inconsistent,
                 TracedOpConsistencyLevel::ConsistentOldValue => {
-                    OpConsistencyLevel::ConsistentOldValue(CHECK_BYTES_EQUAL.clone())
+                    OpConsistencyLevel::ConsistentOldValue {
+                        check_old_value: CHECK_BYTES_EQUAL.clone(),
+                        // TODO: for simplicity, set it to false
+                        is_log_store: false,
+                    }
                 }
             },
             table_option: value.table_option.into(),
@@ -629,7 +672,7 @@ impl From<NewLocalOptions> for TracedNewLocalOptions {
             table_id: value.table_id.into(),
             op_consistency_level: match value.op_consistency_level {
                 OpConsistencyLevel::Inconsistent => TracedOpConsistencyLevel::Inconsistent,
-                OpConsistencyLevel::ConsistentOldValue(_) => {
+                OpConsistencyLevel::ConsistentOldValue { .. } => {
                     TracedOpConsistencyLevel::ConsistentOldValue
                 }
             },
@@ -731,7 +774,7 @@ impl From<SealCurrentEpochOptions> for TracedSealCurrentEpochOptions {
             }),
             switch_op_consistency_level: value
                 .switch_op_consistency_level
-                .map(|level| matches!(level, OpConsistencyLevel::ConsistentOldValue(_))),
+                .map(|level| matches!(level, OpConsistencyLevel::ConsistentOldValue { .. })),
         }
     }
 }
@@ -758,7 +801,10 @@ impl From<TracedSealCurrentEpochOptions> for SealCurrentEpochOptions {
             }),
             switch_op_consistency_level: value.switch_op_consistency_level.map(|enable| {
                 if enable {
-                    OpConsistencyLevel::ConsistentOldValue(CHECK_BYTES_EQUAL.clone())
+                    OpConsistencyLevel::ConsistentOldValue {
+                        check_old_value: CHECK_BYTES_EQUAL.clone(),
+                        is_log_store: false,
+                    }
                 } else {
                     OpConsistencyLevel::Inconsistent
                 }
diff --git a/src/storage/src/store_impl.rs b/src/storage/src/store_impl.rs
index 7fdb838654b70..fdecccc85893e 100644
--- a/src/storage/src/store_impl.rs
+++ b/src/storage/src/store_impl.rs
@@ -13,22 +13,25 @@
 // limitations under the License.
 
 use std::fmt::Debug;
-use std::path::PathBuf;
+use std::ops::Deref;
 use std::sync::Arc;
 use std::time::Duration;
 
 use enum_as_inner::EnumAsInner;
+use foyer::{
+    DirectFsDeviceOptionsBuilder, HybridCacheBuilder, RateLimitPicker, RuntimeConfigBuilder,
+};
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
 use risingwave_common_service::observer_manager::RpcNotificationClient;
+use risingwave_hummock_sdk::HummockSstableObjectId;
 use risingwave_object_store::object::build_remote_object_store;
 
 use crate::error::StorageResult;
 use crate::filter_key_extractor::{RemoteTableAccessor, RpcFilterKeyExtractorManager};
-use crate::hummock::file_cache::preclude::*;
 use crate::hummock::hummock_meta_client::MonitoredHummockMetaClient;
 use crate::hummock::{
-    set_foyer_metrics_registry, FileCache, FileCacheConfig, HummockError, HummockStorage,
-    RecentFilter, SstableStore, SstableStoreConfig,
+    Block, BlockCacheEventListener, HummockError, HummockStorage, RecentFilter, Sstable,
+    SstableBlockIndex, SstableStore, SstableStoreConfig,
 };
 use crate::memory::sled::SledStateStore;
 use crate::memory::MemoryStateStore;
@@ -201,7 +204,7 @@ macro_rules! dispatch_state_store {
     }};
 }
 
-#[cfg(debug_assertions)]
+#[cfg(any(debug_assertions, test, feature = "test"))]
 pub mod verify {
     use std::fmt::Debug;
     use std::future::Future;
@@ -212,7 +215,7 @@ pub mod verify {
     use bytes::Bytes;
     use risingwave_common::buffer::Bitmap;
     use risingwave_hummock_sdk::key::{TableKey, TableKeyRange};
-    use risingwave_hummock_sdk::{HummockReadEpoch, SyncResult};
+    use risingwave_hummock_sdk::HummockReadEpoch;
     use tracing::log::warn;
 
     use crate::error::StorageResult;
@@ -220,7 +223,6 @@ pub mod verify {
     use crate::storage_value::StorageValue;
     use crate::store::*;
     use crate::store_impl::AsHummock;
-    use crate::StateStore;
 
     fn assert_result_eq<Item: PartialEq + Debug, E>(
         first: &std::result::Result<Item, E>,
@@ -256,6 +258,7 @@ pub mod verify {
     impl<A: StateStoreRead, E: StateStoreRead> StateStoreRead for VerifyStateStore<A, E> {
         type ChangeLogIter = impl StateStoreReadChangeLogIter;
         type Iter = impl StateStoreReadIter;
+        type RevIter = impl StateStoreReadIter;
 
         async fn get(
             &self,
@@ -298,6 +301,28 @@ pub mod verify {
             }
         }
 
+        #[allow(clippy::manual_async_fn)]
+        fn rev_iter(
+            &self,
+            key_range: TableKeyRange,
+            epoch: u64,
+            read_options: ReadOptions,
+        ) -> impl Future<Output = StorageResult<Self::RevIter>> + '_ {
+            async move {
+                let actual = self
+                    .actual
+                    .rev_iter(key_range.clone(), epoch, read_options.clone())
+                    .await?;
+                let expected = if let Some(expected) = &self.expected {
+                    Some(expected.rev_iter(key_range, epoch, read_options).await?)
+                } else {
+                    None
+                };
+
+                Ok(verify_iter::<StateStoreIterItem>(actual, expected))
+            }
+        }
+
         async fn iter_log(
             &self,
             epoch_range: (u64, u64),
@@ -379,6 +404,7 @@ pub mod verify {
 
     impl<A: LocalStateStore, E: LocalStateStore> LocalStateStore for VerifyStateStore<A, E> {
         type Iter<'a> = impl StateStoreIter + 'a;
+        type RevIter<'a> = impl StateStoreIter + 'a;
 
         // We don't verify `may_exist` across different state stores because
         // the return value of `may_exist` is implementation specific and may not
@@ -425,6 +451,27 @@ pub mod verify {
             }
         }
 
+        #[allow(clippy::manual_async_fn)]
+        fn rev_iter(
+            &self,
+            key_range: TableKeyRange,
+            read_options: ReadOptions,
+        ) -> impl Future<Output = StorageResult<Self::RevIter<'_>>> + Send + '_ {
+            async move {
+                let actual = self
+                    .actual
+                    .rev_iter(key_range.clone(), read_options.clone())
+                    .await?;
+                let expected = if let Some(expected) = &self.expected {
+                    Some(expected.rev_iter(key_range, read_options).await?)
+                } else {
+                    None
+                };
+
+                Ok(verify_iter::<StateStoreIterItem>(actual, expected))
+            }
+        }
+
         fn insert(
             &mut self,
             key: TableKey<Bytes>,
@@ -511,11 +558,15 @@ pub mod verify {
             self.actual.try_wait_epoch(epoch)
         }
 
-        async fn sync(&self, epoch: u64) -> StorageResult<SyncResult> {
-            if let Some(expected) = &self.expected {
-                let _ = expected.sync(epoch).await;
+        fn sync(&self, epoch: u64) -> impl SyncFuture {
+            let expected_future = self.expected.as_ref().map(|expected| expected.sync(epoch));
+            let actual_future = self.actual.sync(epoch);
+            async move {
+                if let Some(expected_future) = expected_future {
+                    expected_future.await?;
+                }
+                actual_future.await
             }
-            self.actual.sync(epoch).await
         }
 
         fn seal_epoch(&self, epoch: u64, is_checkpoint: bool) {
@@ -556,6 +607,7 @@ pub mod verify {
 impl StateStoreImpl {
     #[cfg_attr(not(target_os = "linux"), allow(unused_variables))]
     #[allow(clippy::too_many_arguments)]
+    #[expect(clippy::borrowed_box)]
     pub async fn new(
         s: &str,
         opts: Arc<StorageOpts>,
@@ -566,75 +618,121 @@ impl StateStoreImpl {
         compactor_metrics: Arc<CompactorMetrics>,
         await_tree_config: Option<await_tree::Config>,
     ) -> StorageResult<Self> {
-        set_foyer_metrics_registry(GLOBAL_METRICS_REGISTRY.clone());
+        const MB: usize = 1 << 20;
+
+        if cfg!(not(madsim)) {
+            metrics_prometheus::Recorder::builder()
+                .with_registry(GLOBAL_METRICS_REGISTRY.deref().clone())
+                .build_and_install();
+        }
+
+        let meta_cache = {
+            let mut builder = HybridCacheBuilder::new()
+                .with_name("foyer.meta")
+                .memory(opts.meta_cache_capacity_mb * MB)
+                .with_shards(opts.meta_cache_shard_num)
+                .with_eviction_config(opts.meta_cache_eviction_config.clone())
+                .with_object_pool_capacity(1024 * opts.meta_cache_shard_num)
+                .with_weighter(|_: &HummockSstableObjectId, value: &Box<Sstable>| {
+                    u64::BITS as usize / 8 + value.estimate_size()
+                })
+                .storage();
+
+            if !opts.meta_file_cache_dir.is_empty() {
+                builder = builder
+                    .with_device_config(
+                        DirectFsDeviceOptionsBuilder::new(&opts.meta_file_cache_dir)
+                            .with_capacity(opts.meta_file_cache_capacity_mb * MB)
+                            .with_file_size(opts.meta_file_cache_file_capacity_mb * MB)
+                            .build(),
+                    )
+                    .with_indexer_shards(opts.meta_file_cache_indexer_shards)
+                    .with_flushers(opts.meta_file_cache_flushers)
+                    .with_reclaimers(opts.meta_file_cache_reclaimers)
+                    .with_clean_region_threshold(
+                        opts.meta_file_cache_reclaimers + opts.meta_file_cache_reclaimers / 2,
+                    )
+                    .with_recover_concurrency(opts.meta_file_cache_recover_concurrency)
+                    .with_compression(
+                        opts.meta_file_cache_compression
+                            .as_str()
+                            .try_into()
+                            .map_err(HummockError::foyer_error)?,
+                    )
+                    .with_runtime_config(
+                        RuntimeConfigBuilder::new()
+                            .with_thread_name("foyer.meta.runtime")
+                            .build(),
+                    );
+                if opts.meta_file_cache_insert_rate_limit_mb > 0 {
+                    builder = builder.with_admission_picker(Arc::new(RateLimitPicker::new(
+                        opts.meta_file_cache_insert_rate_limit_mb * MB,
+                    )));
+                }
+            }
 
-        let (data_file_cache, recent_filter) = if opts.data_file_cache_dir.is_empty() {
-            (FileCache::none(), None)
-        } else {
-            const MB: usize = 1024 * 1024;
-
-            let config = FileCacheConfig {
-                name: "data".to_string(),
-                dir: PathBuf::from(opts.data_file_cache_dir.clone()),
-                capacity: opts.data_file_cache_capacity_mb * MB,
-                file_capacity: opts.data_file_cache_file_capacity_mb * MB,
-                device_align: opts.data_file_cache_device_align,
-                device_io_size: opts.data_file_cache_device_io_size,
-                lfu_window_to_cache_size_ratio: opts.data_file_cache_lfu_window_to_cache_size_ratio,
-                lfu_tiny_lru_capacity_ratio: opts.data_file_cache_lfu_tiny_lru_capacity_ratio,
-                insert_rate_limit: opts.data_file_cache_insert_rate_limit_mb * MB,
-                flushers: opts.data_file_cache_flushers,
-                reclaimers: opts.data_file_cache_reclaimers,
-                recover_concurrency: opts.data_file_cache_recover_concurrency,
-                catalog_bits: opts.data_file_cache_catalog_bits,
-                admissions: vec![],
-                reinsertions: vec![],
-                compression: opts
-                    .data_file_cache_compression
-                    .as_str()
-                    .try_into()
-                    .map_err(HummockError::file_cache)?,
-            };
-            let cache = FileCache::open(config)
-                .await
-                .map_err(HummockError::file_cache)?;
-            let filter = Some(Arc::new(RecentFilter::new(
-                opts.cache_refill_recent_filter_layers,
-                Duration::from_millis(opts.cache_refill_recent_filter_rotate_interval_ms as u64),
-            )));
-            (cache, filter)
+            builder.build().await.map_err(HummockError::foyer_error)?
         };
 
-        let meta_file_cache = if opts.meta_file_cache_dir.is_empty() {
-            FileCache::none()
+        let block_cache = {
+            let mut builder = HybridCacheBuilder::new()
+                .with_name("foyer.data")
+                .with_event_listener(Arc::new(BlockCacheEventListener::new(
+                    state_store_metrics.clone(),
+                )))
+                .memory(opts.block_cache_capacity_mb * MB)
+                .with_shards(opts.block_cache_shard_num)
+                .with_eviction_config(opts.block_cache_eviction_config.clone())
+                .with_object_pool_capacity(1024 * opts.block_cache_shard_num)
+                .with_weighter(|_: &SstableBlockIndex, value: &Box<Block>| {
+                    // FIXME(MrCroxx): Calculate block weight more accurately.
+                    u64::BITS as usize * 2 / 8 + value.raw().len()
+                })
+                .storage();
+
+            if !opts.data_file_cache_dir.is_empty() {
+                builder = builder
+                    .with_device_config(
+                        DirectFsDeviceOptionsBuilder::new(&opts.data_file_cache_dir)
+                            .with_capacity(opts.data_file_cache_capacity_mb * MB)
+                            .with_file_size(opts.data_file_cache_file_capacity_mb * MB)
+                            .build(),
+                    )
+                    .with_indexer_shards(opts.data_file_cache_indexer_shards)
+                    .with_flushers(opts.data_file_cache_flushers)
+                    .with_reclaimers(opts.data_file_cache_reclaimers)
+                    .with_clean_region_threshold(
+                        opts.data_file_cache_reclaimers + opts.data_file_cache_reclaimers / 2,
+                    )
+                    .with_recover_concurrency(opts.data_file_cache_recover_concurrency)
+                    .with_compression(
+                        opts.data_file_cache_compression
+                            .as_str()
+                            .try_into()
+                            .map_err(HummockError::foyer_error)?,
+                    )
+                    .with_runtime_config(
+                        RuntimeConfigBuilder::new()
+                            .with_thread_name("foyer.data.runtime")
+                            .build(),
+                    );
+                if opts.data_file_cache_insert_rate_limit_mb > 0 {
+                    builder = builder.with_admission_picker(Arc::new(RateLimitPicker::new(
+                        opts.data_file_cache_insert_rate_limit_mb * MB,
+                    )));
+                }
+            }
+
+            builder.build().await.map_err(HummockError::foyer_error)?
+        };
+
+        let recent_filter = if opts.data_file_cache_dir.is_empty() {
+            None
         } else {
-            const MB: usize = 1024 * 1024;
-
-            let config = FileCacheConfig {
-                name: "meta".to_string(),
-                dir: PathBuf::from(opts.meta_file_cache_dir.clone()),
-                capacity: opts.meta_file_cache_capacity_mb * MB,
-                file_capacity: opts.meta_file_cache_file_capacity_mb * MB,
-                device_align: opts.meta_file_cache_device_align,
-                device_io_size: opts.meta_file_cache_device_io_size,
-                lfu_window_to_cache_size_ratio: opts.meta_file_cache_lfu_window_to_cache_size_ratio,
-                lfu_tiny_lru_capacity_ratio: opts.meta_file_cache_lfu_tiny_lru_capacity_ratio,
-                insert_rate_limit: opts.meta_file_cache_insert_rate_limit_mb * MB,
-                flushers: opts.meta_file_cache_flushers,
-                reclaimers: opts.meta_file_cache_reclaimers,
-                recover_concurrency: opts.meta_file_cache_recover_concurrency,
-                catalog_bits: opts.meta_file_cache_catalog_bits,
-                admissions: vec![],
-                reinsertions: vec![],
-                compression: opts
-                    .meta_file_cache_compression
-                    .as_str()
-                    .try_into()
-                    .map_err(HummockError::file_cache)?,
-            };
-            FileCache::open(config)
-                .await
-                .map_err(HummockError::file_cache)?
+            Some(Arc::new(RecentFilter::new(
+                opts.cache_refill_recent_filter_layers,
+                Duration::from_millis(opts.cache_refill_recent_filter_rotate_interval_ms as u64),
+            )))
         };
 
         let store = match s {
@@ -643,25 +741,20 @@ impl StateStoreImpl {
                     hummock.strip_prefix("hummock+").unwrap(),
                     object_store_metrics.clone(),
                     "Hummock",
-                    opts.object_store_config.clone(),
+                    Arc::new(opts.object_store_config.clone()),
                 )
                 .await;
 
                 let sstable_store = Arc::new(SstableStore::new(SstableStoreConfig {
                     store: Arc::new(object_store),
                     path: opts.data_directory.to_string(),
-                    block_cache_capacity: opts.block_cache_capacity_mb * (1 << 20),
-                    block_cache_shard_num: opts.block_cache_shard_num,
-                    block_cache_eviction: opts.block_cache_eviction_config.clone(),
-                    meta_cache_capacity: opts.meta_cache_capacity_mb * (1 << 20),
-                    meta_cache_shard_num: opts.meta_cache_shard_num,
-                    meta_cache_eviction: opts.meta_cache_eviction_config.clone(),
                     prefetch_buffer_capacity: opts.prefetch_buffer_capacity_mb * (1 << 20),
                     max_prefetch_block_number: opts.max_prefetch_block_number,
-                    data_file_cache,
-                    meta_file_cache,
                     recent_filter,
                     state_store_metrics: state_store_metrics.clone(),
+
+                    meta_cache,
+                    block_cache,
                 }));
                 let notification_client =
                     RpcNotificationClient::new(hummock_meta_client.get_inner().clone());
@@ -731,6 +824,8 @@ pub mod boxed_state_store {
 
     use bytes::Bytes;
     use dyn_clone::{clone_trait_object, DynClone};
+    use futures::future::BoxFuture;
+    use futures::FutureExt;
     use risingwave_common::buffer::Bitmap;
     use risingwave_hummock_sdk::key::{TableKey, TableKeyRange};
     use risingwave_hummock_sdk::{HummockReadEpoch, SyncResult};
@@ -739,7 +834,6 @@ pub mod boxed_state_store {
     use crate::hummock::HummockStorage;
     use crate::store::*;
     use crate::store_impl::AsHummock;
-    use crate::StateStore;
 
     #[async_trait::async_trait]
     pub trait DynamicDispatchedStateStoreIter<T: IterItem>: Send {
@@ -783,6 +877,13 @@ pub mod boxed_state_store {
             read_options: ReadOptions,
         ) -> StorageResult<BoxStateStoreReadIter>;
 
+        async fn rev_iter(
+            &self,
+            key_range: TableKeyRange,
+            epoch: u64,
+            read_options: ReadOptions,
+        ) -> StorageResult<BoxStateStoreReadIter>;
+
         async fn iter_log(
             &self,
             epoch_range: (u64, u64),
@@ -811,6 +912,17 @@ pub mod boxed_state_store {
             Ok(Box::new(self.iter(key_range, epoch, read_options).await?))
         }
 
+        async fn rev_iter(
+            &self,
+            key_range: TableKeyRange,
+            epoch: u64,
+            read_options: ReadOptions,
+        ) -> StorageResult<BoxStateStoreReadIter> {
+            Ok(Box::new(
+                self.rev_iter(key_range, epoch, read_options).await?,
+            ))
+        }
+
         async fn iter_log(
             &self,
             epoch_range: (u64, u64),
@@ -845,6 +957,12 @@ pub mod boxed_state_store {
             read_options: ReadOptions,
         ) -> StorageResult<BoxLocalStateStoreIterStream<'_>>;
 
+        async fn rev_iter(
+            &self,
+            key_range: TableKeyRange,
+            read_options: ReadOptions,
+        ) -> StorageResult<BoxLocalStateStoreIterStream<'_>>;
+
         fn insert(
             &mut self,
             key: TableKey<Bytes>,
@@ -895,6 +1013,14 @@ pub mod boxed_state_store {
             Ok(Box::new(self.iter(key_range, read_options).await?))
         }
 
+        async fn rev_iter(
+            &self,
+            key_range: TableKeyRange,
+            read_options: ReadOptions,
+        ) -> StorageResult<BoxLocalStateStoreIterStream<'_>> {
+            Ok(Box::new(self.rev_iter(key_range, read_options).await?))
+        }
+
         fn insert(
             &mut self,
             key: TableKey<Bytes>,
@@ -941,6 +1067,7 @@ pub mod boxed_state_store {
 
     impl LocalStateStore for BoxDynamicDispatchedLocalStateStore {
         type Iter<'a> = BoxLocalStateStoreIterStream<'a>;
+        type RevIter<'a> = BoxLocalStateStoreIterStream<'a>;
 
         fn may_exist(
             &self,
@@ -966,6 +1093,14 @@ pub mod boxed_state_store {
             self.deref().iter(key_range, read_options)
         }
 
+        fn rev_iter(
+            &self,
+            key_range: TableKeyRange,
+            read_options: ReadOptions,
+        ) -> impl Future<Output = StorageResult<Self::RevIter<'_>>> + Send + '_ {
+            self.deref().rev_iter(key_range, read_options)
+        }
+
         fn insert(
             &mut self,
             key: TableKey<Bytes>,
@@ -1017,7 +1152,7 @@ pub mod boxed_state_store {
     pub trait DynamicDispatchedStateStoreExt: StaticSendSync {
         async fn try_wait_epoch(&self, epoch: HummockReadEpoch) -> StorageResult<()>;
 
-        async fn sync(&self, epoch: u64) -> StorageResult<SyncResult>;
+        fn sync(&self, epoch: u64) -> BoxFuture<'static, StorageResult<SyncResult>>;
 
         fn seal_epoch(&self, epoch: u64, is_checkpoint: bool);
 
@@ -1034,8 +1169,8 @@ pub mod boxed_state_store {
             self.try_wait_epoch(epoch).await
         }
 
-        async fn sync(&self, epoch: u64) -> StorageResult<SyncResult> {
-            self.sync(epoch).await
+        fn sync(&self, epoch: u64) -> BoxFuture<'static, StorageResult<SyncResult>> {
+            self.sync(epoch).boxed()
         }
 
         fn seal_epoch(&self, epoch: u64, is_checkpoint: bool) {
@@ -1060,6 +1195,7 @@ pub mod boxed_state_store {
     impl StateStoreRead for BoxDynamicDispatchedStateStore {
         type ChangeLogIter = BoxStateStoreReadChangeLogIter;
         type Iter = BoxStateStoreReadIter;
+        type RevIter = BoxStateStoreReadIter;
 
         fn get(
             &self,
@@ -1079,6 +1215,15 @@ pub mod boxed_state_store {
             self.deref().iter(key_range, epoch, read_options)
         }
 
+        fn rev_iter(
+            &self,
+            key_range: TableKeyRange,
+            epoch: u64,
+            read_options: ReadOptions,
+        ) -> impl Future<Output = StorageResult<Self::RevIter>> + '_ {
+            self.deref().rev_iter(key_range, epoch, read_options)
+        }
+
         fn iter_log(
             &self,
             epoch_range: (u64, u64),
@@ -1118,7 +1263,10 @@ pub mod boxed_state_store {
             self.deref().try_wait_epoch(epoch)
         }
 
-        fn sync(&self, epoch: u64) -> impl Future<Output = StorageResult<SyncResult>> + Send + '_ {
+        fn sync(
+            &self,
+            epoch: u64,
+        ) -> impl Future<Output = StorageResult<SyncResult>> + Send + 'static {
             self.deref().sync(epoch)
         }
 
diff --git a/src/storage/src/table/batch_table/storage_table.rs b/src/storage/src/table/batch_table/storage_table.rs
index 3cf02d31ce87c..795668dfe8b1b 100644
--- a/src/storage/src/table/batch_table/storage_table.rs
+++ b/src/storage/src/table/batch_table/storage_table.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::default::Default;
 use std::ops::Bound::{self, Excluded, Included, Unbounded};
 use std::ops::{Index, RangeBounds};
 use std::sync::Arc;
@@ -20,11 +19,12 @@ use std::sync::Arc;
 use auto_enums::auto_enum;
 use await_tree::InstrumentAwait;
 use bytes::Bytes;
-use foyer::memory::CacheContext;
+use foyer::CacheContext;
 use futures::future::try_join_all;
 use futures::{Stream, StreamExt};
 use futures_async_stream::try_stream;
 use itertools::{Either, Itertools};
+use risingwave_common::array::Op;
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::{ColumnDesc, ColumnId, Schema, TableId, TableOption};
 use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
@@ -45,7 +45,7 @@ use crate::hummock::CachePolicy;
 use crate::row_serde::row_serde_util::{serialize_pk, serialize_pk_with_vnode};
 use crate::row_serde::value_serde::{ValueRowSerde, ValueRowSerdeNew};
 use crate::row_serde::{find_columns_by_ids, ColumnMapping};
-use crate::store::{PrefetchOptions, ReadOptions, StateStoreIter};
+use crate::store::{ChangeLogValue, PrefetchOptions, ReadLogOptions, ReadOptions, StateStoreIter};
 use crate::table::merge_sort::merge_sort;
 use crate::table::{KeyedRow, TableDistribution, TableIter};
 use crate::StateStore;
@@ -234,6 +234,7 @@ impl<S: StateStore> StorageTableInner<S, EitherSerde> {
 
         let (output_columns, output_indices) =
             find_columns_by_ids(&table_columns, &output_column_ids);
+
         let mut value_output_indices = vec![];
         let mut key_output_indices = vec![];
 
@@ -414,8 +415,7 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
 pub trait PkAndRowStream = Stream<Item = StorageResult<KeyedRow<Bytes>>> + Send;
 
 /// The row iterator of the storage table.
-/// The wrapper of [`StorageTableInnerIter`] if pk is not persisted.
-pub type StorageTableInnerIter<S: StateStore, SD: ValueRowSerde> = impl PkAndRowStream;
+/// The wrapper of stream item `StorageResult<KeyedRow<Bytes>>` if pk is not persisted.
 
 #[async_trait::async_trait]
 impl<S: PkAndRowStream + Unpin> TableIter for S {
@@ -429,7 +429,7 @@ impl<S: PkAndRowStream + Unpin> TableIter for S {
 
 /// Iterators
 impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
-    /// Get multiple [`StorageTableInnerIter`] based on the specified vnodes of this table with
+    /// Get multiple stream item `StorageResult<KeyedRow<Bytes>>` based on the specified vnodes of this table with
     /// `vnode_hint`, and merge or concat them by given `ordered`.
     async fn iter_with_encoded_key_range(
         &self,
@@ -439,14 +439,14 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
         vnode_hint: Option<VirtualNode>,
         ordered: bool,
         prefetch_options: PrefetchOptions,
-    ) -> StorageResult<StorageTableInnerIter<S, SD>> {
+    ) -> StorageResult<impl Stream<Item = StorageResult<KeyedRow<Bytes>>> + Send> {
         let cache_policy = match (
             encoded_key_range.start_bound(),
             encoded_key_range.end_bound(),
         ) {
             // To prevent unbounded range scan queries from polluting the block cache, use the
             // low priority fill policy.
-            (Unbounded, _) | (_, Unbounded) => CachePolicy::Fill(CacheContext::LruPriorityLow),
+            (Unbounded, _) | (_, Unbounded) => CachePolicy::Fill(CacheContext::LowPriority),
             _ => CachePolicy::Fill(CacheContext::Default),
         };
 
@@ -516,77 +516,67 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
         Ok(iter)
     }
 
-    /// Iterates on the table with the given prefix of the pk in `pk_prefix` and the range bounds.
-    async fn iter_with_pk_bounds(
+    // TODO: directly use `prefixed_range`.
+    fn serialize_pk_bound(
         &self,
-        epoch: HummockReadEpoch,
         pk_prefix: impl Row,
-        range_bounds: impl RangeBounds<OwnedRow>,
-        ordered: bool,
-        prefetch_options: PrefetchOptions,
-    ) -> StorageResult<StorageTableInnerIter<S, SD>> {
-        // TODO: directly use `prefixed_range`.
-        fn serialize_pk_bound(
-            pk_serializer: &OrderedRowSerde,
-            pk_prefix: impl Row,
-            range_bound: Bound<&OwnedRow>,
-            is_start_bound: bool,
-        ) -> Bound<Bytes> {
-            match range_bound {
-                Included(k) => {
-                    let pk_prefix_serializer = pk_serializer.prefix(pk_prefix.len() + k.len());
-                    let key = pk_prefix.chain(k);
-                    let serialized_key = serialize_pk(&key, &pk_prefix_serializer);
-                    if is_start_bound {
-                        Included(serialized_key)
-                    } else {
-                        // Should use excluded next key for end bound.
-                        // Otherwise keys starting with the bound is not included.
-                        end_bound_of_prefix(&serialized_key)
-                    }
+        range_bound: Bound<&OwnedRow>,
+        is_start_bound: bool,
+    ) -> Bound<Bytes> {
+        match range_bound {
+            Included(k) => {
+                let pk_prefix_serializer = self.pk_serializer.prefix(pk_prefix.len() + k.len());
+                let key = pk_prefix.chain(k);
+                let serialized_key = serialize_pk(&key, &pk_prefix_serializer);
+                if is_start_bound {
+                    Included(serialized_key)
+                } else {
+                    // Should use excluded next key for end bound.
+                    // Otherwise keys starting with the bound is not included.
+                    end_bound_of_prefix(&serialized_key)
                 }
-                Excluded(k) => {
-                    let pk_prefix_serializer = pk_serializer.prefix(pk_prefix.len() + k.len());
-                    let key = pk_prefix.chain(k);
-                    let serialized_key = serialize_pk(&key, &pk_prefix_serializer);
-                    if is_start_bound {
-                        // Storage doesn't support excluded begin key yet, so transform it to
-                        // included.
-                        // We always serialize a u8 for null of datum which is not equal to '\xff',
-                        // so we can assert that the next_key would never be empty.
-                        let next_serialized_key = next_key(&serialized_key);
-                        assert!(!next_serialized_key.is_empty());
-                        Included(Bytes::from(next_serialized_key))
-                    } else {
-                        Excluded(serialized_key)
-                    }
+            }
+            Excluded(k) => {
+                let pk_prefix_serializer = self.pk_serializer.prefix(pk_prefix.len() + k.len());
+                let key = pk_prefix.chain(k);
+                let serialized_key = serialize_pk(&key, &pk_prefix_serializer);
+                if is_start_bound {
+                    // Storage doesn't support excluded begin key yet, so transform it to
+                    // included.
+                    // We always serialize a u8 for null of datum which is not equal to '\xff',
+                    // so we can assert that the next_key would never be empty.
+                    let next_serialized_key = next_key(&serialized_key);
+                    assert!(!next_serialized_key.is_empty());
+                    Included(Bytes::from(next_serialized_key))
+                } else {
+                    Excluded(serialized_key)
                 }
-                Unbounded => {
-                    let pk_prefix_serializer = pk_serializer.prefix(pk_prefix.len());
-                    let serialized_pk_prefix = serialize_pk(&pk_prefix, &pk_prefix_serializer);
-                    if pk_prefix.is_empty() {
-                        Unbounded
-                    } else if is_start_bound {
-                        Included(serialized_pk_prefix)
-                    } else {
-                        end_bound_of_prefix(&serialized_pk_prefix)
-                    }
+            }
+            Unbounded => {
+                let pk_prefix_serializer = self.pk_serializer.prefix(pk_prefix.len());
+                let serialized_pk_prefix = serialize_pk(&pk_prefix, &pk_prefix_serializer);
+                if pk_prefix.is_empty() {
+                    Unbounded
+                } else if is_start_bound {
+                    Included(serialized_pk_prefix)
+                } else {
+                    end_bound_of_prefix(&serialized_pk_prefix)
                 }
             }
         }
+    }
 
-        let start_key = serialize_pk_bound(
-            &self.pk_serializer,
-            &pk_prefix,
-            range_bounds.start_bound(),
-            true,
-        );
-        let end_key = serialize_pk_bound(
-            &self.pk_serializer,
-            &pk_prefix,
-            range_bounds.end_bound(),
-            false,
-        );
+    /// Iterates on the table with the given prefix of the pk in `pk_prefix` and the range bounds.
+    async fn iter_with_pk_bounds(
+        &self,
+        epoch: HummockReadEpoch,
+        pk_prefix: impl Row,
+        range_bounds: impl RangeBounds<OwnedRow>,
+        ordered: bool,
+        prefetch_options: PrefetchOptions,
+    ) -> StorageResult<impl Stream<Item = StorageResult<KeyedRow<Bytes>>> + Send> {
+        let start_key = self.serialize_pk_bound(&pk_prefix, range_bounds.start_bound(), true);
+        let end_key = self.serialize_pk_bound(&pk_prefix, range_bounds.end_bound(), false);
 
         assert!(pk_prefix.len() <= self.pk_indices.len());
         let pk_prefix_indices = (0..pk_prefix.len())
@@ -636,7 +626,7 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
         .await
     }
 
-    /// Construct a [`StorageTableInnerIter`] for batch executors.
+    /// Construct a stream item `StorageResult<KeyedRow<Bytes>>` for batch executors.
     /// Differs from the streaming one, this iterator will wait for the epoch before iteration
     pub async fn batch_iter_with_pk_bounds(
         &self,
@@ -645,7 +635,7 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
         range_bounds: impl RangeBounds<OwnedRow>,
         ordered: bool,
         prefetch_options: PrefetchOptions,
-    ) -> StorageResult<StorageTableInnerIter<S, SD>> {
+    ) -> StorageResult<impl Stream<Item = StorageResult<KeyedRow<Bytes>>> + Send> {
         self.iter_with_pk_bounds(epoch, pk_prefix, range_bounds, ordered, prefetch_options)
             .await
     }
@@ -656,10 +646,64 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
         epoch: HummockReadEpoch,
         ordered: bool,
         prefetch_options: PrefetchOptions,
-    ) -> StorageResult<StorageTableInnerIter<S, SD>> {
+    ) -> StorageResult<impl Stream<Item = StorageResult<KeyedRow<Bytes>>> + Send> {
         self.batch_iter_with_pk_bounds(epoch, row::empty(), .., ordered, prefetch_options)
             .await
     }
+
+    pub async fn batch_iter_log_with_pk_bounds(
+        &self,
+        satrt_epoch: HummockReadEpoch,
+        end_epoch: HummockReadEpoch,
+        pk_prefix: impl Row,
+        range_bounds: impl RangeBounds<OwnedRow>,
+    ) -> StorageResult<impl Stream<Item = StorageResult<(Op, KeyedRow<Bytes>)>> + Send> {
+        let start_key = self.serialize_pk_bound(&pk_prefix, range_bounds.start_bound(), true);
+        let end_key = self.serialize_pk_bound(&pk_prefix, range_bounds.end_bound(), false);
+
+        assert!(pk_prefix.len() <= self.pk_indices.len());
+        let table_key_ranges = {
+            // Vnodes that are set and should be accessed.
+            let vnodes = match self.distribution.try_compute_vnode_by_pk_prefix(pk_prefix) {
+                // If `vnode_hint` is set, we can only access this single vnode.
+                Some(vnode) => Either::Left(std::iter::once(vnode)),
+                // Otherwise, we need to access all vnodes of this table.
+                None => Either::Right(self.distribution.vnodes().iter_vnodes()),
+            };
+            vnodes
+                .map(|vnode| prefixed_range_with_vnode((start_key.clone(), end_key.clone()), vnode))
+        };
+
+        let iterators: Vec<_> = try_join_all(table_key_ranges.map(|table_key_range| async move {
+            let read_options = ReadLogOptions {
+                table_id: self.table_id,
+            };
+            let iter = StorageTableInnerIterLogInner::<S, SD>::new(
+                &self.store,
+                self.mapping.clone(),
+                self.row_serde.clone(),
+                table_key_range,
+                read_options,
+                satrt_epoch,
+                end_epoch,
+            )
+            .await?
+            .into_stream();
+            Ok::<_, StorageError>(iter)
+        }))
+        .await?;
+
+        #[auto_enum(futures03::Stream)]
+        let iter = match iterators.len() {
+            0 => unreachable!(),
+            1 => iterators.into_iter().next().unwrap(),
+            // Concat all iterators if not to preserve order.
+            _ => futures::stream::iter(iterators.into_iter().map(Box::pin).collect_vec())
+                .flatten_unordered(1024),
+        };
+
+        Ok(iter)
+    }
 }
 
 /// [`StorageTableInnerIterInner`] iterates on the storage table.
@@ -787,3 +831,119 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInnerIterInner<S, SD> {
         }
     }
 }
+
+/// [`StorageTableInnerIterLogInner`] iterates on the storage table.
+struct StorageTableInnerIterLogInner<S: StateStore, SD: ValueRowSerde> {
+    /// An iterator that returns raw bytes from storage.
+    iter: S::ChangeLogIter,
+
+    mapping: Arc<ColumnMapping>,
+
+    row_deserializer: Arc<SD>,
+}
+
+impl<S: StateStore, SD: ValueRowSerde> StorageTableInnerIterLogInner<S, SD> {
+    /// If `wait_epoch` is true, it will wait for the given epoch to be committed before iteration.
+    #[allow(clippy::too_many_arguments)]
+    async fn new(
+        store: &S,
+        mapping: Arc<ColumnMapping>,
+        row_deserializer: Arc<SD>,
+        table_key_range: TableKeyRange,
+        read_options: ReadLogOptions,
+        satrt_epoch: HummockReadEpoch,
+        end_epoch: HummockReadEpoch,
+    ) -> StorageResult<Self> {
+        let raw_satrt_epoch = satrt_epoch.get_epoch();
+        let raw_end_epoch = end_epoch.get_epoch();
+        store.try_wait_epoch(end_epoch).await?;
+        let iter = store
+            .iter_log(
+                (raw_satrt_epoch, raw_end_epoch),
+                table_key_range,
+                read_options,
+            )
+            .await?;
+        let iter = Self {
+            iter,
+            mapping,
+            row_deserializer,
+        };
+        Ok(iter)
+    }
+
+    /// Yield a row with its primary key.
+    #[try_stream(ok = (Op, KeyedRow<Bytes>), error = StorageError)]
+    async fn into_stream(mut self) {
+        while let Some((k, v)) = self
+            .iter
+            .try_next()
+            .verbose_instrument_await("storage_table_iter_next")
+            .await?
+        {
+            match v {
+                ChangeLogValue::Insert(value) => {
+                    let full_row = self.row_deserializer.deserialize(value)?;
+                    let row = self
+                        .mapping
+                        .project(OwnedRow::new(full_row))
+                        .into_owned_row();
+                    // TODO: may optimize the key clone
+                    yield (
+                        Op::Insert,
+                        KeyedRow::<Bytes> {
+                            vnode_prefixed_key: k.copy_into(),
+                            row,
+                        },
+                    );
+                }
+                ChangeLogValue::Update {
+                    new_value,
+                    old_value,
+                } => {
+                    let full_row = self.row_deserializer.deserialize(old_value)?;
+                    let row = self
+                        .mapping
+                        .project(OwnedRow::new(full_row))
+                        .into_owned_row();
+                    // TODO: may optimize the key clone
+                    yield (
+                        Op::UpdateDelete,
+                        KeyedRow::<Bytes> {
+                            vnode_prefixed_key: k.copy_into(),
+                            row,
+                        },
+                    );
+                    let full_row = self.row_deserializer.deserialize(new_value)?;
+                    let row = self
+                        .mapping
+                        .project(OwnedRow::new(full_row))
+                        .into_owned_row();
+                    // TODO: may optimize the key clone
+                    yield (
+                        Op::UpdateInsert,
+                        KeyedRow::<Bytes> {
+                            vnode_prefixed_key: k.copy_into(),
+                            row,
+                        },
+                    );
+                }
+                ChangeLogValue::Delete(value) => {
+                    let full_row = self.row_deserializer.deserialize(value)?;
+                    let row = self
+                        .mapping
+                        .project(OwnedRow::new(full_row))
+                        .into_owned_row();
+                    // TODO: may optimize the key clone
+                    yield (
+                        Op::Delete,
+                        KeyedRow::<Bytes> {
+                            vnode_prefixed_key: k.copy_into(),
+                            row,
+                        },
+                    );
+                }
+            }
+        }
+    }
+}
diff --git a/src/storage/src/table/mod.rs b/src/storage/src/table/mod.rs
index 9be20b2cce538..d245e4bde3790 100644
--- a/src/storage/src/table/mod.rs
+++ b/src/storage/src/table/mod.rs
@@ -122,6 +122,10 @@ impl<T: AsRef<[u8]>> KeyedRow<T> {
         self.row
     }
 
+    pub fn into_owned_row_key(self) -> (TableKey<T>, OwnedRow) {
+        (self.vnode_prefixed_key, self.row)
+    }
+
     pub fn vnode(&self) -> VirtualNode {
         self.vnode_prefixed_key.vnode_part()
     }
diff --git a/src/stream/Cargo.toml b/src/stream/Cargo.toml
index 2b67865d1c8a3..89f2b61fd3449 100644
--- a/src/stream/Cargo.toml
+++ b/src/stream/Cargo.toml
@@ -24,7 +24,7 @@ await-tree = { workspace = true }
 bytes = "1"
 cfg-if = "1"
 delta_btree_map = { path = "../utils/delta_btree_map" }
-educe = "0.5"
+educe = "0.6"
 either = "1"
 enum-as-inner = "0.6"
 fail = "0.5"
diff --git a/src/stream/spill_test/src/lib.rs b/src/stream/spill_test/src/lib.rs
index 9c2d3d05bda70..deadc48ea9a33 100644
--- a/src/stream/spill_test/src/lib.rs
+++ b/src/stream/spill_test/src/lib.rs
@@ -13,9 +13,7 @@
 // limitations under the License.
 #![feature(proc_macro_hygiene, stmt_expr_attributes)]
 #![feature(custom_test_frameworks)]
-#![feature(bound_map)]
 #![feature(type_alias_impl_trait)]
-#![feature(associated_type_bounds)]
 
 #[cfg(test)]
 mod test_mem_table;
diff --git a/src/stream/src/cache/managed_lru.rs b/src/stream/src/cache/managed_lru.rs
index 42e158a80a78e..14edc4e1914e6 100644
--- a/src/stream/src/cache/managed_lru.rs
+++ b/src/stream/src/cache/managed_lru.rs
@@ -14,15 +14,14 @@
 
 use std::alloc::{Allocator, Global};
 use std::borrow::Borrow;
-use std::cmp::min;
 use std::hash::{BuildHasher, Hash};
 use std::ops::{Deref, DerefMut};
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
-use lru::{DefaultHasher, LruCache};
+use risingwave_common::lru::{LruCache, RandomState};
 use risingwave_common::metrics::LabelGuardedIntGauge;
-use risingwave_common::util::epoch::Epoch;
+use risingwave_common::sequence::AtomicSequence;
 use risingwave_common_estimate_size::EstimateSize;
 
 use crate::common::metrics::MetricsInfo;
@@ -31,37 +30,39 @@ const REPORT_SIZE_EVERY_N_KB_CHANGE: usize = 4096;
 
 /// The managed cache is a lru cache that bounds the memory usage by epoch.
 /// Should be used with `MemoryManager`.
-pub struct ManagedLruCache<K, V, S = DefaultHasher, A: Clone + Allocator = Global> {
+pub struct ManagedLruCache<K, V, S = RandomState, A = Global>
+where
+    K: Hash + Eq,
+    S: BuildHasher + Send + Sync + 'static,
+    A: Clone + Allocator,
+{
     inner: LruCache<K, V, S, A>,
-    /// The entry with epoch less than water should be evicted.
-    /// Should only be updated by the `MemoryManager`.
-    watermark_epoch: Arc<AtomicU64>,
-    /// The heap size of keys/values
-    kv_heap_size: usize,
-    /// The metrics of memory usage
-    memory_usage_metrics: LabelGuardedIntGauge<3>,
-    // The metrics of evicted watermark time
-    lru_evicted_watermark_time_ms: LabelGuardedIntGauge<3>,
+
+    /// The entry with sequence less than `watermark_sequence` should be evicted.
+    /// `watermark_sequence` should only be updatd by `MemoryManager`.
+    watermark_sequence: Arc<AtomicSequence>,
+
     // Metrics info
     _metrics_info: MetricsInfo,
-    /// The size reported last time
-    last_reported_size_bytes: usize,
-}
 
-impl<K, V, S, A: Clone + Allocator> Drop for ManagedLruCache<K, V, S, A> {
-    fn drop(&mut self) {
-        self.memory_usage_metrics.set(0.into());
-    }
+    reporter: HeapSizeReporter,
 }
 
-impl<K: Hash + Eq + EstimateSize, V: EstimateSize, S: BuildHasher, A: Clone + Allocator>
-    ManagedLruCache<K, V, S, A>
+impl<K, V, S, A> ManagedLruCache<K, V, S, A>
+where
+    K: Hash + Eq + EstimateSize,
+    V: EstimateSize,
+    S: BuildHasher + Send + Sync + 'static,
+    A: Clone + Allocator,
 {
-    pub fn new_inner(
-        inner: LruCache<K, V, S, A>,
-        watermark_epoch: Arc<AtomicU64>,
+    pub fn unbounded_with_hasher_in(
+        watermark_sequence: Arc<AtomicSequence>,
         metrics_info: MetricsInfo,
+        hash_builder: S,
+        alloc: A,
     ) -> Self {
+        let inner = LruCache::unbounded_with_hasher_in(hash_builder, alloc);
+
         let memory_usage_metrics = metrics_info
             .metrics
             .stream_memory_usage
@@ -72,80 +73,43 @@ impl<K: Hash + Eq + EstimateSize, V: EstimateSize, S: BuildHasher, A: Clone + Al
             ]);
         memory_usage_metrics.set(0.into());
 
-        let lru_evicted_watermark_time_ms = metrics_info
-            .metrics
-            .lru_evicted_watermark_time_ms
-            .with_guarded_label_values(&[
-                &metrics_info.table_id,
-                &metrics_info.actor_id,
-                &metrics_info.desc,
-            ]);
+        let reporter = HeapSizeReporter::new(memory_usage_metrics, 0, 0);
 
         Self {
             inner,
-            watermark_epoch,
-            kv_heap_size: 0,
-            memory_usage_metrics,
-            lru_evicted_watermark_time_ms,
+            watermark_sequence,
             _metrics_info: metrics_info,
-            last_reported_size_bytes: 0,
+            reporter,
         }
     }
 
     /// Evict epochs lower than the watermark
     pub fn evict(&mut self) {
-        self.evict_by_epoch(self.load_cur_epoch());
-    }
-
-    /// Evict epochs lower than the watermark, except those entry which touched in this epoch
-    pub fn evict_except_cur_epoch(&mut self) {
-        let epoch = min(self.load_cur_epoch(), self.inner.current_epoch());
-        self.evict_by_epoch(epoch);
-    }
-
-    /// Evict epochs lower than the watermark
-    fn evict_by_epoch(&mut self, epoch: u64) {
-        while let Some((key, value, _)) = self.inner.pop_lru_by_epoch(epoch) {
+        let sequence = self.watermark_sequence.load(Ordering::Relaxed);
+        while let Some((key, value, _)) = self.inner.pop_with_sequence(sequence) {
             let charge = key.estimated_size() + value.estimated_size();
-            self.kv_heap_size_dec(charge);
+            self.reporter.dec(charge);
         }
-        self.report_evicted_watermark_time(epoch);
-    }
-
-    pub fn update_epoch(&mut self, epoch: u64) {
-        self.inner.update_epoch(epoch);
-    }
-
-    pub fn current_epoch(&mut self) -> u64 {
-        self.inner.current_epoch()
-    }
-
-    /// An iterator visiting all values in most-recently used order. The iterator element type is
-    /// &V.
-    pub fn values(&self) -> impl Iterator<Item = &V> {
-        self.inner.iter().map(|(_k, v)| v)
     }
 
     pub fn put(&mut self, k: K, v: V) -> Option<V> {
         let key_size = k.estimated_size();
-        self.kv_heap_size_inc(key_size + v.estimated_size());
+        self.reporter.inc(key_size + v.estimated_size());
         let old_val = self.inner.put(k, v);
         if let Some(old_val) = &old_val {
-            self.kv_heap_size_dec(key_size + old_val.estimated_size());
+            self.reporter.dec(key_size + old_val.estimated_size());
         }
         old_val
     }
 
+    // TODO(MrCroxx): REMOVE ME!!!
+    pub fn push(&mut self, k: K, v: V) -> Option<V> {
+        self.put(k, v)
+    }
+
     pub fn get_mut(&mut self, k: &K) -> Option<MutGuard<'_, V>> {
         let v = self.inner.get_mut(k);
-        v.map(|inner| {
-            MutGuard::new(
-                inner,
-                &mut self.kv_heap_size,
-                &mut self.last_reported_size_bytes,
-                &mut self.memory_usage_metrics,
-            )
-        })
+        v.map(|inner| MutGuard::new(inner, &mut self.reporter))
     }
 
     pub fn get<Q>(&mut self, k: &Q) -> Option<&V>
@@ -166,25 +130,7 @@ impl<K: Hash + Eq + EstimateSize, V: EstimateSize, S: BuildHasher, A: Clone + Al
 
     pub fn peek_mut(&mut self, k: &K) -> Option<MutGuard<'_, V>> {
         let v = self.inner.peek_mut(k);
-        v.map(|inner| {
-            MutGuard::new(
-                inner,
-                &mut self.kv_heap_size,
-                &mut self.last_reported_size_bytes,
-                &mut self.memory_usage_metrics,
-            )
-        })
-    }
-
-    pub fn push(&mut self, k: K, v: V) -> Option<(K, V)> {
-        self.kv_heap_size_inc(k.estimated_size() + v.estimated_size());
-
-        let old_kv = self.inner.push(k, v);
-
-        if let Some((old_key, old_val)) = &old_kv {
-            self.kv_heap_size_dec(old_key.estimated_size() + old_val.estimated_size());
-        }
-        old_kv
+        v.map(|inner| MutGuard::new(inner, &mut self.reporter))
     }
 
     pub fn contains<Q>(&self, k: &Q) -> bool
@@ -200,129 +146,66 @@ impl<K: Hash + Eq + EstimateSize, V: EstimateSize, S: BuildHasher, A: Clone + Al
     }
 
     pub fn is_empty(&self) -> bool {
-        self.inner.len() == 0
+        self.inner.is_empty()
     }
 
     pub fn clear(&mut self) {
         self.inner.clear();
     }
-
-    fn kv_heap_size_inc(&mut self, size: usize) {
-        self.kv_heap_size = self.kv_heap_size.saturating_add(size);
-        self.report_memory_usage();
-    }
-
-    fn kv_heap_size_dec(&mut self, size: usize) {
-        self.kv_heap_size = self.kv_heap_size.saturating_sub(size);
-        self.report_memory_usage();
-    }
-
-    fn report_memory_usage(&mut self) -> bool {
-        if self.kv_heap_size.abs_diff(self.last_reported_size_bytes)
-            > REPORT_SIZE_EVERY_N_KB_CHANGE << 10
-        {
-            self.memory_usage_metrics.set(self.kv_heap_size as _);
-            self.last_reported_size_bytes = self.kv_heap_size;
-            true
-        } else {
-            false
-        }
-    }
-
-    fn report_evicted_watermark_time(&self, epoch: u64) {
-        self.lru_evicted_watermark_time_ms
-            .set(Epoch(epoch).physical_time() as _);
-    }
-
-    fn load_cur_epoch(&self) -> u64 {
-        self.watermark_epoch.load(Ordering::Relaxed)
-    }
-}
-
-pub fn new_unbounded<K: Hash + Eq + EstimateSize, V: EstimateSize>(
-    watermark_epoch: Arc<AtomicU64>,
-    metrics_info: MetricsInfo,
-) -> ManagedLruCache<K, V> {
-    ManagedLruCache::new_inner(LruCache::unbounded(), watermark_epoch, metrics_info)
 }
 
-pub fn new_with_hasher_in<
+impl<K, V> ManagedLruCache<K, V>
+where
     K: Hash + Eq + EstimateSize,
     V: EstimateSize,
-    S: BuildHasher,
-    A: Clone + Allocator,
->(
-    watermark_epoch: Arc<AtomicU64>,
-    metrics_info: MetricsInfo,
-    hasher: S,
-    alloc: A,
-) -> ManagedLruCache<K, V, S, A> {
-    ManagedLruCache::new_inner(
-        LruCache::unbounded_with_hasher_in(hasher, alloc),
-        watermark_epoch,
-        metrics_info,
-    )
+{
+    pub fn unbounded(watermark_sequence: Arc<AtomicSequence>, metrics_info: MetricsInfo) -> Self {
+        Self::unbounded_with_hasher(watermark_sequence, metrics_info, RandomState::default())
+    }
 }
 
-pub fn new_with_hasher<K: Hash + Eq + EstimateSize, V: EstimateSize, S: BuildHasher>(
-    watermark_epoch: Arc<AtomicU64>,
-    metrics_info: MetricsInfo,
-    hasher: S,
-) -> ManagedLruCache<K, V, S> {
-    ManagedLruCache::new_inner(
-        LruCache::unbounded_with_hasher(hasher),
-        watermark_epoch,
-        metrics_info,
-    )
+impl<K, V, S> ManagedLruCache<K, V, S>
+where
+    K: Hash + Eq + EstimateSize,
+    V: EstimateSize,
+    S: BuildHasher + Send + Sync + 'static,
+{
+    pub fn unbounded_with_hasher(
+        watermark_sequence: Arc<AtomicSequence>,
+        metrics_info: MetricsInfo,
+        hash_builder: S,
+    ) -> Self {
+        Self::unbounded_with_hasher_in(watermark_sequence, metrics_info, hash_builder, Global)
+    }
 }
 
 pub struct MutGuard<'a, V: EstimateSize> {
     inner: &'a mut V,
-    // The size of the original value
-    original_val_size: usize,
-    // The total size of a collection
-    total_size: &'a mut usize,
-    last_reported_size_bytes: &'a mut usize,
-    memory_usage_metrics: &'a mut LabelGuardedIntGauge<3>,
+    reporter: &'a mut HeapSizeReporter,
+    old_value_size: usize,
 }
 
 impl<'a, V: EstimateSize> MutGuard<'a, V> {
-    pub fn new(
-        inner: &'a mut V,
-        total_size: &'a mut usize,
-        last_reported_size_bytes: &'a mut usize,
-        memory_usage_metrics: &'a mut LabelGuardedIntGauge<3>,
-    ) -> Self {
-        let original_val_size = inner.estimated_size();
+    fn new(inner: &'a mut V, reporter: &'a mut HeapSizeReporter) -> Self {
+        let old_value_size = inner.estimated_size();
         Self {
             inner,
-            original_val_size,
-            total_size,
-            last_reported_size_bytes,
-            memory_usage_metrics,
-        }
-    }
-
-    fn report_memory_usage(&mut self) -> bool {
-        if self.total_size.abs_diff(*self.last_reported_size_bytes)
-            > REPORT_SIZE_EVERY_N_KB_CHANGE << 10
-        {
-            self.memory_usage_metrics.set(*self.total_size as _);
-            *self.last_reported_size_bytes = *self.total_size;
-            true
-        } else {
-            false
+            reporter,
+            old_value_size,
         }
     }
 }
 
 impl<'a, V: EstimateSize> Drop for MutGuard<'a, V> {
     fn drop(&mut self) {
-        *self.total_size = self
-            .total_size
-            .saturating_sub(self.original_val_size)
-            .saturating_add(self.inner.estimated_size());
-        self.report_memory_usage();
+        let new_value_size = self.inner.estimated_size();
+        if new_value_size != self.old_value_size {
+            self.reporter.apply(|size| {
+                *size = size
+                    .saturating_sub(self.old_value_size)
+                    .saturating_add(new_value_size)
+            })
+        }
     }
 }
 
@@ -339,3 +222,57 @@ impl<'a, V: EstimateSize> DerefMut for MutGuard<'a, V> {
         self.inner
     }
 }
+
+struct HeapSizeReporter {
+    metrics: LabelGuardedIntGauge<3>,
+    heap_size: usize,
+    last_reported: usize,
+}
+
+impl HeapSizeReporter {
+    fn new(
+        heap_size_metrics: LabelGuardedIntGauge<3>,
+        heap_size: usize,
+        last_reported: usize,
+    ) -> Self {
+        Self {
+            metrics: heap_size_metrics,
+            heap_size,
+            last_reported,
+        }
+    }
+
+    fn inc(&mut self, size: usize) {
+        self.heap_size = self.heap_size.saturating_add(size);
+        self.try_report();
+    }
+
+    fn dec(&mut self, size: usize) {
+        self.heap_size = self.heap_size.saturating_sub(size);
+        self.try_report();
+    }
+
+    fn apply<F>(&mut self, f: F)
+    where
+        F: FnOnce(&mut usize),
+    {
+        f(&mut self.heap_size);
+        self.try_report();
+    }
+
+    fn try_report(&mut self) -> bool {
+        if self.heap_size.abs_diff(self.last_reported) >= REPORT_SIZE_EVERY_N_KB_CHANGE << 10 {
+            self.metrics.set(self.heap_size as _);
+            self.last_reported = self.heap_size;
+            true
+        } else {
+            false
+        }
+    }
+}
+
+impl Drop for HeapSizeReporter {
+    fn drop(&mut self) {
+        self.metrics.set(0);
+    }
+}
diff --git a/src/stream/src/common/compact_chunk.rs b/src/stream/src/common/compact_chunk.rs
index 32dc4d0a996a6..b6598a80872ac 100644
--- a/src/stream/src/common/compact_chunk.rs
+++ b/src/stream/src/common/compact_chunk.rs
@@ -105,7 +105,7 @@ type OpRowMap<'a, 'b> =
 pub enum RowOp<'a> {
     Insert(RowRef<'a>),
     Delete(RowRef<'a>),
-    /// (old_value, new_value)
+    /// (`old_value`, `new_value`)
     Update((RowRef<'a>, RowRef<'a>)),
 }
 static LOG_SUPPERSSER: LazyLock<LogSuppresser> = LazyLock::new(LogSuppresser::default);
@@ -232,6 +232,7 @@ impl StreamChunkCompactor {
     ///   have three kind of patterns Insert, Delete or Update.
     /// - For the update (-old row, +old row), when old row is exactly same. The two rowOp will be
     ///   removed.
+    ///
     /// All UPDATE INSERT and UPDATE DELETE will be converted to INSERT and DELETE, and dropped according to
     /// certain rules (see `merge_insert` and `merge_delete` for more details).
     pub fn into_compacted_chunks(self) -> impl Iterator<Item = StreamChunk> {
@@ -333,7 +334,6 @@ pub fn merge_chunk_row(stream_chunk: StreamChunk, pk_indices: &[usize]) -> Strea
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::array::StreamChunk;
     use risingwave_common::test_prelude::StreamChunkTestExt;
 
     use super::*;
diff --git a/src/stream/src/common/log_store_impl/in_mem.rs b/src/stream/src/common/log_store_impl/in_mem.rs
index ccb2eb878ede9..d1ce2e6e5aa35 100644
--- a/src/stream/src/common/log_store_impl/in_mem.rs
+++ b/src/stream/src/common/log_store_impl/in_mem.rs
@@ -15,6 +15,7 @@
 use std::sync::Arc;
 
 use anyhow::{anyhow, Context};
+use await_tree::InstrumentAwait;
 use risingwave_common::array::StreamChunk;
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::util::epoch::{EpochExt, EpochPair, INVALID_EPOCH};
@@ -200,7 +201,7 @@ impl LogReader for BoundedInMemLogStoreReader {
         }
     }
 
-    async fn truncate(&mut self, offset: TruncateOffset) -> LogStoreResult<()> {
+    fn truncate(&mut self, offset: TruncateOffset) -> LogStoreResult<()> {
         // check the truncate offset is higher than prev truncate offset
         if self.truncate_offset >= offset {
             return Err(anyhow!(
@@ -260,6 +261,7 @@ impl LogWriter for BoundedInMemLogStoreWriter {
     async fn write_chunk(&mut self, chunk: StreamChunk) -> LogStoreResult<()> {
         self.item_tx
             .send(InMemLogStoreItem::StreamChunk(chunk))
+            .instrument_await("in_mem_send_item_chunk")
             .await
             .map_err(|_| anyhow!("unable to send stream chunk"))?;
         Ok(())
@@ -275,6 +277,7 @@ impl LogWriter for BoundedInMemLogStoreWriter {
                 next_epoch,
                 is_checkpoint,
             })
+            .instrument_await("in_mem_send_item_barrier")
             .await
             .map_err(|_| anyhow!("unable to send barrier"))?;
 
@@ -287,6 +290,7 @@ impl LogWriter for BoundedInMemLogStoreWriter {
             let truncated_epoch = self
                 .truncated_epoch_rx
                 .recv()
+                .instrument_await("in_mem_recv_truncated_epoch")
                 .await
                 .ok_or_else(|| anyhow!("cannot get truncated epoch"))?;
             assert_eq!(truncated_epoch, prev_epoch);
@@ -298,6 +302,7 @@ impl LogWriter for BoundedInMemLogStoreWriter {
     async fn update_vnode_bitmap(&mut self, new_vnodes: Arc<Bitmap>) -> LogStoreResult<()> {
         self.item_tx
             .send(InMemLogStoreItem::UpdateVnodeBitmap(new_vnodes))
+            .instrument_await("in_mem_send_item_vnode_bitmap")
             .await
             .map_err(|_| anyhow!("unable to send vnode bitmap"))
     }
@@ -338,7 +343,8 @@ mod tests {
         let epoch2 = test_epoch(3);
 
         let ops = vec![Op::Insert, Op::Delete, Op::UpdateInsert, Op::UpdateDelete];
-        let mut builder = StreamChunkBuilder::new(10000, vec![DataType::Int64, DataType::Varchar]);
+        let mut builder =
+            StreamChunkBuilder::unlimited(vec![DataType::Int64, DataType::Varchar], None);
         for (i, op) in ops.into_iter().enumerate() {
             assert!(builder
                 .append_row(
@@ -420,7 +426,6 @@ mod tests {
                 epoch: init_epoch,
                 chunk_id: chunk_id1_2,
             })
-            .await
             .unwrap();
         assert!(poll_fn(|cx| Poll::Ready(join_handle.poll_unpin(cx)))
             .await
@@ -430,14 +435,12 @@ mod tests {
                 epoch: epoch1,
                 chunk_id: chunk_id2_1,
             })
-            .await
             .unwrap();
         assert!(poll_fn(|cx| Poll::Ready(join_handle.poll_unpin(cx)))
             .await
             .is_pending());
         reader
             .truncate(TruncateOffset::Barrier { epoch: epoch1 })
-            .await
             .unwrap();
         join_handle.await.unwrap();
     }
diff --git a/src/stream/src/common/log_store_impl/kv_log_store/buffer.rs b/src/stream/src/common/log_store_impl/kv_log_store/buffer.rs
index a02c447935fbd..85926a82373da 100644
--- a/src/stream/src/common/log_store_impl/kv_log_store/buffer.rs
+++ b/src/stream/src/common/log_store_impl/kv_log_store/buffer.rs
@@ -16,13 +16,16 @@ use std::collections::VecDeque;
 use std::ops::DerefMut;
 use std::sync::Arc;
 
+use await_tree::InstrumentAwait;
 use parking_lot::{Mutex, MutexGuard};
 use risingwave_common::array::StreamChunk;
 use risingwave_common::buffer::Bitmap;
 use risingwave_connector::sink::log_store::{ChunkId, LogStoreResult, TruncateOffset};
 use tokio::sync::{oneshot, Notify};
 
-use crate::common::log_store_impl::kv_log_store::{ReaderTruncationOffsetType, SeqIdType};
+use crate::common::log_store_impl::kv_log_store::{
+    KvLogStoreMetrics, ReaderTruncationOffsetType, SeqIdType,
+};
 
 #[derive(Clone)]
 pub(crate) enum LogStoreBufferItem {
@@ -60,9 +63,45 @@ struct LogStoreBufferInner {
     truncation_list: VecDeque<ReaderTruncationOffsetType>,
 
     next_chunk_id: ChunkId,
+
+    metrics: KvLogStoreMetrics,
 }
 
 impl LogStoreBufferInner {
+    fn update_unconsumed_buffer_metrics(&self) {
+        let mut epoch_count = 0;
+        let mut row_count = 0;
+        for (_, item) in &self.unconsumed_queue {
+            match item {
+                LogStoreBufferItem::StreamChunk { chunk, .. } => {
+                    row_count += chunk.cardinality();
+                }
+                LogStoreBufferItem::Flushed {
+                    start_seq_id,
+                    end_seq_id,
+                    ..
+                } => {
+                    row_count += (end_seq_id - start_seq_id) as usize;
+                }
+                LogStoreBufferItem::Barrier { .. } => {
+                    epoch_count += 1;
+                }
+                LogStoreBufferItem::UpdateVnodes(_) => {}
+            }
+        }
+        self.metrics.buffer_unconsumed_epoch_count.set(epoch_count);
+        self.metrics.buffer_unconsumed_row_count.set(row_count as _);
+        self.metrics
+            .buffer_unconsumed_item_count
+            .set(self.unconsumed_queue.len() as _);
+        self.metrics.buffer_unconsumed_min_epoch.set(
+            self.unconsumed_queue
+                .front()
+                .map(|(epoch, _)| *epoch)
+                .unwrap_or_default() as _,
+        );
+    }
+
     fn can_add_stream_chunk(&self) -> bool {
         self.row_count < self.max_row_count
     }
@@ -75,6 +114,7 @@ impl LogStoreBufferInner {
             self.next_chunk_id = 0;
         }
         self.unconsumed_queue.push_front((epoch, item));
+        self.update_unconsumed_buffer_metrics();
     }
 
     pub(crate) fn try_add_stream_chunk(
@@ -100,6 +140,7 @@ impl LogStoreBufferInner {
                     chunk_id,
                 },
             ));
+            self.update_unconsumed_buffer_metrics();
             None
         }
     }
@@ -107,6 +148,7 @@ impl LogStoreBufferInner {
     fn pop_item(&mut self) -> Option<(u64, LogStoreBufferItem)> {
         if let Some((epoch, item)) = self.unconsumed_queue.pop_back() {
             self.consumed_queue.push_front((epoch, item.clone()));
+            self.update_unconsumed_buffer_metrics();
             Some((epoch, item))
         } else {
             None
@@ -154,12 +196,24 @@ impl LogStoreBufferInner {
                 },
             );
         }
+        self.update_unconsumed_buffer_metrics();
+    }
+
+    fn add_truncate_offset(&mut self, (epoch, seq_id): ReaderTruncationOffsetType) {
+        if let Some((prev_epoch, ref mut prev_seq_id)) = self.truncation_list.back_mut()
+            && *prev_epoch == epoch
+        {
+            *prev_seq_id = seq_id;
+        } else {
+            self.truncation_list.push_back((epoch, seq_id));
+        }
     }
 
     fn rewind(&mut self) {
         while let Some((epoch, item)) = self.consumed_queue.pop_front() {
-            self.unconsumed_queue.push_back((epoch, item))
+            self.unconsumed_queue.push_back((epoch, item));
         }
+        self.update_unconsumed_buffer_metrics();
     }
 }
 
@@ -319,7 +373,7 @@ impl LogStoreBufferReceiver {
         } {
             item
         } else {
-            notified.await;
+            notified.instrument_await("Wait For New Buffer Item").await;
             self.buffer
                 .inner()
                 .pop_item()
@@ -327,7 +381,7 @@ impl LogStoreBufferReceiver {
         }
     }
 
-    pub(crate) fn truncate(&mut self, offset: TruncateOffset) {
+    pub(crate) fn truncate_buffer(&mut self, offset: TruncateOffset) {
         let mut inner = self.buffer.inner();
         let mut latest_offset: Option<ReaderTruncationOffsetType> = None;
         while let Some((epoch, item)) = inner.consumed_queue.back() {
@@ -387,17 +441,16 @@ impl LogStoreBufferReceiver {
                 }
             }
         }
-        if let Some((epoch, seq_id)) = latest_offset {
-            if let Some((prev_epoch, ref mut prev_seq_id)) = inner.truncation_list.back_mut()
-                && *prev_epoch == epoch
-            {
-                *prev_seq_id = seq_id;
-            } else {
-                inner.truncation_list.push_back((epoch, seq_id));
-            }
+        if let Some(offset) = latest_offset {
+            inner.add_truncate_offset(offset);
         }
     }
 
+    pub(crate) fn truncate_historical(&mut self, epoch: u64) {
+        let mut inner = self.buffer.inner();
+        inner.add_truncate_offset((epoch, None));
+    }
+
     pub(crate) fn rewind(&self) {
         self.buffer.inner().rewind()
     }
@@ -405,6 +458,7 @@ impl LogStoreBufferReceiver {
 
 pub(crate) fn new_log_store_buffer(
     max_row_count: usize,
+    metrics: KvLogStoreMetrics,
 ) -> (LogStoreBufferSender, LogStoreBufferReceiver) {
     let buffer = SharedMutex::new(LogStoreBufferInner {
         unconsumed_queue: VecDeque::new(),
@@ -413,6 +467,7 @@ pub(crate) fn new_log_store_buffer(
         max_row_count,
         truncation_list: VecDeque::new(),
         next_chunk_id: 0,
+        metrics,
     });
     let update_notify = Arc::new(Notify::new());
     let (init_epoch_tx, init_epoch_rx) = oneshot::channel();
diff --git a/src/stream/src/common/log_store_impl/kv_log_store/mod.rs b/src/stream/src/common/log_store_impl/kv_log_store/mod.rs
index 3d28ccdcb7e6c..3db9723e2ba14 100644
--- a/src/stream/src/common/log_store_impl/kv_log_store/mod.rs
+++ b/src/stream/src/common/log_store_impl/kv_log_store/mod.rs
@@ -16,9 +16,11 @@ use std::sync::Arc;
 
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::{TableId, TableOption};
-use risingwave_common::metrics::{LabelGuardedHistogram, LabelGuardedIntCounter};
+use risingwave_common::metrics::{
+    LabelGuardedHistogram, LabelGuardedIntCounter, LabelGuardedIntGauge,
+};
 use risingwave_connector::sink::log_store::LogStoreFactory;
-use risingwave_connector::sink::{SinkParam, SinkWriterParam};
+use risingwave_connector::sink::SinkParam;
 use risingwave_pb::catalog::Table;
 use risingwave_storage::store::{NewLocalOptions, OpConsistencyLevel};
 use risingwave_storage::StateStore;
@@ -74,6 +76,10 @@ pub(crate) struct KvLogStoreMetrics {
     pub storage_write_size: LabelGuardedIntCounter<3>,
     pub rewind_count: LabelGuardedIntCounter<3>,
     pub rewind_delay: LabelGuardedHistogram<3>,
+    pub buffer_unconsumed_item_count: LabelGuardedIntGauge<3>,
+    pub buffer_unconsumed_row_count: LabelGuardedIntGauge<3>,
+    pub buffer_unconsumed_epoch_count: LabelGuardedIntGauge<3>,
+    pub buffer_unconsumed_min_epoch: LabelGuardedIntGauge<3>,
     pub persistent_log_read_metrics: KvLogStoreReadMetrics,
     pub flushed_buffer_read_metrics: KvLogStoreReadMetrics,
 }
@@ -81,11 +87,11 @@ pub(crate) struct KvLogStoreMetrics {
 impl KvLogStoreMetrics {
     pub(crate) fn new(
         metrics: &StreamingMetrics,
-        writer_param: &SinkWriterParam,
+        identity: &String,
         sink_param: &SinkParam,
         connector: &'static str,
     ) -> Self {
-        let executor_id = format!("{}", writer_param.executor_id);
+        let executor_id = identity;
         let sink_id = format!("{}", sink_param.sink_id.sink_id);
         let labels = &[executor_id.as_str(), connector, sink_id.as_str()];
         let storage_write_size = metrics
@@ -140,10 +146,27 @@ impl KvLogStoreMetrics {
             .kv_log_store_rewind_delay
             .with_guarded_label_values(labels);
 
+        let buffer_unconsumed_item_count = metrics
+            .kv_log_store_buffer_unconsumed_item_count
+            .with_guarded_label_values(labels);
+        let buffer_unconsumed_row_count = metrics
+            .kv_log_store_buffer_unconsumed_row_count
+            .with_guarded_label_values(labels);
+        let buffer_unconsumed_epoch_count = metrics
+            .kv_log_store_buffer_unconsumed_epoch_count
+            .with_guarded_label_values(labels);
+        let buffer_unconsumed_min_epoch = metrics
+            .kv_log_store_buffer_unconsumed_min_epoch
+            .with_guarded_label_values(labels);
+
         Self {
             storage_write_size,
             rewind_count,
             storage_write_count,
+            buffer_unconsumed_item_count,
+            buffer_unconsumed_row_count,
+            buffer_unconsumed_epoch_count,
+            buffer_unconsumed_min_epoch,
             persistent_log_read_metrics: KvLogStoreReadMetrics {
                 storage_read_size: persistent_log_read_size,
                 storage_read_count: persistent_log_read_count,
@@ -161,6 +184,10 @@ impl KvLogStoreMetrics {
         KvLogStoreMetrics {
             storage_write_count: LabelGuardedIntCounter::test_int_counter(),
             storage_write_size: LabelGuardedIntCounter::test_int_counter(),
+            buffer_unconsumed_item_count: LabelGuardedIntGauge::test_int_gauge(),
+            buffer_unconsumed_row_count: LabelGuardedIntGauge::test_int_gauge(),
+            buffer_unconsumed_epoch_count: LabelGuardedIntGauge::test_int_gauge(),
+            buffer_unconsumed_min_epoch: LabelGuardedIntGauge::test_int_gauge(),
             rewind_count: LabelGuardedIntCounter::test_int_counter(),
             rewind_delay: LabelGuardedHistogram::test_histogram(),
             persistent_log_read_metrics: KvLogStoreReadMetrics::for_test(),
@@ -369,7 +396,7 @@ impl<S: StateStore> LogStoreFactory for KvLogStoreFactory<S> {
             })
             .await;
 
-        let (tx, rx) = new_log_store_buffer(self.max_row_count);
+        let (tx, rx) = new_log_store_buffer(self.max_row_count, self.metrics.clone());
 
         let reader = KvLogStoreReader::new(
             table_id,
@@ -629,7 +656,6 @@ mod tests {
         // The truncate does not work because it is after the sync
         reader
             .truncate(TruncateOffset::Barrier { epoch: epoch2 })
-            .await
             .unwrap();
         test_env
             .storage
@@ -821,7 +847,6 @@ mod tests {
                 epoch: epoch1,
                 chunk_id: chunk_id1,
             })
-            .await
             .unwrap();
         let epoch3 = epoch2.next_epoch();
         writer.flush_current_epoch(epoch3, true).await.unwrap();
@@ -1022,7 +1047,6 @@ mod tests {
         // Only reader1 will truncate
         reader1
             .truncate(TruncateOffset::Barrier { epoch: epoch1 })
-            .await
             .unwrap();
 
         match reader1.next_item().await.unwrap() {
@@ -1333,7 +1357,6 @@ mod tests {
                 epoch: epoch1,
                 chunk_id: chunk_ids[0],
             })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids = check_reader(
@@ -1350,7 +1373,6 @@ mod tests {
 
         reader
             .truncate(TruncateOffset::Barrier { epoch: epoch1 })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids = check_reader(
@@ -1369,7 +1391,6 @@ mod tests {
                 epoch: epoch3,
                 chunk_id: chunk_ids[1],
             })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids = check_reader(&mut reader, [(epoch3, None)].iter()).await;
@@ -1408,7 +1429,6 @@ mod tests {
 
         reader
             .truncate(TruncateOffset::Barrier { epoch: epoch1 })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids = check_reader(&mut reader, data[1..].iter()).await;
@@ -1419,7 +1439,6 @@ mod tests {
                 epoch: epoch2,
                 chunk_id: chunk_ids[0],
             })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids = check_reader(&mut reader, data[1..].iter()).await;
@@ -1427,7 +1446,6 @@ mod tests {
 
         reader
             .truncate(TruncateOffset::Barrier { epoch: epoch2 })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids = check_reader(&mut reader, data[2..].iter()).await;
@@ -1476,7 +1494,6 @@ mod tests {
 
         reader
             .truncate(TruncateOffset::Barrier { epoch: epoch1 })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids = check_reader_last_unsealed(&mut reader, data[1..].iter()).await;
@@ -1487,7 +1504,6 @@ mod tests {
                 epoch: epoch2,
                 chunk_id: chunk_ids[0],
             })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids = check_reader_last_unsealed(&mut reader, data[1..].iter()).await;
@@ -1495,7 +1511,6 @@ mod tests {
 
         reader
             .truncate(TruncateOffset::Barrier { epoch: epoch2 })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids = check_reader_last_unsealed(&mut reader, data[2..].iter()).await;
@@ -1503,7 +1518,6 @@ mod tests {
 
         reader
             .truncate(TruncateOffset::Barrier { epoch: epoch3 })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids = check_reader_last_unsealed(&mut reader, data[3..].iter()).await;
@@ -1514,7 +1528,6 @@ mod tests {
                 epoch: epoch4,
                 chunk_id: chunk_ids[0],
             })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids = check_reader_last_unsealed(
@@ -1526,7 +1539,6 @@ mod tests {
 
         reader
             .truncate(TruncateOffset::Barrier { epoch: epoch4 })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids =
@@ -1538,10 +1550,234 @@ mod tests {
                 epoch: epoch5,
                 chunk_id: chunk_ids[0],
             })
-            .await
             .unwrap();
         reader.rewind().await.unwrap();
         let chunk_ids = check_reader_last_unsealed(&mut reader, empty()).await;
         assert!(chunk_ids.is_empty());
     }
+
+    async fn validate_reader(
+        reader: &mut impl LogReader,
+        expected: impl IntoIterator<Item = (u64, LogStoreReadItem)>,
+    ) {
+        for (expected_epoch, expected_item) in expected {
+            let (epoch, item) = reader.next_item().await.unwrap();
+            assert_eq!(expected_epoch, epoch);
+            match (expected_item, item) {
+                (
+                    LogStoreReadItem::StreamChunk {
+                        chunk: expected_chunk,
+                        ..
+                    },
+                    LogStoreReadItem::StreamChunk { chunk, .. },
+                ) => {
+                    check_stream_chunk_eq(&expected_chunk, &chunk);
+                }
+                (
+                    LogStoreReadItem::Barrier {
+                        is_checkpoint: expected_is_checkpoint,
+                    },
+                    LogStoreReadItem::Barrier { is_checkpoint },
+                ) => {
+                    assert_eq!(expected_is_checkpoint, is_checkpoint);
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_truncate_historical() {
+        #[expect(deprecated)]
+        test_truncate_historical_inner(
+            10,
+            &crate::common::log_store_impl::kv_log_store::v1::KV_LOG_STORE_V1_INFO,
+        )
+        .await;
+        test_truncate_historical_inner(10, &KV_LOG_STORE_V2_INFO).await;
+    }
+
+    async fn test_truncate_historical_inner(
+        max_row_count: usize,
+        pk_info: &'static KvLogStorePkInfo,
+    ) {
+        let gen_stream_chunk = |base| gen_stream_chunk_with_info(base, pk_info);
+        let test_env = prepare_hummock_test_env().await;
+
+        let table = gen_test_log_store_table(pk_info);
+
+        test_env.register_table(table.clone()).await;
+
+        let stream_chunk1 = gen_stream_chunk(0);
+        let stream_chunk2 = gen_stream_chunk(10);
+        let bitmap = calculate_vnode_bitmap(stream_chunk1.rows().chain(stream_chunk2.rows()));
+        let bitmap = Arc::new(bitmap);
+
+        let factory = KvLogStoreFactory::new(
+            test_env.storage.clone(),
+            table.clone(),
+            Some(bitmap.clone()),
+            max_row_count,
+            KvLogStoreMetrics::for_test(),
+            "test",
+            pk_info,
+        );
+        let (mut reader, mut writer) = factory.build().await;
+
+        let epoch1 = test_env
+            .storage
+            .get_pinned_version()
+            .version()
+            .max_committed_epoch
+            .next_epoch();
+        writer
+            .init(EpochPair::new_test_epoch(epoch1), false)
+            .await
+            .unwrap();
+        writer.write_chunk(stream_chunk1.clone()).await.unwrap();
+        let epoch2 = epoch1.next_epoch();
+        writer.flush_current_epoch(epoch2, false).await.unwrap();
+        writer.write_chunk(stream_chunk2.clone()).await.unwrap();
+        let epoch3 = epoch2.next_epoch();
+        writer.flush_current_epoch(epoch3, true).await.unwrap();
+
+        test_env.storage.seal_epoch(epoch1, false);
+        test_env.commit_epoch(epoch2).await;
+
+        reader.init().await.unwrap();
+        validate_reader(
+            &mut reader,
+            [
+                (
+                    epoch1,
+                    LogStoreReadItem::StreamChunk {
+                        chunk: stream_chunk1.clone(),
+                        chunk_id: 0,
+                    },
+                ),
+                (
+                    epoch1,
+                    LogStoreReadItem::Barrier {
+                        is_checkpoint: false,
+                    },
+                ),
+                (
+                    epoch2,
+                    LogStoreReadItem::StreamChunk {
+                        chunk: stream_chunk2.clone(),
+                        chunk_id: 0,
+                    },
+                ),
+                (
+                    epoch2,
+                    LogStoreReadItem::Barrier {
+                        is_checkpoint: true,
+                    },
+                ),
+            ],
+        )
+        .await;
+
+        drop(writer);
+
+        // Recovery
+        test_env.storage.clear_shared_buffer(epoch2).await;
+
+        // Rebuild log reader and writer in recovery
+        let factory = KvLogStoreFactory::new(
+            test_env.storage.clone(),
+            table.clone(),
+            Some(bitmap.clone()),
+            max_row_count,
+            KvLogStoreMetrics::for_test(),
+            "test",
+            pk_info,
+        );
+        let (mut reader, mut writer) = factory.build().await;
+        writer
+            .init(EpochPair::new_test_epoch(epoch3), false)
+            .await
+            .unwrap();
+        reader.init().await.unwrap();
+        validate_reader(
+            &mut reader,
+            [
+                (
+                    epoch1,
+                    LogStoreReadItem::StreamChunk {
+                        chunk: stream_chunk1.clone(),
+                        chunk_id: 0,
+                    },
+                ),
+                (
+                    epoch1,
+                    LogStoreReadItem::Barrier {
+                        is_checkpoint: false,
+                    },
+                ),
+                (
+                    epoch2,
+                    LogStoreReadItem::StreamChunk {
+                        chunk: stream_chunk2.clone(),
+                        chunk_id: 0,
+                    },
+                ),
+                (
+                    epoch2,
+                    LogStoreReadItem::Barrier {
+                        is_checkpoint: true,
+                    },
+                ),
+            ],
+        )
+        .await;
+        // The truncate should take effect
+        reader
+            .truncate(TruncateOffset::Barrier { epoch: epoch1 })
+            .unwrap();
+        let epoch4 = epoch3.next_epoch();
+        writer.flush_current_epoch(epoch4, true).await.unwrap();
+        test_env.commit_epoch(epoch3).await;
+
+        drop(writer);
+
+        // Recovery
+        test_env.storage.clear_shared_buffer(epoch3).await;
+
+        // Rebuild log reader and writer in recovery
+        let factory = KvLogStoreFactory::new(
+            test_env.storage.clone(),
+            table.clone(),
+            Some(bitmap),
+            max_row_count,
+            KvLogStoreMetrics::for_test(),
+            "test",
+            pk_info,
+        );
+        let (mut reader, mut writer) = factory.build().await;
+        writer
+            .init(EpochPair::new_test_epoch(epoch4), false)
+            .await
+            .unwrap();
+        reader.init().await.unwrap();
+        validate_reader(
+            &mut reader,
+            [
+                (
+                    epoch2,
+                    LogStoreReadItem::StreamChunk {
+                        chunk: stream_chunk2.clone(),
+                        chunk_id: 0,
+                    },
+                ),
+                (
+                    epoch2,
+                    LogStoreReadItem::Barrier {
+                        is_checkpoint: true,
+                    },
+                ),
+            ],
+        )
+        .await;
+    }
 }
diff --git a/src/stream/src/common/log_store_impl/kv_log_store/reader.rs b/src/stream/src/common/log_store_impl/kv_log_store/reader.rs
index 3e9e9ad4833e8..4df114ab3df11 100644
--- a/src/stream/src/common/log_store_impl/kv_log_store/reader.rs
+++ b/src/stream/src/common/log_store_impl/kv_log_store/reader.rs
@@ -13,12 +13,15 @@
 // limitations under the License.
 
 use std::future::Future;
+use std::ops::Bound;
 use std::ops::Bound::{Excluded, Included, Unbounded};
 use std::pin::Pin;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use anyhow::anyhow;
-use foyer::memory::CacheContext;
+use await_tree::InstrumentAwait;
+use bytes::Bytes;
+use foyer::CacheContext;
 use futures::future::{try_join_all, BoxFuture};
 use futures::{FutureExt, TryFutureExt};
 use risingwave_common::array::StreamChunk;
@@ -30,11 +33,14 @@ use risingwave_common::util::epoch::EpochExt;
 use risingwave_connector::sink::log_store::{
     ChunkId, LogReader, LogStoreReadItem, LogStoreResult, TruncateOffset,
 };
-use risingwave_hummock_sdk::key::prefixed_range_with_vnode;
+use risingwave_hummock_sdk::key::{prefixed_range_with_vnode, FullKey, TableKey, TableKeyRange};
 use risingwave_hummock_sdk::HummockEpoch;
+use risingwave_storage::error::StorageResult;
 use risingwave_storage::hummock::CachePolicy;
-use risingwave_storage::store::{PrefetchOptions, ReadOptions};
-use risingwave_storage::StateStore;
+use risingwave_storage::store::{
+    PrefetchOptions, ReadOptions, StateStoreIterItemRef, StateStoreRead,
+};
+use risingwave_storage::{StateStore, StateStoreIter};
 use tokio::sync::watch;
 use tokio::time::sleep;
 use tokio_stream::StreamExt;
@@ -112,7 +118,7 @@ pub struct KvLogStoreReader<S: StateStore> {
     first_write_epoch: Option<u64>,
 
     /// `Some` means consuming historical log data
-    state_store_stream: Option<Pin<Box<LogStoreItemMergeStream<S::Iter>>>>,
+    state_store_stream: Option<Pin<Box<LogStoreItemMergeStream<TimeoutAutoRebuildIter<S>>>>>,
 
     /// Store the future that attempts to read a flushed stream chunk.
     /// This is for cancellation safety. Since it is possible that the future of `next_item`
@@ -167,7 +173,7 @@ impl<S: StateStore> KvLogStoreReader<S> {
         &mut self,
     ) -> LogStoreResult<Option<(ChunkId, StreamChunk, u64)>> {
         if let Some(future) = self.read_flushed_chunk_future.as_mut() {
-            let result = future.await;
+            let result = future.instrument_await("Read Flushed Chunk").await;
             let _fut = self
                 .read_flushed_chunk_future
                 .take()
@@ -179,12 +185,141 @@ impl<S: StateStore> KvLogStoreReader<S> {
     }
 }
 
+struct AutoRebuildStateStoreReadIter<S: StateStoreRead, F> {
+    state_store: S,
+    iter: S::Iter,
+    // call to get whether to rebuild the iter. Once return true, the closure should reset itself.
+    should_rebuild: F,
+    end_bound: Bound<TableKey<Bytes>>,
+    epoch: HummockEpoch,
+    options: ReadOptions,
+}
+
+impl<S: StateStoreRead, F: FnMut() -> bool> AutoRebuildStateStoreReadIter<S, F> {
+    async fn new(
+        state_store: S,
+        should_rebuild: F,
+        range: TableKeyRange,
+        epoch: HummockEpoch,
+        options: ReadOptions,
+    ) -> StorageResult<Self> {
+        let (start_bound, end_bound) = range;
+        let iter = state_store
+            .iter((start_bound, end_bound.clone()), epoch, options.clone())
+            .await?;
+        Ok(Self {
+            state_store,
+            iter,
+            should_rebuild,
+            end_bound,
+            epoch,
+            options,
+        })
+    }
+}
+
+type TimeoutAutoRebuildIter<S: StateStoreRead> =
+    AutoRebuildStateStoreReadIter<S, impl FnMut() -> bool + Send>;
+
+async fn iter_with_timeout_rebuild<S: StateStoreRead>(
+    state_store: S,
+    range: TableKeyRange,
+    epoch: HummockEpoch,
+    options: ReadOptions,
+    timeout: Duration,
+) -> StorageResult<TimeoutAutoRebuildIter<S>> {
+    const CHECK_TIMEOUT_PERIOD: usize = 100;
+    // use a struct here to avoid accidental copy instead of move on primitive usize
+    struct Count(usize);
+    let mut check_count = Count(0);
+    let mut total_count = Count(0);
+    let mut curr_iter_item_count = Count(0);
+    let mut start_time = Instant::now();
+    let initial_start_time = start_time;
+    AutoRebuildStateStoreReadIter::new(
+        state_store,
+        move || {
+            check_count.0 += 1;
+            curr_iter_item_count.0 += 1;
+            total_count.0 += 1;
+            if check_count.0 == CHECK_TIMEOUT_PERIOD {
+                check_count.0 = 0;
+                if start_time.elapsed() > timeout {
+                    let prev_iter_item_count = curr_iter_item_count.0;
+                    curr_iter_item_count.0 = 0;
+                    start_time = Instant::now();
+                    info!(
+                        table_id = options.table_id.table_id,
+                        iter_exist_time_secs = initial_start_time.elapsed().as_secs(),
+                        prev_iter_item_count,
+                        total_iter_item_count = total_count.0,
+                        "kv log store iter is rebuilt"
+                    );
+                    true
+                } else {
+                    false
+                }
+            } else {
+                false
+            }
+        },
+        range,
+        epoch,
+        options,
+    )
+    .await
+}
+
+impl<S: StateStoreRead, F: FnMut() -> bool + Send> StateStoreIter
+    for AutoRebuildStateStoreReadIter<S, F>
+{
+    async fn try_next(&mut self) -> StorageResult<Option<StateStoreIterItemRef<'_>>> {
+        let should_rebuild = (self.should_rebuild)();
+        if should_rebuild {
+            let Some((key, _value)) = self.iter.try_next().await? else {
+                return Ok(None);
+            };
+            let key: FullKey<&[u8]> = key;
+            let range_start = Bytes::copy_from_slice(key.user_key.table_key.as_ref());
+            let new_iter = self
+                .state_store
+                .iter(
+                    (
+                        Included(TableKey(range_start.clone())),
+                        self.end_bound.clone(),
+                    ),
+                    self.epoch,
+                    self.options.clone(),
+                )
+                .await?;
+            self.iter = new_iter;
+            let item: Option<StateStoreIterItemRef<'_>> = self.iter.try_next().await?;
+            if let Some((key, value)) = item {
+                assert_eq!(
+                    key.user_key.table_key.0,
+                    range_start.as_ref(),
+                    "the first key should be the previous key"
+                );
+                Ok(Some((key, value)))
+            } else {
+                unreachable!(
+                    "the first key should be the previous key {:?}, but get None",
+                    range_start
+                )
+            }
+        } else {
+            self.iter.try_next().await
+        }
+    }
+}
+
 impl<S: StateStore> KvLogStoreReader<S> {
     fn read_persisted_log_store(
         &self,
         last_persisted_epoch: Option<u64>,
-    ) -> impl Future<Output = LogStoreResult<Pin<Box<LogStoreItemMergeStream<S::Iter>>>>> + Send
-    {
+    ) -> impl Future<
+        Output = LogStoreResult<Pin<Box<LogStoreItemMergeStream<TimeoutAutoRebuildIter<S>>>>>,
+    > + Send {
         let range_start = if let Some(last_persisted_epoch) = last_persisted_epoch {
             // start from the next epoch of last_persisted_epoch
             Included(
@@ -209,19 +344,21 @@ impl<S: StateStore> KvLogStoreReader<S> {
             );
             let state_store = self.state_store.clone();
             async move {
-                state_store
-                    .iter(
-                        key_range,
-                        HummockEpoch::MAX,
-                        ReadOptions {
-                            // This stream lives too long, the connection of prefetch object may break. So use a short connection prefetch.
-                            prefetch_options: PrefetchOptions::prefetch_for_small_range_scan(),
-                            cache_policy: CachePolicy::Fill(CacheContext::LruPriorityLow),
-                            table_id,
-                            ..Default::default()
-                        },
-                    )
-                    .await
+                // rebuild the iter every 10 minutes to avoid pinning hummock version for too long
+                iter_with_timeout_rebuild(
+                    state_store,
+                    key_range,
+                    HummockEpoch::MAX,
+                    ReadOptions {
+                        // This stream lives too long, the connection of prefetch object may break. So use a short connection prefetch.
+                        prefetch_options: PrefetchOptions::prefetch_for_small_range_scan(),
+                        cache_policy: CachePolicy::Fill(CacheContext::LowPriority),
+                        table_id,
+                        ..Default::default()
+                    },
+                    Duration::from_secs(10 * 60),
+                )
+                .await
             }
         }));
 
@@ -256,11 +393,16 @@ impl<S: StateStore> LogReader for KvLogStoreReader<S> {
             info!("next_item of {} get blocked by is_pause", self.identity);
             self.is_paused
                 .changed()
+                .instrument_await("Wait for Pause Resume")
                 .await
                 .map_err(|_| anyhow!("unable to subscribe resume"))?;
         }
         if let Some(state_store_stream) = &mut self.state_store_stream {
-            match state_store_stream.try_next().await? {
+            match state_store_stream
+                .try_next()
+                .instrument_await("Try Next for Historical Stream")
+                .await?
+            {
                 Some((epoch, item)) => {
                     if let Some(latest_offset) = &self.latest_offset {
                         latest_offset.check_next_item_epoch(epoch)?;
@@ -307,7 +449,11 @@ impl<S: StateStore> LogReader for KvLogStoreReader<S> {
         }
 
         // Now the historical state store has been consumed.
-        let (item_epoch, item) = self.rx.next_item().await;
+        let (item_epoch, item) = self
+            .rx
+            .next_item()
+            .instrument_await("Wait Next Item from Buffer")
+            .await;
         if let Some(latest_offset) = &self.latest_offset {
             latest_offset.check_next_item_epoch(item_epoch)?;
         }
@@ -358,7 +504,7 @@ impl<S: StateStore> LogReader for KvLogStoreReader<S> {
                                             ReadOptions {
                                                 prefetch_options:
                                                     PrefetchOptions::prefetch_for_large_range_scan(),
-                                                cache_policy: CachePolicy::Fill(CacheContext::LruPriorityLow),
+                                                cache_policy: CachePolicy::Fill(CacheContext::LowPriority),
                                                 table_id,
                                                 ..Default::default()
                                             },
@@ -367,6 +513,7 @@ impl<S: StateStore> LogReader for KvLogStoreReader<S> {
                                 )
                             }
                         }))
+                            .instrument_await("Wait Create Iter Stream")
                         .await?;
 
                         let chunk = serde
@@ -377,6 +524,7 @@ impl<S: StateStore> LogReader for KvLogStoreReader<S> {
                                 item_epoch,
                                 &read_metrics,
                             )
+                            .instrument_await("Deserialize Stream Chunk")
                             .await?;
 
                         Ok((chunk_id, chunk, item_epoch))
@@ -435,7 +583,7 @@ impl<S: StateStore> LogReader for KvLogStoreReader<S> {
         })
     }
 
-    async fn truncate(&mut self, offset: TruncateOffset) -> LogStoreResult<()> {
+    fn truncate(&mut self, offset: TruncateOffset) -> LogStoreResult<()> {
         if offset > self.latest_offset.expect("should exist before truncation") {
             return Err(anyhow!(
                 "truncate at a later offset {:?} than the current latest offset {:?}",
@@ -453,11 +601,11 @@ impl<S: StateStore> LogReader for KvLogStoreReader<S> {
             }
         }
         if offset.epoch() >= self.first_write_epoch.expect("should have init") {
-            self.rx.truncate(offset);
+            self.rx.truncate_buffer(offset);
         } else {
             // For historical data, no need to truncate at seq id level. Only truncate at barrier.
-            if let TruncateOffset::Barrier { .. } = &offset {
-                self.rx.truncate(offset);
+            if let TruncateOffset::Barrier { epoch } = &offset {
+                self.rx.truncate_historical(*epoch);
             }
         }
         self.truncate_offset = Some(offset);
@@ -488,3 +636,98 @@ impl<S: StateStore> LogReader for KvLogStoreReader<S> {
         Ok((true, Some((**self.serde.vnodes()).clone())))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::ops::Bound::Unbounded;
+
+    use bytes::Bytes;
+    use itertools::Itertools;
+    use risingwave_common::util::epoch::test_epoch;
+    use risingwave_hummock_sdk::key::TableKey;
+    use risingwave_storage::hummock::iterator::test_utils::{
+        iterator_test_table_key_of, iterator_test_value_of,
+    };
+    use risingwave_storage::memory::MemoryStateStore;
+    use risingwave_storage::storage_value::StorageValue;
+    use risingwave_storage::store::{ReadOptions, StateStoreRead, StateStoreWrite, WriteOptions};
+    use risingwave_storage::StateStoreIter;
+
+    use crate::common::log_store_impl::kv_log_store::reader::AutoRebuildStateStoreReadIter;
+    use crate::common::log_store_impl::kv_log_store::test_utils::TEST_TABLE_ID;
+
+    #[tokio::test]
+    async fn test_auto_rebuild_iter() {
+        let state_store = MemoryStateStore::new();
+        let key_count = 100;
+        let pairs = (0..key_count)
+            .map(|i| {
+                let key = iterator_test_table_key_of(i);
+                let value = iterator_test_value_of(i);
+                (TableKey(Bytes::from(key)), StorageValue::new_put(value))
+            })
+            .collect_vec();
+        let epoch = test_epoch(1);
+        state_store
+            .ingest_batch(
+                pairs.clone(),
+                vec![],
+                WriteOptions {
+                    epoch,
+                    table_id: TEST_TABLE_ID,
+                },
+            )
+            .unwrap();
+
+        async fn validate(
+            mut kv_iter: impl Iterator<Item = (TableKey<Bytes>, StorageValue)>,
+            mut iter: impl StateStoreIter,
+        ) {
+            while let Some((key, value)) = iter.try_next().await.unwrap() {
+                let (k, v) = kv_iter.next().unwrap();
+                assert_eq!(key.user_key.table_key, k.to_ref());
+                assert_eq!(v.user_value.as_deref(), Some(value));
+            }
+            assert!(kv_iter.next().is_none());
+        }
+
+        let read_options = ReadOptions {
+            table_id: TEST_TABLE_ID,
+            ..Default::default()
+        };
+
+        let kv_iter = pairs.clone().into_iter();
+        let iter = state_store
+            .iter((Unbounded, Unbounded), epoch, read_options.clone())
+            .await
+            .unwrap();
+        validate(kv_iter, iter).await;
+
+        let kv_iter = pairs.clone().into_iter();
+        let mut count = 0;
+        let count_mut_ref = &mut count;
+        let rebuild_period = 8;
+        let mut rebuild_count = 0;
+        let rebuild_count_mut_ref = &mut rebuild_count;
+        let iter = AutoRebuildStateStoreReadIter::new(
+            state_store,
+            move || {
+                *count_mut_ref += 1;
+                if *count_mut_ref % rebuild_period == 0 {
+                    *rebuild_count_mut_ref += 1;
+                    true
+                } else {
+                    false
+                }
+            },
+            (Unbounded, Unbounded),
+            epoch,
+            read_options,
+        )
+        .await
+        .unwrap();
+        validate(kv_iter, iter).await;
+        assert_eq!(count, key_count + 1); // with an extra call on the last None
+        assert_eq!(rebuild_count, key_count / rebuild_period);
+    }
+}
diff --git a/src/stream/src/common/log_store_impl/kv_log_store/serde.rs b/src/stream/src/common/log_store_impl/kv_log_store/serde.rs
index 67167f466a50b..57d838044171f 100644
--- a/src/stream/src/common/log_store_impl/kv_log_store/serde.rs
+++ b/src/stream/src/common/log_store_impl/kv_log_store/serde.rs
@@ -544,7 +544,7 @@ impl<S: StateStoreReadIter> LogStoreRowOpStream<S> {
     }
 }
 
-pub(crate) type LogStoreItemMergeStream<S> =
+pub(crate) type LogStoreItemMergeStream<S: StateStoreReadIter> =
     impl Stream<Item = LogStoreResult<(u64, KvLogStoreItem)>>;
 pub(crate) fn merge_log_store_item_stream<S: StateStoreReadIter>(
     iters: Vec<S>,
@@ -742,7 +742,7 @@ impl<S: StateStoreReadIter> LogStoreRowOpStream<S> {
                 "a stream has reached the end but some other stream has not started yet"
             ));
         }
-        if cfg!(debug_assertion) {
+        if cfg!(debug_assertions) {
             while let Some((opt, _stream)) = self.row_streams.next().await {
                 if let Some(result) = opt {
                     return Err(
diff --git a/src/stream/src/common/log_store_impl/mod.rs b/src/stream/src/common/log_store_impl/mod.rs
index bcd15ab12a7da..bd600fc80086f 100644
--- a/src/stream/src/common/log_store_impl/mod.rs
+++ b/src/stream/src/common/log_store_impl/mod.rs
@@ -14,4 +14,3 @@
 
 pub mod in_mem;
 pub mod kv_log_store;
-pub mod subscription_log_store;
diff --git a/src/stream/src/common/log_store_impl/subscription_log_store.rs b/src/stream/src/common/log_store_impl/subscription_log_store.rs
deleted file mode 100644
index c7de7073f85ed..0000000000000
--- a/src/stream/src/common/log_store_impl/subscription_log_store.rs
+++ /dev/null
@@ -1,113 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use itertools::Itertools;
-use risingwave_common::array::StreamChunk;
-use risingwave_common::buffer::Bitmap;
-use risingwave_common::catalog::TableId;
-use risingwave_connector::sink::log_store::LogStoreResult;
-use risingwave_hummock_sdk::table_watermark::{VnodeWatermark, WatermarkDirection};
-use risingwave_storage::store::{InitOptions, LocalStateStore, SealCurrentEpochOptions};
-
-use super::kv_log_store::ReaderTruncationOffsetType;
-use crate::common::log_store_impl::kv_log_store::serde::LogStoreRowSerde;
-use crate::common::log_store_impl::kv_log_store::{SeqIdType, FIRST_SEQ_ID};
-
-pub struct SubscriptionLogStoreWriter<LS: LocalStateStore> {
-    _table_id: TableId,
-
-    seq_id: SeqIdType,
-
-    state_store: LS,
-
-    serde: LogStoreRowSerde,
-
-    _identity: String,
-}
-
-impl<LS: LocalStateStore> SubscriptionLogStoreWriter<LS> {
-    pub(crate) fn new(
-        table_id: TableId,
-        state_store: LS,
-        serde: LogStoreRowSerde,
-        identity: String,
-    ) -> Self {
-        Self {
-            _table_id: table_id,
-            seq_id: FIRST_SEQ_ID,
-            state_store,
-            serde,
-            _identity: identity,
-        }
-    }
-
-    pub async fn init(
-        &mut self,
-        epoch: risingwave_common::util::epoch::EpochPair,
-        _pause_read_on_bootstrap: bool,
-    ) -> LogStoreResult<()> {
-        self.state_store.init(InitOptions::new(epoch)).await?;
-        self.seq_id = FIRST_SEQ_ID;
-        Ok(())
-    }
-
-    pub fn write_chunk(&mut self, chunk: StreamChunk) -> LogStoreResult<()> {
-        if chunk.cardinality() == 0 {
-            return Ok(());
-        }
-        let epoch = self.state_store.epoch();
-        let start_seq_id = self.seq_id;
-        self.seq_id += chunk.cardinality() as SeqIdType;
-        for (i, (op, row)) in chunk.rows().enumerate() {
-            let seq_id = start_seq_id + (i as SeqIdType);
-            let (_vnode, key, value) = self.serde.serialize_data_row(epoch, seq_id, op, row);
-            self.state_store.insert(key, value, None)?;
-        }
-        Ok(())
-    }
-
-    pub async fn flush_current_epoch(
-        &mut self,
-        next_epoch: u64,
-        truncate_offset: Option<ReaderTruncationOffsetType>,
-    ) -> LogStoreResult<()> {
-        // Because barrier has no effect on subscription, barrier will not be inserted here
-        let watermark = truncate_offset.map(|truncate_offset| {
-            VnodeWatermark::new(
-                self.serde.vnodes().clone(),
-                self.serde
-                    .serialize_truncation_offset_watermark(truncate_offset),
-            )
-        });
-        self.state_store.flush().await?;
-        let watermark = watermark.into_iter().collect_vec();
-        self.state_store.seal_current_epoch(
-            next_epoch,
-            SealCurrentEpochOptions {
-                table_watermarks: Some((WatermarkDirection::Ascending, watermark)),
-                switch_op_consistency_level: None,
-            },
-        );
-        self.seq_id = FIRST_SEQ_ID;
-        Ok(())
-    }
-
-    pub fn update_vnode_bitmap(&mut self, new_vnodes: Arc<Bitmap>) -> LogStoreResult<()> {
-        self.serde.update_vnode_bitmap(new_vnodes.clone());
-        self.state_store.update_vnode_bitmap(new_vnodes);
-        Ok(())
-    }
-}
diff --git a/src/stream/src/common/metrics.rs b/src/stream/src/common/metrics.rs
index 2c82bee7d97d9..fb7267d3fd5d1 100644
--- a/src/stream/src/common/metrics.rs
+++ b/src/stream/src/common/metrics.rs
@@ -30,13 +30,13 @@ impl MetricsInfo {
         metrics: Arc<StreamingMetrics>,
         table_id: u32,
         actor_id: ActorId,
-        desc: &str,
+        desc: impl Into<String>,
     ) -> Self {
         Self {
             metrics,
             table_id: table_id.to_string(),
             actor_id: actor_id.to_string(),
-            desc: desc.to_string(),
+            desc: desc.into(),
         }
     }
 
diff --git a/src/stream/src/common/mod.rs b/src/stream/src/common/mod.rs
index 23dc006bf8d63..9d79cecd2ec53 100644
--- a/src/stream/src/common/mod.rs
+++ b/src/stream/src/common/mod.rs
@@ -19,4 +19,5 @@ mod column_mapping;
 pub mod compact_chunk;
 pub mod log_store_impl;
 pub mod metrics;
+pub mod rate_limit;
 pub mod table;
diff --git a/java/udf/src/main/java/com/risingwave/functions/DataTypeHint.java b/src/stream/src/common/rate_limit.rs
similarity index 68%
rename from java/udf/src/main/java/com/risingwave/functions/DataTypeHint.java
rename to src/stream/src/common/rate_limit.rs
index 7baf0fe4c6115..6144ae6810eba 100644
--- a/java/udf/src/main/java/com/risingwave/functions/DataTypeHint.java
+++ b/src/stream/src/common/rate_limit.rs
@@ -12,12 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.risingwave.functions;
-
-import java.lang.annotation.*;
-
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.METHOD, ElementType.FIELD, ElementType.PARAMETER})
-public @interface DataTypeHint {
-    String value();
+/// Get the rate-limited max chunk size.
+pub(crate) fn limited_chunk_size(rate_limit: Option<u32>) -> usize {
+    let config_chunk_size = crate::config::chunk_size();
+    rate_limit
+        .map(|limit| config_chunk_size.min(limit as usize))
+        .unwrap_or(config_chunk_size)
 }
diff --git a/src/stream/src/common/table/state_table.rs b/src/stream/src/common/table/state_table.rs
index 09e0488f6ff91..1d1a5e2a6047f 100644
--- a/src/stream/src/common/table/state_table.rs
+++ b/src/stream/src/common/table/state_table.rs
@@ -13,14 +13,13 @@
 // limitations under the License.
 
 use std::collections::HashMap;
-use std::default::Default;
 use std::ops::Bound;
 use std::ops::Bound::*;
 use std::sync::Arc;
 
 use bytes::{BufMut, Bytes, BytesMut};
 use either::Either;
-use foyer::memory::CacheContext;
+use foyer::CacheContext;
 use futures::{pin_mut, FutureExt, Stream, StreamExt, TryStreamExt};
 use futures_async_stream::for_await;
 use itertools::{izip, Itertools};
@@ -165,7 +164,7 @@ pub struct StateTableInner<
     /// 2. Computing output pk indices to used them for backfill state.
     output_indices: Vec<usize>,
 
-    is_consistent_op: bool,
+    op_consistency_level: StateTableOpConsistencyLevel,
 }
 
 /// `StateTable` will use `BasicSerde` as default
@@ -218,32 +217,51 @@ where
     }
 }
 
-fn consistent_old_value_op(row_serde: impl ValueRowSerde) -> OpConsistencyLevel {
-    OpConsistencyLevel::ConsistentOldValue(Arc::new(move |first: &Bytes, second: &Bytes| {
-        if first == second {
-            return true;
-        }
-        let first = match row_serde.deserialize(first) {
-            Ok(rows) => rows,
-            Err(e) => {
-                error!(error = %e.as_report(), value = ?first, "fail to deserialize serialized value");
-                return false;
+fn consistent_old_value_op(
+    row_serde: impl ValueRowSerde,
+    is_log_store: bool,
+) -> OpConsistencyLevel {
+    OpConsistencyLevel::ConsistentOldValue {
+        check_old_value: Arc::new(move |first: &Bytes, second: &Bytes| {
+            if first == second {
+                return true;
             }
-        };
-        let second = match row_serde.deserialize(second) {
-            Ok(rows) => rows,
-            Err(e) => {
-                error!(error = %e.as_report(), value = ?second, "fail to deserialize serialized value");
-                return false;
+            let first = match row_serde.deserialize(first) {
+                Ok(rows) => rows,
+                Err(e) => {
+                    error!(error = %e.as_report(), value = ?first, "fail to deserialize serialized value");
+                    return false;
+                }
+            };
+            let second = match row_serde.deserialize(second) {
+                Ok(rows) => rows,
+                Err(e) => {
+                    error!(error = %e.as_report(), value = ?second, "fail to deserialize serialized value");
+                    return false;
+                }
+            };
+            if first != second {
+                error!(first = ?first, second = ?second, "sanity check fail");
+                false
+            } else {
+                true
             }
-        };
-        if first != second {
-            error!(first = ?first, second = ?second, "sanity check fail");
-            false
-        } else {
-            true
-        }
-    }))
+        }),
+        is_log_store,
+    }
+}
+
+#[derive(Eq, PartialEq, Copy, Clone, Debug)]
+pub enum StateTableOpConsistencyLevel {
+    /// Op is inconsistent
+    Inconsistent,
+    /// Op is consistent.
+    /// - Insert op should ensure that the key does not exist previously
+    /// - Delete and Update op should ensure that the key exists and the previous value matches the passed old value
+    ConsistentOldValue,
+    /// The requirement on operation consistency is the same as `ConsistentOldValue`.
+    /// The difference is that in the `LogStoreEnabled`, the state table should also flush and store and old value.
+    LogStoreEnabled,
 }
 
 // initialize
@@ -258,12 +276,20 @@ where
     W: WatermarkBufferStrategy,
 {
     /// Create state table from table catalog and store.
+    ///
+    /// If `vnodes` is `None`, [`TableDistribution::singleton()`] will be used.
     pub async fn from_table_catalog(
         table_catalog: &Table,
         store: S,
         vnodes: Option<Arc<Bitmap>>,
     ) -> Self {
-        Self::from_table_catalog_inner(table_catalog, store, vnodes, true, vec![]).await
+        Self::from_table_catalog_with_consistency_level(
+            table_catalog,
+            store,
+            vnodes,
+            StateTableOpConsistencyLevel::ConsistentOldValue,
+        )
+        .await
     }
 
     /// Create state table from table catalog and store with sanity check disabled.
@@ -272,7 +298,23 @@ where
         store: S,
         vnodes: Option<Arc<Bitmap>>,
     ) -> Self {
-        Self::from_table_catalog_inner(table_catalog, store, vnodes, false, vec![]).await
+        Self::from_table_catalog_with_consistency_level(
+            table_catalog,
+            store,
+            vnodes,
+            StateTableOpConsistencyLevel::Inconsistent,
+        )
+        .await
+    }
+
+    pub async fn from_table_catalog_with_consistency_level(
+        table_catalog: &Table,
+        store: S,
+        vnodes: Option<Arc<Bitmap>>,
+        consistency_level: StateTableOpConsistencyLevel,
+    ) -> Self {
+        Self::from_table_catalog_inner(table_catalog, store, vnodes, consistency_level, vec![])
+            .await
     }
 
     /// Create state table from table catalog and store.
@@ -280,7 +322,7 @@ where
         table_catalog: &Table,
         store: S,
         vnodes: Option<Arc<Bitmap>>,
-        is_consistent_op: bool,
+        op_consistency_level: StateTableOpConsistencyLevel,
         output_column_ids: Vec<ColumnId>,
     ) -> Self {
         let table_id = TableId::new(table_catalog.id);
@@ -366,18 +408,17 @@ where
             )
         };
 
-        let is_consistent_op = if crate::consistency::insane() {
-            // In insane mode, we will have inconsistent operations applied on the table, even if
-            // our executor code do not expect that.
-            false
-        } else {
-            is_consistent_op
-        };
-        let op_consistency_level = if is_consistent_op {
-            let row_serde = make_row_serde();
-            consistent_old_value_op(row_serde)
-        } else {
-            OpConsistencyLevel::Inconsistent
+        let state_table_op_consistency_level = op_consistency_level;
+        let op_consistency_level = match op_consistency_level {
+            StateTableOpConsistencyLevel::Inconsistent => OpConsistencyLevel::Inconsistent,
+            StateTableOpConsistencyLevel::ConsistentOldValue => {
+                let row_serde = make_row_serde();
+                consistent_old_value_op(row_serde, false)
+            }
+            StateTableOpConsistencyLevel::LogStoreEnabled => {
+                let row_serde = make_row_serde();
+                consistent_old_value_op(row_serde, true)
+            }
         };
 
         let table_option = TableOption::new(table_catalog.retention_seconds);
@@ -463,7 +504,7 @@ where
             data_types,
             output_indices,
             i2o_mapping,
-            is_consistent_op,
+            op_consistency_level: state_table_op_consistency_level,
         }
     }
 
@@ -553,6 +594,7 @@ where
         .await
     }
 
+    /// Create a state table with distribution and without sanity check, used for unit tests.
     pub async fn new_with_distribution_inconsistent_op(
         store: S,
         table_id: TableId,
@@ -599,7 +641,7 @@ where
         };
         let op_consistency_level = if is_consistent_op {
             let row_serde = make_row_serde();
-            consistent_old_value_op(row_serde)
+            consistent_old_value_op(row_serde, false)
         } else {
             OpConsistencyLevel::Inconsistent
         };
@@ -645,7 +687,11 @@ where
             data_types,
             output_indices: vec![],
             i2o_mapping: ColIndexMapping::new(vec![], 0),
-            is_consistent_op,
+            op_consistency_level: if is_consistent_op {
+                StateTableOpConsistencyLevel::ConsistentOldValue
+            } else {
+                StateTableOpConsistencyLevel::Inconsistent
+            },
         }
     }
 
@@ -708,7 +754,11 @@ where
     }
 
     pub fn is_consistent_op(&self) -> bool {
-        self.is_consistent_op
+        matches!(
+            self.op_consistency_level,
+            StateTableOpConsistencyLevel::ConsistentOldValue
+                | StateTableOpConsistencyLevel::LogStoreEnabled
+        )
     }
 }
 
@@ -725,7 +775,14 @@ where
         vnodes: Option<Arc<Bitmap>>,
         output_column_ids: Vec<ColumnId>,
     ) -> Self {
-        Self::from_table_catalog_inner(table_catalog, store, vnodes, false, output_column_ids).await
+        Self::from_table_catalog_inner(
+            table_catalog,
+            store,
+            vnodes,
+            StateTableOpConsistencyLevel::Inconsistent,
+            output_column_ids,
+        )
+        .await
     }
 }
 
@@ -1081,22 +1138,39 @@ where
     }
 
     pub async fn commit(&mut self, new_epoch: EpochPair) -> StreamExecutorResult<()> {
-        self.commit_with_switch_consistent_op(new_epoch, None).await
+        self.commit_inner(new_epoch, None).await
     }
 
-    pub async fn commit_with_switch_consistent_op(
+    pub async fn commit_may_switch_consistent_op(
         &mut self,
         new_epoch: EpochPair,
-        switch_consistent_op: Option<bool>,
+        op_consistency_level: StateTableOpConsistencyLevel,
+    ) -> StreamExecutorResult<()> {
+        if self.op_consistency_level != op_consistency_level {
+            self.commit_inner(new_epoch, Some(op_consistency_level))
+                .await
+        } else {
+            self.commit_inner(new_epoch, None).await
+        }
+    }
+
+    async fn commit_inner(
+        &mut self,
+        new_epoch: EpochPair,
+        switch_consistent_op: Option<StateTableOpConsistencyLevel>,
     ) -> StreamExecutorResult<()> {
         assert_eq!(self.epoch(), new_epoch.prev);
-        let switch_op_consistency_level = switch_consistent_op.map(|enable_consistent_op| {
-            assert_ne!(self.is_consistent_op, enable_consistent_op);
-            self.is_consistent_op = enable_consistent_op;
-            if enable_consistent_op {
-                consistent_old_value_op(self.row_serde.clone())
-            } else {
-                OpConsistencyLevel::Inconsistent
+        let switch_op_consistency_level = switch_consistent_op.map(|new_consistency_level| {
+            assert_ne!(self.op_consistency_level, new_consistency_level);
+            self.op_consistency_level = new_consistency_level;
+            match new_consistency_level {
+                StateTableOpConsistencyLevel::Inconsistent => OpConsistencyLevel::Inconsistent,
+                StateTableOpConsistencyLevel::ConsistentOldValue => {
+                    consistent_old_value_op(self.row_serde.clone(), false)
+                }
+                StateTableOpConsistencyLevel::LogStoreEnabled => {
+                    consistent_old_value_op(self.row_serde.clone(), true)
+                }
             }
         });
         trace!(
@@ -1356,9 +1430,6 @@ where
     /// This function scans rows from the relational table with specific `prefix` and `sub_range` under the same
     /// `vnode`. If `sub_range` is (Unbounded, Unbounded), it scans rows from the relational table with specific `pk_prefix`.
     /// `pk_prefix` is used to identify the exact vnode the scan should perform on.
-
-    /// This function scans rows from the relational table with specific `prefix` and `pk_sub_range` under the same
-    /// `vnode`.
     pub async fn iter_with_prefix(
         &self,
         pk_prefix: impl Row,
diff --git a/src/stream/src/error.rs b/src/stream/src/error.rs
index 7aaffc7824d5e..defefa7b474ae 100644
--- a/src/stream/src/error.rs
+++ b/src/stream/src/error.rs
@@ -87,7 +87,7 @@ pub enum ErrorKind {
     },
 
     #[error(transparent)]
-    Internal(
+    Uncategorized(
         #[from]
         #[backtrace]
         anyhow::Error,
diff --git a/src/stream/src/executor/actor.rs b/src/stream/src/executor/actor.rs
index d876fbe6c33b5..1c73a3aeddad6 100644
--- a/src/stream/src/executor/actor.rs
+++ b/src/stream/src/executor/actor.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::{HashMap, HashSet};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, LazyLock};
 
@@ -19,6 +20,7 @@ use anyhow::anyhow;
 use await_tree::InstrumentAwait;
 use futures::future::join_all;
 use hytra::TrAdder;
+use risingwave_common::catalog::TableId;
 use risingwave_common::log::LogSuppresser;
 use risingwave_common::metrics::GLOBAL_ERROR_METRICS;
 use risingwave_common::util::epoch::EpochPair;
@@ -49,7 +51,10 @@ pub struct ActorContext {
 
     pub streaming_metrics: Arc<StreamingMetrics>,
 
-    pub dispatch_num: usize,
+    /// This is the number of dispatchers when the actor is created. It will not be updated during runtime when new downstreams are added.
+    pub initial_dispatch_num: usize,
+    // mv_table_id to subscription id
+    pub related_subscriptions: HashMap<TableId, HashSet<u32>>,
 }
 
 pub type ActorContextRef = Arc<ActorContext>;
@@ -65,7 +70,8 @@ impl ActorContext {
             total_mem_val: Arc::new(TrAdder::new()),
             streaming_metrics: Arc::new(StreamingMetrics::unused()),
             // Set 1 for test to enable sanity check on table
-            dispatch_num: 1,
+            initial_dispatch_num: 1,
+            related_subscriptions: HashMap::new(),
         })
     }
 
@@ -73,7 +79,8 @@ impl ActorContext {
         stream_actor: &PbStreamActor,
         total_mem_val: Arc<TrAdder<i64>>,
         streaming_metrics: Arc<StreamingMetrics>,
-        dispatch_num: usize,
+        initial_dispatch_num: usize,
+        related_subscriptions: HashMap<TableId, HashSet<u32>>,
     ) -> ActorContextRef {
         Arc::new(Self {
             id: stream_actor.actor_id,
@@ -83,7 +90,8 @@ impl ActorContext {
             last_mem_val: Arc::new(0.into()),
             total_mem_val,
             streaming_metrics,
-            dispatch_num,
+            initial_dispatch_num,
+            related_subscriptions,
         })
     }
 
@@ -215,14 +223,15 @@ where
             )
             .into()));
 
-            // Collect barriers to local barrier manager
-            self.barrier_manager.collect(id, &barrier);
-
             // Then stop this actor if asked
             if barrier.is_stop(id) {
-                break Ok(());
+                debug!(actor_id = id, epoch = ?barrier.epoch, "stop at barrier");
+                break Ok(barrier);
             }
 
+            // Collect barriers to local barrier manager
+            self.barrier_manager.collect(id, &barrier);
+
             // Tracing related work
             last_epoch = Some(barrier.epoch);
             span = barrier.tracing_context().attach(new_span(last_epoch));
@@ -230,7 +239,12 @@ where
 
         spawn_blocking_drop_stream(stream).await;
 
-        tracing::trace!(actor_id = id, "actor exit");
+        let result = result.map(|stop_barrier| {
+            // Collect the stop barrier after the stream has been dropped to ensure that all resources
+            self.barrier_manager.collect(id, &stop_barrier);
+        });
+
+        tracing::debug!(actor_id = id, ok = result.is_ok(), "actor exit");
         result
     }
 }
diff --git a/src/stream/src/executor/aggregation/agg_group.rs b/src/stream/src/executor/aggregation/agg_group.rs
index b374b3d3165ff..325a9001811e6 100644
--- a/src/stream/src/executor/aggregation/agg_group.rs
+++ b/src/stream/src/executor/aggregation/agg_group.rs
@@ -365,10 +365,11 @@ impl<S: StateStore, Strtg: Strategy> AggGroup<S, Strtg> {
 
     /// Reset all in-memory states to their initial state, i.e. to reset all agg state structs to
     /// the status as if they are just created, no input applied and no row in state table.
-    fn reset(&mut self, funcs: &[BoxedAggregateFunction]) {
+    fn reset(&mut self, funcs: &[BoxedAggregateFunction]) -> StreamExecutorResult<()> {
         for (state, func) in self.states.iter_mut().zip_eq_fast(funcs) {
-            state.reset(func);
+            state.reset(func)?;
         }
+        Ok(())
     }
 
     /// Encode intermediate states.
@@ -409,7 +410,7 @@ impl<S: StateStore, Strtg: Strategy> AggGroup<S, Strtg> {
             // they should output NULL, for some other calls (e.g. `sum0`), they should output 0.
             // FIXME(rc): Deciding whether to reset states according to `row_count` is not precisely
             // correct, see https://github.com/risingwavelabs/risingwave/issues/7412 for bug description.
-            self.reset(funcs);
+            self.reset(funcs)?;
         }
         futures::future::try_join_all(
             self.states
diff --git a/src/stream/src/executor/aggregation/agg_state.rs b/src/stream/src/executor/aggregation/agg_state.rs
index bfcea1cf048be..6ef55fcedd5ed 100644
--- a/src/stream/src/executor/aggregation/agg_state.rs
+++ b/src/stream/src/executor/aggregation/agg_state.rs
@@ -81,7 +81,7 @@ impl AggState {
             AggStateStorage::Value => {
                 let state = match encoded_state {
                     Some(encoded) => agg_func.decode_state(encoded.clone())?,
-                    None => agg_func.create_state(),
+                    None => agg_func.create_state()?,
                 };
                 Self::Value(state)
             }
@@ -146,10 +146,11 @@ impl AggState {
     }
 
     /// Reset the value state to initial state.
-    pub fn reset(&mut self, func: &BoxedAggregateFunction) {
+    pub fn reset(&mut self, func: &BoxedAggregateFunction) -> StreamExecutorResult<()> {
         if let Self::Value(state) = self {
             // now only value states need to be reset
-            *state = func.create_state();
+            *state = func.create_state()?;
         }
+        Ok(())
     }
 }
diff --git a/src/stream/src/executor/aggregation/distinct.rs b/src/stream/src/executor/aggregation/distinct.rs
index 8bb298b47c99d..3433708957aba 100644
--- a/src/stream/src/executor/aggregation/distinct.rs
+++ b/src/stream/src/executor/aggregation/distinct.rs
@@ -15,34 +15,50 @@
 use std::collections::HashMap;
 use std::marker::PhantomData;
 use std::sync::atomic::AtomicU64;
-use std::sync::Arc;
 
 use itertools::Itertools;
 use risingwave_common::array::{ArrayRef, Op};
 use risingwave_common::buffer::{Bitmap, BitmapBuilder};
-use risingwave_common::row::{self, CompactedRow, OwnedRow, Row, RowExt};
-use risingwave_common::types::{ScalarImpl, ScalarRefImpl};
+use risingwave_common::row::{self, CompactedRow, RowExt};
 use risingwave_common::util::iter_util::ZipEqFast;
-use risingwave_storage::StateStore;
 
 use super::{AggCall, GroupKey};
-use crate::cache::{new_unbounded, ManagedLruCache};
+use crate::cache::ManagedLruCache;
 use crate::common::metrics::MetricsInfo;
-use crate::common::table::state_table::StateTable;
-use crate::executor::{ActorContextRef, StreamExecutorResult};
+use crate::executor::monitor::AggDistinctDedupMetrics;
+use crate::executor::prelude::*;
 
 type DedupCache = ManagedLruCache<CompactedRow, Box<[i64]>>;
 
 /// Deduplicater for one distinct column.
 struct ColumnDeduplicater<S: StateStore> {
     cache: DedupCache,
+    metrics: AggDistinctDedupMetrics,
     _phantom: PhantomData<S>,
 }
 
 impl<S: StateStore> ColumnDeduplicater<S> {
-    fn new(watermark_epoch: Arc<AtomicU64>, metrics_info: MetricsInfo) -> Self {
+    fn new(
+        distinct_col: usize,
+        dedup_table: &StateTable<S>,
+        watermark_sequence: Arc<AtomicU64>,
+        actor_ctx: &ActorContext,
+    ) -> Self {
+        let metrics_info = MetricsInfo::new(
+            actor_ctx.streaming_metrics.clone(),
+            dedup_table.table_id(),
+            actor_ctx.id,
+            format!("distinct dedup column {}", distinct_col),
+        );
+        let metrics = actor_ctx.streaming_metrics.new_agg_distinct_dedup_metrics(
+            dedup_table.table_id(),
+            actor_ctx.id,
+            actor_ctx.fragment_id,
+        );
+
         Self {
-            cache: new_unbounded(watermark_epoch, metrics_info),
+            cache: ManagedLruCache::unbounded(watermark_sequence, metrics_info),
+            metrics,
             _phantom: PhantomData,
         }
     }
@@ -54,7 +70,6 @@ impl<S: StateStore> ColumnDeduplicater<S> {
         mut visibilities: Vec<&mut Bitmap>,
         dedup_table: &mut StateTable<S>,
         group_key: Option<&GroupKey>,
-        ctx: ActorContextRef,
     ) -> StreamExecutorResult<()> {
         let n_calls = visibilities.len();
 
@@ -64,9 +79,6 @@ impl<S: StateStore> ColumnDeduplicater<S> {
         let mut vis_masks_inv = (0..visibilities.len())
             .map(|_| BitmapBuilder::zeroed(column.len()))
             .collect_vec();
-        let actor_id_str = ctx.id.to_string();
-        let fragment_id_str = ctx.fragment_id.to_string();
-        let table_id_str = dedup_table.table_id().to_string();
         for (datum_idx, (op, datum)) in ops.iter().zip_eq_fast(column.iter()).enumerate() {
             // skip if this item is hidden to all agg calls (this is likely to happen)
             if !visibilities.iter().any(|vis| vis.is_set(datum_idx)) {
@@ -79,19 +91,13 @@ impl<S: StateStore> ColumnDeduplicater<S> {
             let cache_key =
                 CompactedRow::from(group_key.map(GroupKey::cache_key).chain(row::once(datum)));
 
-            ctx.streaming_metrics
-                .agg_distinct_total_cache_count
-                .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-                .inc();
+            self.metrics.agg_distinct_total_cache_count.inc();
             // TODO(yuhao): avoid this `contains`.
             // https://github.com/risingwavelabs/risingwave/issues/9233
             let mut counts = if self.cache.contains(&cache_key) {
                 self.cache.get_mut(&cache_key).unwrap()
             } else {
-                ctx.streaming_metrics
-                    .agg_distinct_cache_miss_count
-                    .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-                    .inc();
+                self.metrics.agg_distinct_cache_miss_count.inc();
                 // load from table into the cache
                 let counts = if let Some(counts_row) =
                     dedup_table.get_row(&table_pk).await? as Option<OwnedRow>
@@ -180,18 +186,14 @@ impl<S: StateStore> ColumnDeduplicater<S> {
     }
 
     /// Flush the deduplication table.
-    fn flush(&mut self, dedup_table: &StateTable<S>, ctx: ActorContextRef) {
+    fn flush(&mut self, _dedup_table: &StateTable<S>) {
         // TODO(rc): now we flush the table in `dedup` method.
         // WARN: if you want to change to batching the write to table. please remember to change
         // `self.cache.evict()` too.
         self.cache.evict();
 
-        let actor_id_str = ctx.id.to_string();
-        let fragment_id_str = ctx.fragment_id.to_string();
-        let table_id_str = dedup_table.table_id().to_string();
-        ctx.streaming_metrics
+        self.metrics
             .agg_distinct_cached_entry_count
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
             .set(self.cache.len() as i64);
     }
 }
@@ -212,7 +214,6 @@ pub struct DistinctDeduplicater<S: StateStore> {
     /// Key: distinct column index;
     /// Value: (agg call indices that distinct on the column, deduplicater for the column).
     deduplicaters: HashMap<usize, (Box<[usize]>, ColumnDeduplicater<S>)>,
-    ctx: ActorContextRef,
 }
 
 impl<S: StateStore> DistinctDeduplicater<S> {
@@ -220,9 +221,8 @@ impl<S: StateStore> DistinctDeduplicater<S> {
         agg_calls: &[AggCall],
         watermark_epoch: Arc<AtomicU64>,
         distinct_dedup_tables: &HashMap<usize, StateTable<S>>,
-        ctx: ActorContextRef,
+        actor_ctx: &ActorContext,
     ) -> Self {
-        let actor_id = ctx.id;
         let deduplicaters: HashMap<_, _> = agg_calls
             .iter()
             .enumerate()
@@ -230,19 +230,18 @@ impl<S: StateStore> DistinctDeduplicater<S> {
             .into_group_map_by(|(_, call)| call.args.val_indices()[0])
             .into_iter()
             .map(|(distinct_col, indices_and_calls)| {
-                let table_id = distinct_dedup_tables.get(&distinct_col).unwrap().table_id();
-                let metrics_info = MetricsInfo::new(
-                    ctx.streaming_metrics.clone(),
-                    table_id,
-                    actor_id,
-                    "distinct dedup",
-                );
+                let dedup_table = distinct_dedup_tables.get(&distinct_col).unwrap();
                 let call_indices: Box<[_]> = indices_and_calls.into_iter().map(|v| v.0).collect();
-                let deduplicater = ColumnDeduplicater::new(watermark_epoch.clone(), metrics_info);
+                let deduplicater = ColumnDeduplicater::new(
+                    distinct_col,
+                    dedup_table,
+                    watermark_epoch.clone(),
+                    actor_ctx,
+                );
                 (distinct_col, (call_indices, deduplicater))
             })
             .collect();
-        Self { deduplicaters, ctx }
+        Self { deduplicaters }
     }
 
     pub fn dedup_caches_mut(&mut self) -> impl Iterator<Item = &mut DedupCache> {
@@ -269,14 +268,7 @@ impl<S: StateStore> DistinctDeduplicater<S> {
             // SAFETY: all items in `agg_call_indices` are unique by nature, see `new`.
             let visibilities = unsafe { get_many_mut_from_slice(&mut visibilities, call_indices) };
             deduplicater
-                .dedup(
-                    ops,
-                    column,
-                    visibilities,
-                    dedup_table,
-                    group_key,
-                    self.ctx.clone(),
-                )
+                .dedup(ops, column, visibilities, dedup_table, group_key)
                 .await?;
         }
         Ok(visibilities)
@@ -289,7 +281,7 @@ impl<S: StateStore> DistinctDeduplicater<S> {
     ) -> StreamExecutorResult<()> {
         for (distinct_col, (_, deduplicater)) in &mut self.deduplicaters {
             let dedup_table = dedup_tables.get_mut(distinct_col).unwrap();
-            deduplicater.flush(dedup_table, self.ctx.clone());
+            deduplicater.flush(dedup_table);
         }
         Ok(())
     }
@@ -297,16 +289,13 @@ impl<S: StateStore> DistinctDeduplicater<S> {
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::array::StreamChunk;
     use risingwave_common::catalog::{ColumnDesc, ColumnId, TableId};
     use risingwave_common::test_prelude::StreamChunkTestExt;
-    use risingwave_common::types::DataType;
     use risingwave_common::util::epoch::{test_epoch, EpochPair};
     use risingwave_common::util::sort_util::OrderType;
     use risingwave_storage::memory::MemoryStateStore;
 
     use super::*;
-    use crate::executor::ActorContext;
 
     async fn infer_dedup_tables<S: StateStore>(
         agg_calls: &[AggCall],
@@ -392,7 +381,7 @@ mod tests {
             &agg_calls,
             Arc::new(AtomicU64::new(0)),
             &dedup_tables,
-            ActorContext::for_test(0),
+            &ActorContext::for_test(0),
         );
 
         // --- chunk 1 ---
@@ -483,7 +472,7 @@ mod tests {
             &agg_calls,
             Arc::new(AtomicU64::new(0)),
             &dedup_tables,
-            ActorContext::for_test(0),
+            &ActorContext::for_test(0),
         );
 
         // --- chunk 3 ---
@@ -572,7 +561,7 @@ mod tests {
             &agg_calls,
             Arc::new(AtomicU64::new(0)),
             &dedup_tables,
-            ActorContext::for_test(0),
+            &ActorContext::for_test(0),
         );
 
         let chunk = StreamChunk::from_pretty(
diff --git a/src/stream/src/executor/aggregation/minput.rs b/src/stream/src/executor/aggregation/minput.rs
index 55a3da1db493a..c8bae26f6e1b3 100644
--- a/src/stream/src/executor/aggregation/minput.rs
+++ b/src/stream/src/executor/aggregation/minput.rs
@@ -224,7 +224,7 @@ impl MaterializedInputState {
         } else {
             const CHUNK_SIZE: usize = 1024;
             let chunks = self.cache.output_batches(CHUNK_SIZE).collect_vec();
-            let mut state = func.create_state();
+            let mut state = func.create_state()?;
             for chunk in chunks {
                 func.update(&mut state, &chunk).await?;
             }
diff --git a/src/stream/src/executor/backfill/arrangement_backfill.rs b/src/stream/src/executor/backfill/arrangement_backfill.rs
index f43d7767b45ce..7920e8dceee80 100644
--- a/src/stream/src/executor/backfill/arrangement_backfill.rs
+++ b/src/stream/src/executor/backfill/arrangement_backfill.rs
@@ -99,7 +99,7 @@ where
 
     #[try_stream(ok = Message, error = StreamExecutorError)]
     async fn execute_inner(mut self) {
-        tracing::debug!("Arrangement Backfill Executor started");
+        tracing::debug!("backfill executor started");
         // The primary key columns, in the output columns of the upstream_table scan.
         // Table scan scans a subset of the columns of the upstream table.
         let pk_in_output_indices = self.upstream_table.pk_in_output_indices().unwrap();
@@ -207,21 +207,9 @@ where
 
             let mut rate_limiter = rate_limit.and_then(create_limiter);
 
-            let backfill_snapshot_read_row_count_metric = self
+            let metrics = self
                 .metrics
-                .backfill_snapshot_read_row_count
-                .with_guarded_label_values(&[
-                    upstream_table_id.to_string().as_str(),
-                    self.actor_id.to_string().as_str(),
-                ]);
-
-            let backfill_upstream_output_row_count_metric = self
-                .metrics
-                .backfill_upstream_output_row_count
-                .with_guarded_label_values(&[
-                    upstream_table_id.to_string().as_str(),
-                    self.actor_id.to_string().as_str(),
-                ]);
+                .new_backfill_metrics(upstream_table_id, self.actor_id);
 
             'backfill_loop: loop {
                 let mut cur_barrier_snapshot_processed_rows: u64 = 0;
@@ -313,9 +301,11 @@ where
                                                 &self.output_indices,
                                             ));
                                         }
-                                        backfill_snapshot_read_row_count_metric
+                                        metrics
+                                            .backfill_snapshot_read_row_count
                                             .inc_by(cur_barrier_snapshot_processed_rows);
-                                        backfill_upstream_output_row_count_metric
+                                        metrics
+                                            .backfill_upstream_output_row_count
                                             .inc_by(cur_barrier_upstream_processed_rows);
                                         break 'backfill_loop;
                                     }
@@ -341,9 +331,17 @@ where
                     // Before processing barrier, if did not snapshot read,
                     // do a snapshot read first.
                     // This is so we don't lose the tombstone iteration progress.
-                    // If paused, we also can't read any snapshot records.
-                    if !has_snapshot_read && !paused {
-                        // If we have not snapshot read, builders must all be empty.
+                    // Or if s3 read latency is high, we don't fail to read from s3.
+                    //
+                    // If paused, we can't read any snapshot records, skip this.
+                    //
+                    // If rate limit is set, respect the rate limit, check if we can read,
+                    // If we can't, skip it. If no rate limit set, we can read.
+                    let rate_limit_ready = rate_limiter
+                        .as_ref()
+                        .map(|r| r.check().is_ok())
+                        .unwrap_or(true);
+                    if !has_snapshot_read && !paused && rate_limit_ready {
                         debug_assert!(builders.values().all(|b| b.is_empty()));
                         let (_, snapshot) = backfill_stream.into_inner();
                         #[for_await]
@@ -437,6 +435,7 @@ where
                                 &chunk,
                                 &backfill_state,
                                 &pk_in_output_indices,
+                                &upstream_table,
                                 &pk_order,
                             )?,
                             &self.output_indices,
@@ -449,8 +448,11 @@ where
 
                 upstream_table.commit(barrier.epoch).await?;
 
-                backfill_snapshot_read_row_count_metric.inc_by(cur_barrier_snapshot_processed_rows);
-                backfill_upstream_output_row_count_metric
+                metrics
+                    .backfill_snapshot_read_row_count
+                    .inc_by(cur_barrier_snapshot_processed_rows);
+                metrics
+                    .backfill_upstream_output_row_count
                     .inc_by(cur_barrier_upstream_processed_rows);
 
                 // Update snapshot read epoch.
@@ -477,7 +479,6 @@ where
                 .await?;
 
                 tracing::trace!(
-                    actor = self.actor_id,
                     barrier = ?barrier,
                     "barrier persisted"
                 );
@@ -499,9 +500,8 @@ where
                                 if new_rate_limit != rate_limit {
                                     rate_limit = new_rate_limit;
                                     tracing::info!(
-                                        id = self.actor_id,
                                         new_rate_limit = ?rate_limit,
-                                        "actor rate limit changed",
+                                        "rate limit changed",
                                     );
                                     // The builder is emptied above via `DataChunkBuilder::consume_all`.
                                     for (_, builder) in builders {
@@ -540,10 +540,7 @@ where
             }
         }
 
-        tracing::trace!(
-            actor = self.actor_id,
-            "Arrangement Backfill has finished and forward messages directly to the downstream"
-        );
+        tracing::debug!("snapshot read finished, wait to commit state on next barrier");
 
         // Update our progress as finished in state table.
 
@@ -551,15 +548,11 @@ where
         // So we can update our progress + persist the status.
         while let Some(Ok(msg)) = upstream.next().await {
             if let Some(msg) = mapping_message(msg, &self.output_indices) {
-                tracing::trace!(
-                    actor = self.actor_id,
-                    message = ?msg,
-                    "backfill_finished_wait_for_barrier"
-                );
                 // If not finished then we need to update state, otherwise no need.
                 if let Message::Barrier(barrier) = &msg {
                     if is_completely_finished {
-                        // If already finished, no need to persist any state.
+                        // If already finished, no need to persist any state. But we need to advance the epoch anyway
+                        self.state_table.commit(barrier.epoch).await?;
                     } else {
                         // If snapshot was empty, we do not need to backfill,
                         // but we still need to persist the finished state.
@@ -585,10 +578,6 @@ where
 
                     self.progress
                         .finish(barrier.epoch.curr, total_snapshot_processed_rows);
-                    tracing::trace!(
-                        epoch = ?barrier.epoch,
-                        "Updated CreateMaterializedTracker"
-                    );
                     yield msg;
                     break;
                 }
@@ -599,9 +588,7 @@ where
             }
         }
 
-        tracing::trace!(
-            "Arrangement Backfill has already finished and forward messages directly to the downstream"
-        );
+        tracing::debug!("backfill finished");
 
         // After progress finished + state persisted,
         // we can forward messages directly to the downstream,
@@ -609,6 +596,10 @@ where
         #[for_await]
         for msg in upstream {
             if let Some(msg) = mapping_message(msg?, &self.output_indices) {
+                if let Message::Barrier(barrier) = &msg {
+                    // If already finished, no need persist any state, but we need to advance the epoch of the state table anyway.
+                    self.state_table.commit(barrier.epoch).await?;
+                }
                 yield msg;
             }
         }
diff --git a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs
index dc8e6bc67ddaa..168e619cc956a 100644
--- a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs
+++ b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs
@@ -20,12 +20,12 @@ use futures::stream::select_with_strategy;
 use itertools::Itertools;
 use risingwave_common::array::DataChunk;
 use risingwave_common::bail;
-use risingwave_common::catalog::{ColumnDesc, ColumnId};
+use risingwave_common::catalog::ColumnDesc;
 use risingwave_common::row::RowExt;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_connector::parser::{
-    DebeziumParser, DebeziumProps, EncodingProperties, JsonProperties, ProtocolProperties,
-    SourceStreamChunkBuilder, SpecificParserConfig,
+    ByteStreamSourceParser, DebeziumParser, DebeziumProps, EncodingProperties, JsonProperties,
+    ProtocolProperties, SourceStreamChunkBuilder, SpecificParserConfig,
 };
 use risingwave_connector::source::cdc::external::CdcOffset;
 use risingwave_connector::source::{SourceColumnDesc, SourceContext};
@@ -39,6 +39,8 @@ use crate::executor::backfill::cdc::upstream_table::snapshot::{
 use crate::executor::backfill::utils::{
     get_cdc_chunk_last_offset, get_new_pos, mapping_chunk, mapping_message, mark_cdc_chunk,
 };
+use crate::executor::backfill::CdcScanOptions;
+use crate::executor::monitor::CdcBackfillMetrics;
 use crate::executor::prelude::*;
 use crate::task::CreateMviewProgress;
 
@@ -55,19 +57,24 @@ pub struct CdcBackfillExecutor<S: StateStore> {
     upstream: Executor,
 
     /// The column indices need to be forwarded to the downstream from the upstream and table scan.
-    /// User may select a subset of columns from the upstream table.
     output_indices: Vec<usize>,
 
+    /// The schema of output chunk, including additional columns if any
+    output_columns: Vec<ColumnDesc>,
+
     /// State table of the `CdcBackfill` executor
-    state_table: StateTable<S>,
+    state_impl: CdcBackfillState<S>,
 
+    // TODO: introduce a CdcBackfillProgress to report finish to Meta
+    // This object is just a stub right now
     progress: Option<CreateMviewProgress>,
 
-    metrics: Arc<StreamingMetrics>,
+    metrics: CdcBackfillMetrics,
 
-    chunk_size: usize,
+    /// Rate limit in rows/s.
+    rate_limit_rps: Option<u32>,
 
-    disable_backfill: bool,
+    options: CdcScanOptions,
 }
 
 impl<S: StateStore> CdcBackfillExecutor<S> {
@@ -77,36 +84,70 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
         external_table: ExternalStorageTable,
         upstream: Executor,
         output_indices: Vec<usize>,
+        output_columns: Vec<ColumnDesc>,
         progress: Option<CreateMviewProgress>,
         metrics: Arc<StreamingMetrics>,
         state_table: StateTable<S>,
-        chunk_size: usize,
-        disable_backfill: bool,
+        rate_limit_rps: Option<u32>,
+        options: CdcScanOptions,
     ) -> Self {
+        let pk_indices = external_table.pk_indices();
+        let upstream_table_id = external_table.table_id().table_id;
+        let state_impl = CdcBackfillState::new(
+            upstream_table_id,
+            state_table,
+            pk_indices.len() + METADATA_STATE_LEN,
+        );
+
+        let metrics = metrics.new_cdc_backfill_metrics(external_table.table_id(), actor_ctx.id);
+
         Self {
             actor_ctx,
             external_table,
             upstream,
             output_indices,
-            state_table,
+            output_columns,
+            state_impl,
             progress,
             metrics,
-            chunk_size,
-            disable_backfill,
+            rate_limit_rps,
+            options,
         }
     }
 
+    fn report_metrics(
+        metrics: &CdcBackfillMetrics,
+        snapshot_processed_row_count: u64,
+        upstream_processed_row_count: u64,
+    ) {
+        metrics
+            .cdc_backfill_snapshot_read_row_count
+            .inc_by(snapshot_processed_row_count);
+
+        metrics
+            .cdc_backfill_upstream_output_row_count
+            .inc_by(upstream_processed_row_count);
+    }
+
     #[try_stream(ok = Message, error = StreamExecutorError)]
     async fn execute_inner(mut self) {
-        // The primary key columns, in the output columns of the upstream_table scan.
-        let pk_in_output_indices = self.external_table.pk_in_output_indices().unwrap();
+        // The indices to primary key columns
+        let pk_indices = self.external_table.pk_indices().to_vec();
         let pk_order = self.external_table.pk_order_types().to_vec();
 
         let upstream_table_id = self.external_table.table_id().table_id;
         let upstream_table_name = self.external_table.qualified_table_name();
-        let upstream_table_schema = self.external_table.schema().clone();
+        let schema_table_name = self.external_table.schema_table_name().clone();
+        let external_database_name = self.external_table.database_name().to_owned();
         let upstream_table_reader = UpstreamTableReader::new(self.external_table);
 
+        let additional_columns = self
+            .output_columns
+            .iter()
+            .filter(|col| col.additional_column.column_type.is_some())
+            .cloned()
+            .collect_vec();
+
         let mut upstream = self.upstream.execute();
 
         // Current position of the upstream_table storage primary key.
@@ -120,13 +161,9 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
 
         // Check whether this parallelism has been assigned splits,
         // if not, we should bypass the backfill directly.
-        let mut state_impl = CdcBackfillState::new(
-            upstream_table_id,
-            self.state_table,
-            pk_in_output_indices.len() + METADATA_STATE_LEN,
-        );
+        let mut state_impl = self.state_impl;
 
-        let mut upstream = transform_upstream(upstream, &upstream_table_schema)
+        let mut upstream = transform_upstream(upstream, &self.output_columns)
             .boxed()
             .peekable();
 
@@ -136,18 +173,17 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
         let state = state_impl.restore_state().await?;
         current_pk_pos = state.current_pk_pos.clone();
 
-        let to_backfill = !self.disable_backfill && !state.is_finished;
+        let to_backfill = !self.options.disable_backfill && !state.is_finished;
 
         // The first barrier message should be propagated.
         yield Message::Barrier(first_barrier);
 
         // Keep track of rows from the snapshot.
         let mut total_snapshot_row_count = state.row_count as u64;
-        let mut snapshot_read_epoch;
 
         let mut last_binlog_offset: Option<CdcOffset> = state
             .last_cdc_offset
-            .map_or(upstream_table_reader.current_binlog_offset().await?, Some);
+            .map_or(upstream_table_reader.current_cdc_offset().await?, Some);
 
         let offset_parse_func = upstream_table_reader
             .inner()
@@ -161,10 +197,12 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
             initial_binlog_offset = ?last_binlog_offset,
             ?current_pk_pos,
             is_finished = state.is_finished,
-            disable_backfill = self.disable_backfill,
             snapshot_row_count = total_snapshot_row_count,
-            chunk_size = self.chunk_size,
-            "start cdc backfill"
+            rate_limit = self.rate_limit_rps,
+            disable_backfill = self.options.disable_backfill,
+            snapshot_interval = self.options.snapshot_interval,
+            snapshot_batch_size = self.options.snapshot_batch_size,
+            "start cdc backfill",
         );
 
         // CDC Backfill Algorithm:
@@ -216,19 +254,24 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                 ?current_pk_pos,
                 "start cdc backfill loop");
 
-            // TODO: make the limit configurable
-            let snapshot_read_limit: usize = 1000;
-
             // the buffer will be drained when a barrier comes
             let mut upstream_chunk_buffer: Vec<StreamChunk> = vec![];
+
             'backfill_loop: loop {
                 let left_upstream = upstream.by_ref().map(Either::Left);
 
                 let mut snapshot_read_row_cnt: usize = 0;
-                let args = SnapshotReadArgs::new_for_cdc(current_pk_pos.clone(), self.chunk_size);
+                let read_args = SnapshotReadArgs::new(
+                    current_pk_pos.clone(),
+                    self.rate_limit_rps,
+                    pk_indices.clone(),
+                    additional_columns.clone(),
+                    schema_table_name.clone(),
+                    external_database_name.clone(),
+                );
 
                 let right_snapshot = pin!(upstream_table_reader
-                    .snapshot_read(args, snapshot_read_limit as u32)
+                    .snapshot_read_full_table(read_args, self.options.snapshot_batch_size)
                     .map(Either::Right));
 
                 let (right_snapshot, valve) = pausable(right_snapshot);
@@ -244,6 +287,8 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
 
                 let mut cur_barrier_snapshot_processed_rows: u64 = 0;
                 let mut cur_barrier_upstream_processed_rows: u64 = 0;
+                let mut barrier_count: u32 = 0;
+                let mut pending_barrier = None;
 
                 #[for_await]
                 for either in &mut backfill_stream {
@@ -252,6 +297,11 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                         Either::Left(msg) => {
                             match msg? {
                                 Message::Barrier(barrier) => {
+                                    // increase the barrier count and check whether need to start a new snapshot
+                                    barrier_count += 1;
+                                    let can_start_new_snapshot =
+                                        barrier_count == self.options.snapshot_interval;
+
                                     if let Some(mutation) = barrier.mutation.as_deref() {
                                         use crate::executor::Mutation;
                                         match mutation {
@@ -264,13 +314,12 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                                                 valve.resume();
                                             }
                                             Mutation::Throttle(some) => {
-                                                if let Some(rate_limit) =
+                                                if let Some(new_rate_limit) =
                                                     some.get(&self.actor_ctx.id)
+                                                    && *new_rate_limit != self.rate_limit_rps
                                                 {
-                                                    self.chunk_size = rate_limit
-                                                        .map(|x| x as usize)
-                                                        .unwrap_or(self.chunk_size);
-                                                    // rebuild the new reader stream with new chunk size
+                                                    self.rate_limit_rps = *new_rate_limit;
+                                                    // rebuild the new reader stream with new rate limit
                                                     continue 'backfill_loop;
                                                 }
                                             }
@@ -278,157 +327,41 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                                         }
                                     }
 
-                                    // ensure the snapshot stream is consumed once
-                                    // otherwise, reconstruct a snapshot stream with same pk offset may return an
-                                    // empty stream (don't know the cause)
-                                    if snapshot_read_row_cnt == 0 {
-                                        pin_mut!(backfill_stream);
-                                        let (_, mut snapshot_stream) =
-                                            backfill_stream.get_pin_mut();
-                                        if let Some(msg) = snapshot_stream.next().await {
-                                            let Either::Right(msg) = msg else {
-                                                bail!(
-                                                    "BUG: snapshot_read contains upstream messages"
-                                                );
-                                            };
-                                            match msg? {
-                                                None => {
-                                                    tracing::info!(
-                                                        upstream_table_id,
-                                                        ?last_binlog_offset,
-                                                        ?current_pk_pos,
-                                                        "snapshot read stream ends"
-                                                    );
-                                                    // End of the snapshot read stream.
-                                                    // Consume the buffered upstream chunk without filtering by `binlog_low`.
-                                                    for chunk in upstream_chunk_buffer.drain(..) {
-                                                        yield Message::Chunk(mapping_chunk(
-                                                            chunk,
-                                                            &self.output_indices,
-                                                        ));
-                                                    }
-
-                                                    // mark backfill has finished
-                                                    state_impl
-                                                        .mutate_state(
-                                                            current_pk_pos,
-                                                            last_binlog_offset.clone(),
-                                                            total_snapshot_row_count,
-                                                            true,
-                                                        )
-                                                        .await?;
-
-                                                    // commit state because we have received a barrier message
-                                                    state_impl.commit_state(barrier.epoch).await?;
-                                                    yield Message::Barrier(barrier);
-                                                    // end of backfill loop, since backfill has finished
-                                                    break 'backfill_loop;
-                                                }
-                                                Some(chunk) => {
-                                                    // Raise the current pk position.
-                                                    // Because snapshot streams are ordered by pk, so we can
-                                                    // use the last row to update `current_pk_pos`.
-                                                    current_pk_pos = Some(get_new_pos(
-                                                        &chunk,
-                                                        &pk_in_output_indices,
-                                                    ));
-
-                                                    let chunk_cardinality =
-                                                        chunk.cardinality() as u64;
-                                                    cur_barrier_snapshot_processed_rows +=
-                                                        chunk_cardinality;
-                                                    total_snapshot_row_count += chunk_cardinality;
-                                                    snapshot_read_row_cnt +=
-                                                        chunk_cardinality as usize;
-
-                                                    tracing::debug!(
-                                                        upstream_table_id,
-                                                        ?current_pk_pos,
-                                                        ?snapshot_read_row_cnt,
-                                                        "force emit a snapshot chunk"
-                                                    );
-                                                    yield Message::Chunk(mapping_chunk(
-                                                        chunk,
-                                                        &self.output_indices,
-                                                    ));
-                                                }
-                                            }
-                                        }
-                                    }
+                                    Self::report_metrics(
+                                        &self.metrics,
+                                        cur_barrier_snapshot_processed_rows,
+                                        cur_barrier_upstream_processed_rows,
+                                    );
 
-                                    // If it is a barrier, switch snapshot and consume buffered
-                                    // upstream chunk.
-                                    // If no current_pos, means we did not process any snapshot yet.
-                                    // In that case we can just ignore the upstream buffer chunk.
-                                    if let Some(current_pos) = &current_pk_pos {
-                                        for chunk in upstream_chunk_buffer.drain(..) {
-                                            cur_barrier_upstream_processed_rows +=
-                                                chunk.cardinality() as u64;
-
-                                            // record the consumed binlog offset that will be
-                                            // persisted later
-                                            consumed_binlog_offset = get_cdc_chunk_last_offset(
-                                                &offset_parse_func,
-                                                &chunk,
-                                            )?;
-                                            yield Message::Chunk(mapping_chunk(
-                                                mark_cdc_chunk(
-                                                    &offset_parse_func,
-                                                    chunk,
-                                                    current_pos,
-                                                    &pk_in_output_indices,
-                                                    &pk_order,
-                                                    last_binlog_offset.clone(),
-                                                )?,
-                                                &self.output_indices,
-                                            ));
-                                        }
-                                    }
+                                    // when processing a barrier, check whether can start a new snapshot
+                                    // if the number of barriers reaches the snapshot interval
+                                    if can_start_new_snapshot {
+                                        // staging the barrier
+                                        pending_barrier = Some(barrier);
+                                        tracing::debug!(
+                                            upstream_table_id,
+                                            ?current_pk_pos,
+                                            ?snapshot_read_row_cnt,
+                                            "Prepare to start a new snapshot"
+                                        );
+                                        // Break the loop for consuming snapshot and prepare to start a new snapshot
+                                        break;
+                                    } else {
+                                        // update and persist current backfill progress
+                                        state_impl
+                                            .mutate_state(
+                                                current_pk_pos.clone(),
+                                                last_binlog_offset.clone(),
+                                                total_snapshot_row_count,
+                                                false,
+                                            )
+                                            .await?;
 
-                                    self.metrics
-                                        .cdc_backfill_snapshot_read_row_count
-                                        .with_label_values(&[
-                                            upstream_table_id.to_string().as_str(),
-                                            self.actor_ctx.id.to_string().as_str(),
-                                        ])
-                                        .inc_by(cur_barrier_snapshot_processed_rows);
-
-                                    self.metrics
-                                        .cdc_backfill_upstream_output_row_count
-                                        .with_label_values(&[
-                                            upstream_table_id.to_string().as_str(),
-                                            self.actor_ctx.id.to_string().as_str(),
-                                        ])
-                                        .inc_by(cur_barrier_upstream_processed_rows);
-
-                                    // Update last seen binlog offset
-                                    if consumed_binlog_offset.is_some() {
-                                        last_binlog_offset.clone_from(&consumed_binlog_offset);
-                                    }
+                                        state_impl.commit_state(barrier.epoch).await?;
 
-                                    // update and persist backfill state
-                                    state_impl
-                                        .mutate_state(
-                                            current_pk_pos.clone(),
-                                            last_binlog_offset.clone(),
-                                            total_snapshot_row_count,
-                                            false,
-                                        )
-                                        .await?;
-                                    state_impl.commit_state(barrier.epoch).await?;
-
-                                    snapshot_read_epoch = barrier.epoch.prev;
-                                    if let Some(progress) = self.progress.as_mut() {
-                                        progress.update(
-                                            barrier.epoch.curr,
-                                            snapshot_read_epoch,
-                                            total_snapshot_row_count,
-                                        );
+                                        // emit barrier and continue consume the backfill stream
+                                        yield Message::Barrier(barrier);
                                     }
-
-                                    yield Message::Barrier(barrier);
-                                    // Break the for loop and start a new snapshot read stream.
-                                    break;
                                 }
                                 Message::Chunk(chunk) => {
                                     // skip empty upstream chunk
@@ -440,7 +373,6 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                                         get_cdc_chunk_last_offset(&offset_parse_func, &chunk)?;
 
                                     tracing::trace!(
-                                        target: "events::stream::cdc_backfill",
                                         "recv changelog chunk: chunk_offset {:?}, capactiy {}",
                                         chunk_binlog_offset,
                                         chunk.capacity()
@@ -454,7 +386,6 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                                             && chunk_offset < *last_binlog_offset
                                         {
                                             tracing::trace!(
-                                                target: "events::stream::cdc_backfill",
                                                 "skip changelog chunk: chunk_offset {:?}, capacity {}",
                                                 chunk_offset,
                                                 chunk.capacity()
@@ -474,49 +405,31 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                         Either::Right(msg) => {
                             match msg? {
                                 None => {
-                                    if snapshot_read_row_cnt < snapshot_read_limit {
-                                        tracing::info!(
-                                            upstream_table_id,
-                                            ?last_binlog_offset,
-                                            ?current_pk_pos,
-                                            "snapshot read stream ends"
-                                        );
-                                        // If the snapshot read stream ends with less than `limit` rows,
-                                        // it means all historical data has been loaded.
-                                        // We should not mark the chunk anymore,
-                                        // otherwise, we will ignore some rows in the buffer.
-                                        // Here we choose to never mark the chunk.
-                                        // Consume with the renaming stream buffer chunk without mark.
-                                        for chunk in upstream_chunk_buffer.drain(..) {
-                                            yield Message::Chunk(mapping_chunk(
-                                                chunk,
-                                                &self.output_indices,
-                                            ));
-                                        }
-
-                                        state_impl
-                                            .mutate_state(
-                                                current_pk_pos.clone(),
-                                                last_binlog_offset.clone(),
-                                                total_snapshot_row_count,
-                                                true,
-                                            )
-                                            .await?;
-
-                                        // exit backfill
-                                        break 'backfill_loop;
-                                    } else {
-                                        // break the for loop to reconstruct a new snapshot with pk offset
-                                        // to ensure we load all historical data
-                                        break;
+                                    tracing::info!(
+                                        upstream_table_id,
+                                        ?last_binlog_offset,
+                                        ?current_pk_pos,
+                                        "snapshot read stream ends"
+                                    );
+                                    // If the snapshot read stream ends, it means all historical
+                                    // data has been loaded.
+                                    // We should not mark the chunk anymore,
+                                    // otherwise, we will ignore some rows in the buffer.
+                                    for chunk in upstream_chunk_buffer.drain(..) {
+                                        yield Message::Chunk(mapping_chunk(
+                                            chunk,
+                                            &self.output_indices,
+                                        ));
                                     }
+
+                                    // backfill has finished, exit the backfill loop and persist the state when we recv a barrier
+                                    break 'backfill_loop;
                                 }
                                 Some(chunk) => {
                                     // Raise the current position.
                                     // As snapshot read streams are ordered by pk, so we can
                                     // just use the last row to update `current_pos`.
-                                    current_pk_pos =
-                                        Some(get_new_pos(&chunk, &pk_in_output_indices));
+                                    current_pk_pos = Some(get_new_pos(&chunk, &pk_indices));
 
                                     tracing::trace!(
                                         "got a snapshot chunk: len {}, current_pk_pos {:?}",
@@ -526,8 +439,6 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                                     let chunk_cardinality = chunk.cardinality() as u64;
                                     cur_barrier_snapshot_processed_rows += chunk_cardinality;
                                     total_snapshot_row_count += chunk_cardinality;
-                                    // count the number of rows in the snapshot
-                                    snapshot_read_row_cnt += chunk_cardinality as usize;
                                     yield Message::Chunk(mapping_chunk(
                                         chunk,
                                         &self.output_indices,
@@ -537,8 +448,123 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
                         }
                     }
                 }
+
+                assert!(pending_barrier.is_some(), "pending_barrier must exist");
+                let pending_barrier = pending_barrier.unwrap();
+
+                // Here we have to ensure the snapshot stream is consumed at least once,
+                // since the barrier event can kick in anytime.
+                // Otherwise, the result set of the new snapshot stream may become empty.
+                // It maybe a cancellation bug of the mysql driver.
+                let (_, mut snapshot_stream) = backfill_stream.into_inner();
+                if let Some(msg) = snapshot_stream.next().await {
+                    let Either::Right(msg) = msg else {
+                        bail!("BUG: snapshot_read contains upstream messages");
+                    };
+                    match msg? {
+                        None => {
+                            tracing::info!(
+                                upstream_table_id,
+                                ?last_binlog_offset,
+                                ?current_pk_pos,
+                                "snapshot read stream ends in the force emit branch"
+                            );
+                            // End of the snapshot read stream.
+                            // Consume the buffered upstream chunk without filtering by `binlog_low`.
+                            for chunk in upstream_chunk_buffer.drain(..) {
+                                yield Message::Chunk(mapping_chunk(chunk, &self.output_indices));
+                            }
+
+                            // mark backfill has finished
+                            state_impl
+                                .mutate_state(
+                                    current_pk_pos.clone(),
+                                    last_binlog_offset.clone(),
+                                    total_snapshot_row_count,
+                                    true,
+                                )
+                                .await?;
+
+                            // commit state because we have received a barrier message
+                            state_impl.commit_state(pending_barrier.epoch).await?;
+                            yield Message::Barrier(pending_barrier);
+                            // end of backfill loop, since backfill has finished
+                            break 'backfill_loop;
+                        }
+                        Some(chunk) => {
+                            // Raise the current pk position.
+                            current_pk_pos = Some(get_new_pos(&chunk, &pk_indices));
+
+                            let row_count = chunk.cardinality() as u64;
+                            cur_barrier_snapshot_processed_rows += row_count;
+                            total_snapshot_row_count += row_count;
+                            snapshot_read_row_cnt += row_count as usize;
+
+                            tracing::debug!(
+                                upstream_table_id,
+                                ?current_pk_pos,
+                                ?snapshot_read_row_cnt,
+                                "force emit a snapshot chunk"
+                            );
+                            yield Message::Chunk(mapping_chunk(chunk, &self.output_indices));
+                        }
+                    }
+                }
+
+                // If the number of barriers reaches the snapshot interval,
+                // consume the buffered upstream chunks.
+                if let Some(current_pos) = &current_pk_pos {
+                    for chunk in upstream_chunk_buffer.drain(..) {
+                        cur_barrier_upstream_processed_rows += chunk.cardinality() as u64;
+
+                        // record the consumed binlog offset that will be
+                        // persisted later
+                        consumed_binlog_offset =
+                            get_cdc_chunk_last_offset(&offset_parse_func, &chunk)?;
+
+                        yield Message::Chunk(mapping_chunk(
+                            mark_cdc_chunk(
+                                &offset_parse_func,
+                                chunk,
+                                current_pos,
+                                &pk_indices,
+                                &pk_order,
+                                last_binlog_offset.clone(),
+                            )?,
+                            &self.output_indices,
+                        ));
+                    }
+                } else {
+                    // If no current_pos, means we did not process any snapshot yet.
+                    // we can just ignore the upstream buffer chunk in that case.
+                    upstream_chunk_buffer.clear();
+                }
+
+                // Update last seen binlog offset
+                if consumed_binlog_offset.is_some() {
+                    last_binlog_offset.clone_from(&consumed_binlog_offset);
+                }
+
+                Self::report_metrics(
+                    &self.metrics,
+                    cur_barrier_snapshot_processed_rows,
+                    cur_barrier_upstream_processed_rows,
+                );
+
+                // update and persist current backfill progress
+                state_impl
+                    .mutate_state(
+                        current_pk_pos.clone(),
+                        last_binlog_offset.clone(),
+                        total_snapshot_row_count,
+                        false,
+                    )
+                    .await?;
+
+                state_impl.commit_state(pending_barrier.epoch).await?;
+                yield Message::Barrier(pending_barrier);
             }
-        } else if self.disable_backfill {
+        } else if self.options.disable_backfill {
             // If backfill is disabled, we just mark the backfill as finished
             tracing::info!(
                 upstream_table_id,
@@ -547,7 +573,7 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
             );
             state_impl
                 .mutate_state(
-                    current_pk_pos,
+                    current_pk_pos.clone(),
                     last_binlog_offset.clone(),
                     total_snapshot_row_count,
                     true,
@@ -570,7 +596,16 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
             if let Some(msg) = mapping_message(msg, &self.output_indices) {
                 // If not finished then we need to update state, otherwise no need.
                 if let Message::Barrier(barrier) = &msg {
-                    // persist the backfill state
+                    // finalized the backfill state
+                    // TODO: unify `mutate_state` and `commit_state` into one method
+                    state_impl
+                        .mutate_state(
+                            current_pk_pos.clone(),
+                            last_binlog_offset.clone(),
+                            total_snapshot_row_count,
+                            true,
+                        )
+                        .await?;
                     state_impl.commit_state(barrier.epoch).await?;
 
                     // mark progress as finished
@@ -604,7 +639,7 @@ impl<S: StateStore> CdcBackfillExecutor<S> {
 }
 
 #[try_stream(ok = Message, error = StreamExecutorError)]
-pub async fn transform_upstream(upstream: BoxedMessageStream, schema: &Schema) {
+pub async fn transform_upstream(upstream: BoxedMessageStream, output_columns: &[ColumnDesc]) {
     let props = SpecificParserConfig {
         key_encoding_config: None,
         encoding_config: EncodingProperties::Json(JsonProperties {
@@ -614,9 +649,16 @@ pub async fn transform_upstream(upstream: BoxedMessageStream, schema: &Schema) {
         // the cdc message is generated internally so the key must exist.
         protocol_config: ProtocolProperties::Debezium(DebeziumProps::default()),
     };
+
+    // convert to source column desc to feed into parser
+    let columns_with_meta = output_columns
+        .iter()
+        .map(SourceColumnDesc::from)
+        .collect_vec();
+
     let mut parser = DebeziumParser::new(
         props,
-        get_rw_columns(schema),
+        columns_with_meta.clone(),
         Arc::new(SourceContext::dummy()),
     )
     .await
@@ -627,7 +669,7 @@ pub async fn transform_upstream(upstream: BoxedMessageStream, schema: &Schema) {
     for msg in upstream {
         let mut msg = msg?;
         if let Message::Chunk(chunk) = &mut msg {
-            let parsed_chunk = parse_debezium_chunk(&mut parser, chunk, schema).await?;
+            let parsed_chunk = parse_debezium_chunk(&mut parser, chunk).await?;
             let _ = std::mem::replace(chunk, parsed_chunk);
         }
         yield msg;
@@ -637,14 +679,13 @@ pub async fn transform_upstream(upstream: BoxedMessageStream, schema: &Schema) {
 async fn parse_debezium_chunk(
     parser: &mut DebeziumParser,
     chunk: &StreamChunk,
-    schema: &Schema,
 ) -> StreamExecutorResult<StreamChunk> {
     // here we transform the input chunk in (payload varchar, _rw_offset varchar, _rw_table_name varchar) schema
     // to chunk with downstream table schema `info.schema` of MergeNode contains the schema of the
     // table job with `_rw_offset` in the end
     // see `gen_create_table_plan_for_cdc_source` for details
-    let column_descs = get_rw_columns(schema);
-    let mut builder = SourceStreamChunkBuilder::with_capacity(column_descs, chunk.capacity());
+    let mut builder =
+        SourceStreamChunkBuilder::with_capacity(parser.columns().to_vec(), chunk.capacity());
 
     // The schema of input chunk (payload varchar, _rw_offset varchar, _rw_table_name varchar, _row_id)
     // We should use the debezium parser to parse the first column,
@@ -683,10 +724,10 @@ async fn parse_debezium_chunk(
         new_rows.push(combined);
     }
 
-    let data_types = schema
-        .fields
+    let data_types = parser
+        .columns()
         .iter()
-        .map(|field| field.data_type.clone())
+        .map(|col| col.data_type.clone())
         .chain(std::iter::once(DataType::Varchar)) // _rw_offset column
         .collect_vec();
 
@@ -696,21 +737,6 @@ async fn parse_debezium_chunk(
     ))
 }
 
-fn get_rw_columns(schema: &Schema) -> Vec<SourceColumnDesc> {
-    schema
-        .fields
-        .iter()
-        .map(|field| {
-            let column_desc = ColumnDesc::named(
-                field.name.clone(),
-                ColumnId::placeholder(),
-                field.data_type.clone(),
-            );
-            SourceColumnDesc::from(&column_desc)
-        })
-        .collect_vec()
-}
-
 impl<S: StateStore> Execute for CdcBackfillExecutor<S> {
     fn execute(self: Box<Self>) -> BoxedMessageStream {
         self.execute_inner().boxed()
@@ -723,7 +749,7 @@ mod tests {
 
     use futures::{pin_mut, StreamExt};
     use risingwave_common::array::{DataChunk, Op, StreamChunk};
-    use risingwave_common::catalog::{Field, Schema};
+    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, Schema};
     use risingwave_common::types::{DataType, Datum, JsonbVal};
     use risingwave_common::util::iter_util::ZipEqFast;
 
@@ -766,16 +792,17 @@ mod tests {
         tx.push_chunk(chunk);
         let upstream = Box::new(source).execute();
 
-        // schema of the CDC table
-        let rw_schema = Schema::new(vec![
-            Field::with_name(DataType::Int64, "O_ORDERKEY"), // orderkey
-            Field::with_name(DataType::Int64, "O_CUSTKEY"),  // custkey
-            Field::with_name(DataType::Varchar, "O_ORDERSTATUS"), // orderstatus
-            Field::with_name(DataType::Decimal, "O_TOTALPRICE"), // totalprice
-            Field::with_name(DataType::Date, "O_ORDERDATE"), // orderdate
-        ]);
+        // schema to the debezium parser
+        let columns = vec![
+            ColumnDesc::named("O_ORDERKEY", ColumnId::new(1), DataType::Int64),
+            ColumnDesc::named("O_CUSTKEY", ColumnId::new(2), DataType::Int64),
+            ColumnDesc::named("O_ORDERSTATUS", ColumnId::new(3), DataType::Varchar),
+            ColumnDesc::named("O_TOTALPRICE", ColumnId::new(4), DataType::Decimal),
+            ColumnDesc::named("O_ORDERDATE", ColumnId::new(5), DataType::Date),
+            ColumnDesc::named("commit_ts", ColumnId::new(6), DataType::Timestamptz),
+        ];
 
-        let parsed_stream = transform_upstream(upstream, &rw_schema);
+        let parsed_stream = transform_upstream(upstream, &columns);
         pin_mut!(parsed_stream);
         // the output chunk must contain the offset column
         if let Some(message) = parsed_stream.next().await {
diff --git a/src/stream/src/executor/backfill/cdc/mod.rs b/src/stream/src/executor/backfill/cdc/mod.rs
index e35facc6a5431..3e06ec791503a 100644
--- a/src/stream/src/executor/backfill/cdc/mod.rs
+++ b/src/stream/src/executor/backfill/cdc/mod.rs
@@ -17,4 +17,35 @@ mod state;
 mod upstream_table;
 
 pub use cdc_backfill::CdcBackfillExecutor;
+use risingwave_pb::stream_plan::StreamCdcScanOptions;
 pub use upstream_table::external::ExternalStorageTable;
+
+#[derive(Debug, Clone)]
+pub struct CdcScanOptions {
+    /// Whether to disable backfill
+    pub disable_backfill: bool,
+    /// Barreir interval to start a new snapshot read
+    pub snapshot_interval: u32,
+    /// Batch size for a snapshot read query
+    pub snapshot_batch_size: u32,
+}
+
+impl Default for CdcScanOptions {
+    fn default() -> Self {
+        Self {
+            disable_backfill: false,
+            snapshot_interval: 1,
+            snapshot_batch_size: 1000,
+        }
+    }
+}
+
+impl CdcScanOptions {
+    pub fn from_proto(proto: &StreamCdcScanOptions) -> Self {
+        Self {
+            disable_backfill: proto.disable_backfill,
+            snapshot_interval: proto.snapshot_barrier_interval,
+            snapshot_batch_size: proto.snapshot_batch_size,
+        }
+    }
+}
diff --git a/src/stream/src/executor/backfill/cdc/upstream_table/external.rs b/src/stream/src/executor/backfill/cdc/upstream_table/external.rs
index a09d8e2954d90..bd99eb59821ee 100644
--- a/src/stream/src/executor/backfill/cdc/upstream_table/external.rs
+++ b/src/stream/src/executor/backfill/cdc/upstream_table/external.rs
@@ -26,6 +26,8 @@ pub struct ExternalStorageTable {
 
     schema_name: String,
 
+    database_name: String,
+
     table_reader: ExternalTableReaderImpl,
 
     /// The schema of the output columns, i.e., this table VIEWED BY some executor like
@@ -38,8 +40,6 @@ pub struct ExternalStorageTable {
     /// Indices of primary key.
     /// Note that the index is based on the all columns of the table.
     pk_indices: Vec<usize>,
-
-    output_indices: Vec<usize>,
 }
 
 impl ExternalStorageTable {
@@ -49,21 +49,21 @@ impl ExternalStorageTable {
             table_name,
             schema_name,
         }: SchemaTableName,
+        database_name: String,
         table_reader: ExternalTableReaderImpl,
         schema: Schema,
         pk_order_types: Vec<OrderType>,
         pk_indices: Vec<usize>,
-        output_indices: Vec<usize>,
     ) -> Self {
         Self {
             table_id,
             table_name,
             schema_name,
+            database_name,
             table_reader,
             schema,
             pk_order_types,
             pk_indices,
-            output_indices,
         }
     }
 
@@ -83,16 +83,6 @@ impl ExternalStorageTable {
         &self.pk_indices
     }
 
-    /// Get the indices of the primary key columns in the output columns.
-    ///
-    /// Returns `None` if any of the primary key columns is not in the output columns.
-    pub fn pk_in_output_indices(&self) -> Option<Vec<usize>> {
-        self.pk_indices
-            .iter()
-            .map(|&i| self.output_indices.iter().position(|&j| i == j))
-            .collect()
-    }
-
     pub fn schema_table_name(&self) -> SchemaTableName {
         SchemaTableName {
             schema_name: self.schema_name.clone(),
@@ -107,4 +97,8 @@ impl ExternalStorageTable {
     pub fn qualified_table_name(&self) -> String {
         format!("{}.{}", self.schema_name, self.table_name)
     }
+
+    pub fn database_name(&self) -> &str {
+        self.database_name.as_str()
+    }
 }
diff --git a/src/stream/src/executor/backfill/cdc/upstream_table/snapshot.rs b/src/stream/src/executor/backfill/cdc/upstream_table/snapshot.rs
index be98a6822035e..85c2f6b2ab178 100644
--- a/src/stream/src/executor/backfill/cdc/upstream_table/snapshot.rs
+++ b/src/stream/src/executor/backfill/cdc/upstream_table/snapshot.rs
@@ -21,41 +21,58 @@ use governor::clock::MonotonicClock;
 use governor::{Quota, RateLimiter};
 use itertools::Itertools;
 use risingwave_common::array::StreamChunk;
+use risingwave_common::catalog::ColumnDesc;
 use risingwave_common::row::OwnedRow;
+use risingwave_common::types::{Scalar, ScalarImpl, Timestamptz};
 use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
-use risingwave_connector::source::cdc::external::{CdcOffset, ExternalTableReader};
+use risingwave_connector::source::cdc::external::{
+    CdcOffset, ExternalTableReader, SchemaTableName,
+};
+use risingwave_pb::plan_common::additional_column::ColumnType;
 
 use super::external::ExternalStorageTable;
-use crate::executor::backfill::utils::iter_chunks;
-use crate::executor::{StreamExecutorError, StreamExecutorResult, INVALID_EPOCH};
+use crate::common::rate_limit::limited_chunk_size;
+use crate::executor::backfill::utils::{get_new_pos, iter_chunks};
+use crate::executor::{StreamExecutorError, StreamExecutorResult};
 
 pub trait UpstreamTableRead {
-    fn snapshot_read(
+    fn snapshot_read_full_table(
         &self,
         args: SnapshotReadArgs,
-        limit: u32,
+        batch_size: u32,
     ) -> impl Stream<Item = StreamExecutorResult<Option<StreamChunk>>> + Send + '_;
 
-    fn current_binlog_offset(
+    fn current_cdc_offset(
         &self,
     ) -> impl Future<Output = StreamExecutorResult<Option<CdcOffset>>> + Send + '_;
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug, Clone)]
 pub struct SnapshotReadArgs {
-    pub epoch: u64,
     pub current_pos: Option<OwnedRow>,
-    pub ordered: bool,
-    pub chunk_size: usize,
+    pub rate_limit_rps: Option<u32>,
+    pub pk_indices: Vec<usize>,
+    pub additional_columns: Vec<ColumnDesc>,
+    pub schema_table_name: SchemaTableName,
+    pub database_name: String,
 }
 
 impl SnapshotReadArgs {
-    pub fn new_for_cdc(current_pos: Option<OwnedRow>, chunk_size: usize) -> Self {
+    pub fn new(
+        current_pos: Option<OwnedRow>,
+        rate_limit_rps: Option<u32>,
+        pk_indices: Vec<usize>,
+        additional_columns: Vec<ColumnDesc>,
+        schema_table_name: SchemaTableName,
+        database_name: String,
+    ) -> Self {
         Self {
-            epoch: INVALID_EPOCH,
             current_pos,
-            ordered: false,
-            chunk_size,
+            rate_limit_rps,
+            pk_indices,
+            additional_columns,
+            schema_table_name,
+            database_name,
         }
     }
 }
@@ -77,9 +94,53 @@ impl<T> UpstreamTableReader<T> {
     }
 }
 
+/// Append additional columns with value as null to the snapshot chunk
+fn with_additional_columns(
+    snapshot_chunk: StreamChunk,
+    additional_columns: &[ColumnDesc],
+    schema_table_name: SchemaTableName,
+    database_name: String,
+) -> StreamChunk {
+    let (ops, mut columns, visibility) = snapshot_chunk.into_inner();
+    for desc in additional_columns {
+        let mut builder = desc.data_type.create_array_builder(visibility.len());
+        match *desc.additional_column.column_type.as_ref().unwrap() {
+            // set default value for timestamp
+            ColumnType::Timestamp(_) => builder.append_n(
+                visibility.len(),
+                Some(Timestamptz::default().to_scalar_value()),
+            ),
+            ColumnType::DatabaseName(_) => {
+                builder.append_n(
+                    visibility.len(),
+                    Some(ScalarImpl::from(database_name.clone())),
+                );
+            }
+            ColumnType::SchemaName(_) => {
+                builder.append_n(
+                    visibility.len(),
+                    Some(ScalarImpl::from(schema_table_name.schema_name.clone())),
+                );
+            }
+            ColumnType::TableName(_) => {
+                builder.append_n(
+                    visibility.len(),
+                    Some(ScalarImpl::from(schema_table_name.table_name.clone())),
+                );
+            }
+            // set null for other additional columns
+            _ => {
+                builder.append_n_null(visibility.len());
+            }
+        }
+        columns.push(builder.finish().into());
+    }
+    StreamChunk::with_visibility(ops, columns, visibility)
+}
+
 impl UpstreamTableRead for UpstreamTableReader<ExternalStorageTable> {
     #[try_stream(ok = Option<StreamChunk>, error = StreamExecutorError)]
-    async fn snapshot_read(&self, args: SnapshotReadArgs, limit: u32) {
+    async fn snapshot_read_full_table(&self, args: SnapshotReadArgs, batch_size: u32) {
         let primary_keys = self
             .inner
             .pk_indices()
@@ -90,52 +151,180 @@ impl UpstreamTableRead for UpstreamTableReader<ExternalStorageTable> {
             })
             .collect_vec();
 
-        tracing::debug!(
-            "snapshot_read primary keys: {:?}, current_pos: {:?}",
-            primary_keys,
-            args.current_pos
-        );
-
-        let row_stream = self.inner.table_reader().snapshot_read(
-            self.inner.schema_table_name(),
-            args.current_pos,
-            primary_keys,
-            limit,
-        );
-
-        pin_mut!(row_stream);
-
-        let mut builder = DataChunkBuilder::new(self.inner.schema().data_types(), args.chunk_size);
-        let chunk_stream = iter_chunks(row_stream, &mut builder);
-
-        if args.chunk_size == 0 {
+        // prepare rate limiter
+        if args.rate_limit_rps == Some(0) {
             // If limit is 0, we should not read any data from the upstream table.
             // Keep waiting util the stream is rebuilt.
             let future = futures::future::pending::<()>();
             future.await;
+            unreachable!();
         }
-        let limiter = {
-            let quota = Quota::per_second(NonZeroU32::new(args.chunk_size as u32).unwrap());
-            let clock = MonotonicClock;
-            RateLimiter::direct_with_clock(quota, &clock)
-        };
-        #[for_await]
-        for chunk in chunk_stream {
-            let chunk = chunk?;
-            if chunk.cardinality() != 0 {
-                limiter
-                    .until_n_ready(NonZeroU32::new(chunk.cardinality() as u32).unwrap())
-                    .await
-                    .unwrap();
+        let limiter = args.rate_limit_rps.map(|limit| {
+            tracing::info!(rate_limit = limit, "rate limit applied");
+            RateLimiter::direct_with_clock(
+                Quota::per_second(NonZeroU32::new(limit).unwrap()),
+                &MonotonicClock,
+            )
+        });
+
+        let mut read_args = args;
+        let schema_table_name = read_args.schema_table_name.clone();
+        let database_name = read_args.database_name.clone();
+        // loop to read all data from the table
+        loop {
+            tracing::debug!(
+                "snapshot_read primary keys: {:?}, current_pos: {:?}",
+                primary_keys,
+                read_args.current_pos
+            );
+
+            let mut read_count: usize = 0;
+            let row_stream = self.inner.table_reader().snapshot_read(
+                self.inner.schema_table_name(),
+                read_args.current_pos.clone(),
+                primary_keys.clone(),
+                batch_size,
+            );
+
+            pin_mut!(row_stream);
+            let mut builder = DataChunkBuilder::new(
+                self.inner.schema().data_types(),
+                limited_chunk_size(read_args.rate_limit_rps),
+            );
+            let chunk_stream = iter_chunks(row_stream, &mut builder);
+            let mut current_pk_pos = read_args.current_pos.clone().unwrap_or_default();
+
+            #[for_await]
+            for chunk in chunk_stream {
+                let chunk = chunk?;
+                let chunk_size = chunk.capacity();
+                read_count += chunk.cardinality();
+                current_pk_pos = get_new_pos(&chunk, &read_args.pk_indices);
+
+                if read_args.rate_limit_rps.is_none() || chunk_size == 0 {
+                    // no limit, or empty chunk
+                    yield Some(with_additional_columns(
+                        chunk,
+                        &read_args.additional_columns,
+                        schema_table_name.clone(),
+                        database_name.clone(),
+                    ));
+                    continue;
+                } else {
+                    // Apply rate limit, see `risingwave_stream::executor::source::apply_rate_limit` for more.
+                    // May be should be refactored to a common function later.
+                    let limiter = limiter.as_ref().unwrap();
+                    let limit = read_args.rate_limit_rps.unwrap() as usize;
+
+                    // Because we produce chunks with limited-sized data chunk builder and all rows
+                    // are `Insert`s, the chunk size should never exceed the limit.
+                    assert!(chunk_size <= limit);
+
+                    // `InsufficientCapacity` should never happen because we have check the cardinality
+                    limiter
+                        .until_n_ready(NonZeroU32::new(chunk_size as u32).unwrap())
+                        .await
+                        .unwrap();
+                    yield Some(with_additional_columns(
+                        chunk,
+                        &read_args.additional_columns,
+                        schema_table_name.clone(),
+                        database_name.clone(),
+                    ));
+                }
+            }
+
+            // check read_count if the snapshot batch is finished
+            if read_count < batch_size as _ {
+                tracing::debug!("finished loading of full table snapshot");
+                yield None;
+                unreachable!()
+            } else {
+                // update PK position and continue to read the table
+                read_args.current_pos = Some(current_pk_pos);
             }
-            yield Some(chunk);
         }
-        yield None;
     }
 
-    async fn current_binlog_offset(&self) -> StreamExecutorResult<Option<CdcOffset>> {
+    async fn current_cdc_offset(&self) -> StreamExecutorResult<Option<CdcOffset>> {
         let binlog = self.inner.table_reader().current_cdc_offset();
         let binlog = binlog.await?;
         Ok(Some(binlog))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use futures::pin_mut;
+    use futures_async_stream::for_await;
+    use maplit::{convert_args, hashmap};
+    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, Schema};
+    use risingwave_common::row::OwnedRow;
+    use risingwave_common::types::{DataType, ScalarImpl};
+    use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
+    use risingwave_connector::source::cdc::external::mysql::MySqlExternalTableReader;
+    use risingwave_connector::source::cdc::external::{
+        ExternalTableConfig, ExternalTableReader, SchemaTableName,
+    };
+
+    use crate::executor::backfill::utils::{get_new_pos, iter_chunks};
+
+    #[ignore]
+    #[tokio::test]
+    async fn test_mysql_table_reader() {
+        let columns = vec![
+            ColumnDesc::named("o_orderkey", ColumnId::new(1), DataType::Int64),
+            ColumnDesc::named("o_custkey", ColumnId::new(2), DataType::Int64),
+            ColumnDesc::named("o_orderstatus", ColumnId::new(3), DataType::Varchar),
+        ];
+        let rw_schema = Schema {
+            fields: columns.iter().map(Field::from).collect(),
+        };
+        let props: HashMap<String, String> = convert_args!(hashmap!(
+                "hostname" => "localhost",
+                "port" => "8306",
+                "username" => "root",
+                "password" => "123456",
+                "database.name" => "mydb",
+                "table.name" => "orders_rw"));
+
+        let config =
+            serde_json::from_value::<ExternalTableConfig>(serde_json::to_value(props).unwrap())
+                .unwrap();
+        let reader = MySqlExternalTableReader::new(config, rw_schema.clone())
+            .await
+            .unwrap();
+
+        let mut cnt: usize = 0;
+        let mut start_pk = Some(OwnedRow::new(vec![Some(ScalarImpl::Int64(0))]));
+        loop {
+            let row_stream = reader.snapshot_read(
+                SchemaTableName {
+                    schema_name: "mydb".to_string(),
+                    table_name: "orders_rw".to_string(),
+                },
+                start_pk.clone(),
+                vec!["o_orderkey".to_string()],
+                1000,
+            );
+            let mut builder = DataChunkBuilder::new(rw_schema.clone().data_types(), 256);
+            let chunk_stream = iter_chunks(row_stream, &mut builder);
+            let pk_indices = vec![0];
+            pin_mut!(chunk_stream);
+            #[for_await]
+            for chunk in chunk_stream {
+                let chunk = chunk.expect("data");
+                start_pk = Some(get_new_pos(&chunk, &pk_indices));
+                cnt += chunk.capacity();
+                // println!("chunk: {:#?}", chunk);
+                println!("cnt: {}", cnt);
+            }
+            if cnt >= 1499900 {
+                println!("bye!");
+                break;
+            }
+        }
+    }
+}
diff --git a/src/stream/src/executor/backfill/mod.rs b/src/stream/src/executor/backfill/mod.rs
index 99ad412548a8e..48ef5ac7fb6cf 100644
--- a/src/stream/src/executor/backfill/mod.rs
+++ b/src/stream/src/executor/backfill/mod.rs
@@ -16,3 +16,5 @@ pub mod arrangement_backfill;
 pub mod cdc;
 pub mod no_shuffle_backfill;
 pub mod utils;
+
+pub use cdc::CdcScanOptions;
diff --git a/src/stream/src/executor/backfill/no_shuffle_backfill.rs b/src/stream/src/executor/backfill/no_shuffle_backfill.rs
index fdf955fbbdfc7..e368086a97737 100644
--- a/src/stream/src/executor/backfill/no_shuffle_backfill.rs
+++ b/src/stream/src/executor/backfill/no_shuffle_backfill.rs
@@ -209,21 +209,9 @@ where
             let mut pending_barrier: Option<Barrier> = None;
             let mut rate_limiter = rate_limit.and_then(create_limiter);
 
-            let backfill_snapshot_read_row_count_metric = self
+            let metrics = self
                 .metrics
-                .backfill_snapshot_read_row_count
-                .with_guarded_label_values(&[
-                    upstream_table_id.to_string().as_str(),
-                    self.actor_id.to_string().as_str(),
-                ]);
-
-            let backfill_upstream_output_row_count_metric = self
-                .metrics
-                .backfill_upstream_output_row_count
-                .with_guarded_label_values(&[
-                    upstream_table_id.to_string().as_str(),
-                    self.actor_id.to_string().as_str(),
-                ]);
+                .new_backfill_metrics(upstream_table_id, self.actor_id);
 
             'backfill_loop: loop {
                 let mut cur_barrier_snapshot_processed_rows: u64 = 0;
@@ -311,9 +299,11 @@ where
                                                 &self.output_indices,
                                             ));
                                         }
-                                        backfill_snapshot_read_row_count_metric
+                                        metrics
+                                            .backfill_snapshot_read_row_count
                                             .inc_by(cur_barrier_snapshot_processed_rows);
-                                        backfill_upstream_output_row_count_metric
+                                        metrics
+                                            .backfill_upstream_output_row_count
                                             .inc_by(cur_barrier_upstream_processed_rows);
                                         break 'backfill_loop;
                                     }
@@ -416,8 +406,11 @@ where
                     upstream_chunk_buffer.clear()
                 }
 
-                backfill_snapshot_read_row_count_metric.inc_by(cur_barrier_snapshot_processed_rows);
-                backfill_upstream_output_row_count_metric
+                metrics
+                    .backfill_snapshot_read_row_count
+                    .inc_by(cur_barrier_snapshot_processed_rows);
+                metrics
+                    .backfill_upstream_output_row_count
                     .inc_by(cur_barrier_upstream_processed_rows);
 
                 // Update snapshot read epoch.
@@ -505,7 +498,10 @@ where
                 // If not finished then we need to update state, otherwise no need.
                 if let Message::Barrier(barrier) = &msg {
                     if is_finished {
-                        // If already finished, no need persist any state.
+                        // If already finished, no need persist any state, but we need to advance the epoch of the state table anyway.
+                        if let Some(table) = &mut self.state_table {
+                            table.commit(barrier.epoch).await?;
+                        }
                     } else {
                         // If snapshot was empty, we do not need to backfill,
                         // but we still need to persist the finished state.
@@ -571,6 +567,13 @@ where
         #[for_await]
         for msg in upstream {
             if let Some(msg) = mapping_message(msg?, &self.output_indices) {
+                if let Message::Barrier(barrier) = &msg {
+                    // If already finished, no need persist any state, but we need to advance the epoch of the state table anyway.
+                    if let Some(table) = &mut self.state_table {
+                        table.commit(barrier.epoch).await?;
+                    }
+                }
+
                 yield msg;
             }
         }
diff --git a/src/stream/src/executor/backfill/utils.rs b/src/stream/src/executor/backfill/utils.rs
index 1d469b311fd26..f67442e723198 100644
--- a/src/stream/src/executor/backfill/utils.rs
+++ b/src/stream/src/executor/backfill/utils.rs
@@ -41,10 +41,11 @@ use risingwave_common::util::sort_util::{cmp_datum_iter, OrderType};
 use risingwave_common::util::value_encoding::BasicSerde;
 use risingwave_connector::error::ConnectorError;
 use risingwave_connector::source::cdc::external::{CdcOffset, CdcOffsetParseFunc};
+use risingwave_storage::row_serde::value_serde::ValueRowSerde;
 use risingwave_storage::table::{collect_data_chunk_with_builder, KeyedRow};
 use risingwave_storage::StateStore;
 
-use crate::common::table::state_table::StateTableInner;
+use crate::common::table::state_table::{ReplicatedStateTable, StateTableInner};
 use crate::executor::{
     Message, PkIndicesRef, StreamExecutorError, StreamExecutorResult, Watermark,
 };
@@ -336,10 +337,11 @@ pub(crate) fn mark_cdc_chunk(
 /// For each row of the chunk, forward it to downstream if its pk <= `current_pos` for the
 /// corresponding `vnode`, otherwise ignore it.
 /// We implement it by changing the visibility bitmap.
-pub(crate) fn mark_chunk_ref_by_vnode(
+pub(crate) fn mark_chunk_ref_by_vnode<S: StateStore, SD: ValueRowSerde>(
     chunk: &StreamChunk,
     backfill_state: &BackfillState,
     pk_in_output_indices: PkIndicesRef<'_>,
+    upstream_table: &ReplicatedStateTable<S, SD>,
     pk_order: &[OrderType],
 ) -> StreamExecutorResult<StreamChunk> {
     let chunk = chunk.clone();
@@ -347,7 +349,8 @@ pub(crate) fn mark_chunk_ref_by_vnode(
     let mut new_visibility = BitmapBuilder::with_capacity(ops.len());
     // Use project to avoid allocation.
     for row in data.rows() {
-        let vnode = VirtualNode::compute_row(row, pk_in_output_indices);
+        let pk = row.project(pk_in_output_indices);
+        let vnode = upstream_table.compute_vnode_by_pk(pk);
         let v = match backfill_state.get_progress(&vnode)? {
             // We want to just forward the row, if the vnode has finished backfill.
             BackfillProgressPerVnode::Completed { .. } => true,
@@ -355,10 +358,7 @@ pub(crate) fn mark_chunk_ref_by_vnode(
             BackfillProgressPerVnode::NotStarted => false,
             // If in progress, we need to check row <= current_pos.
             BackfillProgressPerVnode::InProgress { current_pos, .. } => {
-                let lhs = row.project(pk_in_output_indices);
-                let rhs = current_pos;
-                let order = cmp_datum_iter(lhs.iter(), rhs.iter(), pk_order.iter().copied());
-                match order {
+                match cmp_datum_iter(pk.iter(), current_pos.iter(), pk_order.iter().copied()) {
                     Ordering::Less | Ordering::Equal => true,
                     Ordering::Greater => false,
                 }
@@ -716,6 +716,7 @@ where
 /// - Format: | vnode | pk | true | `row_count` |
 /// - If previous state is `InProgress` / `NotStarted`: Persist.
 /// - If previous state is Completed: Do not persist.
+///
 /// TODO(kwannoel): we should check committed state to be all `finished` in the tests.
 /// TODO(kwannoel): Instead of persisting state per vnode each time,
 /// we can optimize by persisting state for a subset of vnodes which were updated.
diff --git a/src/stream/src/executor/barrier_align.rs b/src/stream/src/executor/barrier_align.rs
index 25b363f0a42ce..f485615791945 100644
--- a/src/stream/src/executor/barrier_align.rs
+++ b/src/stream/src/executor/barrier_align.rs
@@ -46,15 +46,22 @@ pub async fn barrier_align(
     actor_id: ActorId,
     fragment_id: FragmentId,
     metrics: Arc<StreamingMetrics>,
+    executor_name: &str,
 ) {
     let actor_id = actor_id.to_string();
     let fragment_id = fragment_id.to_string();
-    let left_join_barrier_align_duration = metrics
-        .join_barrier_align_duration
-        .with_label_values(&[&actor_id, &fragment_id, "left"]);
-    let right_join_barrier_align_duration = metrics
-        .join_barrier_align_duration
-        .with_label_values(&[&actor_id, &fragment_id, "right"]);
+    let left_barrier_align_duration = metrics.barrier_align_duration.with_label_values(&[
+        &actor_id,
+        &fragment_id,
+        "left",
+        executor_name,
+    ]);
+    let right_barrier_align_duration = metrics.barrier_align_duration.with_label_values(&[
+        &actor_id,
+        &fragment_id,
+        "right",
+        executor_name,
+    ]);
     loop {
         let prefer_left: bool = rand::random();
         let select_result = if prefer_left {
@@ -113,7 +120,7 @@ pub async fn barrier_align(
                         Message::Chunk(chunk) => yield AlignedMessage::Right(chunk),
                         Message::Barrier(barrier) => {
                             yield AlignedMessage::Barrier(barrier);
-                            right_join_barrier_align_duration
+                            right_barrier_align_duration
                                 .observe(start_time.elapsed().as_secs_f64());
                             break;
                         }
@@ -137,8 +144,7 @@ pub async fn barrier_align(
                         Message::Chunk(chunk) => yield AlignedMessage::Left(chunk),
                         Message::Barrier(barrier) => {
                             yield AlignedMessage::Barrier(barrier);
-                            left_join_barrier_align_duration
-                                .observe(start_time.elapsed().as_secs_f64());
+                            left_barrier_align_duration.observe(start_time.elapsed().as_secs_f64());
                             break;
                         }
                     }
@@ -164,7 +170,14 @@ mod tests {
         left: BoxedMessageStream,
         right: BoxedMessageStream,
     ) -> impl Stream<Item = Result<AlignedMessage, StreamExecutorError>> {
-        barrier_align(left, right, 0, 0, Arc::new(StreamingMetrics::unused()))
+        barrier_align(
+            left,
+            right,
+            0,
+            0,
+            Arc::new(StreamingMetrics::unused()),
+            "dummy_executor",
+        )
     }
 
     #[tokio::test]
diff --git a/src/stream/src/executor/chain.rs b/src/stream/src/executor/chain.rs
index ebcbe1e4e49bb..96b9422a97b2c 100644
--- a/src/stream/src/executor/chain.rs
+++ b/src/stream/src/executor/chain.rs
@@ -103,7 +103,6 @@ impl Execute for ChainExecutor {
 
 #[cfg(test)]
 mod test {
-    use std::default::Default;
 
     use futures::StreamExt;
     use risingwave_common::array::stream_chunk::StreamChunkTestExt;
diff --git a/src/stream/src/executor/dedup/append_only_dedup.rs b/src/stream/src/executor/dedup/append_only_dedup.rs
index f73e196815400..8455bd2643e5c 100644
--- a/src/stream/src/executor/dedup/append_only_dedup.rs
+++ b/src/stream/src/executor/dedup/append_only_dedup.rs
@@ -129,8 +129,6 @@ impl<S: StateStore> AppendOnlyDedupExecutor<S> {
                         }
                     }
 
-                    self.cache.update_epoch(barrier.epoch.curr);
-
                     yield Message::Barrier(barrier);
                 }
 
@@ -182,19 +180,15 @@ impl<S: StateStore> Execute for AppendOnlyDedupExecutor<S> {
 #[cfg(test)]
 mod tests {
     use std::sync::atomic::AtomicU64;
-    use std::sync::Arc;
 
-    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, Schema, TableId};
+    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, TableId};
     use risingwave_common::test_prelude::StreamChunkTestExt;
-    use risingwave_common::types::DataType;
     use risingwave_common::util::epoch::test_epoch;
     use risingwave_common::util::sort_util::OrderType;
     use risingwave_storage::memory::MemoryStateStore;
 
     use super::*;
-    use crate::common::table::state_table::StateTable;
     use crate::executor::test_utils::MockSource;
-    use crate::executor::ActorContext;
 
     #[tokio::test]
     async fn test_dedup_executor() {
diff --git a/src/stream/src/executor/dedup/cache.rs b/src/stream/src/executor/dedup/cache.rs
index 245e7b7dbae95..3760583ea9547 100644
--- a/src/stream/src/executor/dedup/cache.rs
+++ b/src/stream/src/executor/dedup/cache.rs
@@ -16,7 +16,7 @@ use std::hash::Hash;
 
 use risingwave_common_estimate_size::EstimateSize;
 
-use crate::cache::{new_unbounded, ManagedLruCache};
+use crate::cache::ManagedLruCache;
 use crate::common::metrics::MetricsInfo;
 use crate::executor::prelude::*;
 
@@ -27,8 +27,8 @@ pub struct DedupCache<K: Hash + Eq + EstimateSize> {
 }
 
 impl<K: Hash + Eq + EstimateSize> DedupCache<K> {
-    pub fn new(watermark_epoch: AtomicU64Ref, metrics_info: MetricsInfo) -> Self {
-        let cache = new_unbounded(watermark_epoch, metrics_info);
+    pub fn new(watermark_sequence: AtomicU64Ref, metrics_info: MetricsInfo) -> Self {
+        let cache = ManagedLruCache::unbounded(watermark_sequence, metrics_info);
         Self { inner: cache }
     }
 
@@ -53,11 +53,6 @@ impl<K: Hash + Eq + EstimateSize> DedupCache<K> {
         self.inner.evict()
     }
 
-    pub fn update_epoch(&mut self, epoch: u64) {
-        // Update the current epoch in `ManagedLruCache`
-        self.inner.update_epoch(epoch)
-    }
-
     /// Clear everything in the cache.
     pub fn clear(&mut self) {
         self.inner.clear()
diff --git a/src/stream/src/executor/dispatch.rs b/src/stream/src/executor/dispatch.rs
index 6c644466f9683..9e6503c2d9a28 100644
--- a/src/stream/src/executor/dispatch.rs
+++ b/src/stream/src/executor/dispatch.rs
@@ -1029,26 +1029,21 @@ impl Dispatcher for SimpleDispatcher {
 #[cfg(test)]
 mod tests {
     use std::hash::{BuildHasher, Hasher};
-    use std::sync::{Arc, Mutex};
+    use std::sync::Mutex;
 
     use async_trait::async_trait;
-    use futures::{pin_mut, StreamExt};
-    use itertools::Itertools;
+    use futures::pin_mut;
     use risingwave_common::array::stream_chunk::StreamChunkTestExt;
-    use risingwave_common::array::{Array, ArrayBuilder, I32ArrayBuilder, Op};
-    use risingwave_common::catalog::Schema;
+    use risingwave_common::array::{Array, ArrayBuilder, I32ArrayBuilder};
     use risingwave_common::config;
-    use risingwave_common::hash::VirtualNode;
     use risingwave_common::util::epoch::test_epoch;
     use risingwave_common::util::hash_util::Crc32FastBuilder;
-    use risingwave_common::util::iter_util::ZipEqFast;
     use risingwave_pb::stream_plan::DispatcherType;
 
     use super::*;
     use crate::executor::exchange::output::Output;
     use crate::executor::exchange::permit::channel_for_test;
     use crate::executor::receiver::ReceiverExecutor;
-    use crate::executor::Execute;
     use crate::task::test_utils::helper_make_local_actor;
 
     #[derive(Debug)]
diff --git a/src/stream/src/executor/dml.rs b/src/stream/src/executor/dml.rs
index b8839d76000c1..adcb3f01ab8bd 100644
--- a/src/stream/src/executor/dml.rs
+++ b/src/stream/src/executor/dml.rs
@@ -273,13 +273,9 @@ impl Execute for DmlExecutor {
 
 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
 
-    use risingwave_common::array::StreamChunk;
-    use risingwave_common::catalog::{ColumnId, Field, Schema, INITIAL_TABLE_VERSION_ID};
+    use risingwave_common::catalog::{ColumnId, Field, INITIAL_TABLE_VERSION_ID};
     use risingwave_common::test_prelude::StreamChunkTestExt;
-    use risingwave_common::transaction::transaction_id::TxnId;
-    use risingwave_common::types::DataType;
     use risingwave_common::util::epoch::test_epoch;
     use risingwave_dml::dml_manager::DmlManager;
 
diff --git a/src/stream/src/executor/dynamic_filter.rs b/src/stream/src/executor/dynamic_filter.rs
index ddbe8352b2e8a..adba9aded3fa9 100644
--- a/src/stream/src/executor/dynamic_filter.rs
+++ b/src/stream/src/executor/dynamic_filter.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::ops::Bound::{self, *};
-use std::sync::Arc;
 
 use futures::stream;
 use risingwave_common::array::{Array, ArrayImpl, Op};
@@ -21,7 +20,7 @@ use risingwave_common::bail;
 use risingwave_common::buffer::{Bitmap, BitmapBuilder};
 use risingwave_common::hash::VnodeBitmapExt;
 use risingwave_common::row::{self, once, OwnedRow as RowData};
-use risingwave_common::types::{DataType, Datum, DefaultOrd, ScalarImpl, ToDatumRef, ToOwnedDatum};
+use risingwave_common::types::{DefaultOrd, ToDatumRef, ToOwnedDatum};
 use risingwave_common::util::iter_util::ZipEqDebug;
 use risingwave_expr::expr::{
     build_func_non_strict, InputRefExpression, LiteralExpression, NonStrictExpression,
@@ -290,6 +289,7 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> DynamicFilterExecutor<S, US
             self.ctx.id,
             self.ctx.fragment_id,
             self.metrics.clone(),
+            "Dynamic Filter",
         );
 
         pin_mut!(aligned_stream);
@@ -489,9 +489,8 @@ impl<S: StateStore, const USE_WATERMARK_CACHE: bool> Execute
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::array::stream_chunk::StreamChunkTestExt;
     use risingwave_common::array::*;
-    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, Schema, TableId};
+    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, TableId};
     use risingwave_common::util::epoch::test_epoch;
     use risingwave_common::util::sort_util::OrderType;
     use risingwave_hummock_sdk::HummockReadEpoch;
@@ -500,7 +499,6 @@ mod tests {
 
     use super::*;
     use crate::executor::test_utils::{MessageSender, MockSource, StreamExecutorTestExt};
-    use crate::executor::{ActorContext, StreamExecutorResult};
 
     async fn create_in_memory_state_table(
         mem_state: MemoryStateStore,
diff --git a/src/stream/src/executor/error.rs b/src/stream/src/executor/error.rs
index db70dcffa7fdd..41b8198b646a3 100644
--- a/src/stream/src/executor/error.rs
+++ b/src/stream/src/executor/error.rs
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::convert::AsRef;
-
 use risingwave_common::array::ArrayError;
 use risingwave_common::error::{BoxedError, NotImplemented};
 use risingwave_common::util::value_encoding::error::ValueEncodingError;
@@ -113,7 +111,7 @@ pub enum ErrorKind {
     NotImplemented(#[from] NotImplemented),
 
     #[error(transparent)]
-    Internal(
+    Uncategorized(
         #[from]
         #[backtrace]
         anyhow::Error,
@@ -150,7 +148,7 @@ impl From<PbFieldNotFound> for StreamExecutorError {
 
 impl From<String> for StreamExecutorError {
     fn from(s: String) -> Self {
-        ErrorKind::Internal(anyhow::anyhow!(s)).into()
+        ErrorKind::Uncategorized(anyhow::anyhow!(s)).into()
     }
 }
 
diff --git a/src/stream/src/executor/exchange/input.rs b/src/stream/src/executor/exchange/input.rs
index 11796441326aa..4c5648ab1dd49 100644
--- a/src/stream/src/executor/exchange/input.rs
+++ b/src/stream/src/executor/exchange/input.rs
@@ -16,7 +16,7 @@ use std::pin::Pin;
 use std::task::{Context, Poll};
 
 use anyhow::Context as _;
-use futures::{pin_mut, Stream};
+use futures::pin_mut;
 use futures_async_stream::try_stream;
 use pin_project::pin_project;
 use risingwave_common::util::addr::{is_local_address, HostAddr};
@@ -25,9 +25,6 @@ use risingwave_rpc_client::ComputeClientPool;
 
 use super::error::ExchangeChannelClosed;
 use super::permit::Receiver;
-use crate::error::StreamResult;
-use crate::executor::error::StreamExecutorError;
-use crate::executor::monitor::StreamingMetrics;
 use crate::executor::prelude::*;
 use crate::task::{
     FragmentId, LocalBarrierManager, SharedContext, UpDownActorIds, UpDownFragmentIds,
@@ -158,6 +155,9 @@ impl RemoteInput {
         let up_actor_id = up_down_ids.0.to_string();
         let up_fragment_id = up_down_frag.0.to_string();
         let down_fragment_id = up_down_frag.1.to_string();
+        let exchange_frag_recv_size_metrics = metrics
+            .exchange_frag_recv_size
+            .with_guarded_label_values(&[&up_fragment_id, &down_fragment_id]);
 
         let span: await_tree::Span = format!("RemoteInput (actor {up_actor_id})").into();
 
@@ -170,10 +170,7 @@ impl RemoteInput {
                     let msg = message.unwrap();
                     let bytes = Message::get_encoded_len(&msg);
 
-                    metrics
-                        .exchange_frag_recv_size
-                        .with_label_values(&[&up_fragment_id, &down_fragment_id])
-                        .inc_by(bytes as u64);
+                    exchange_frag_recv_size_metrics.inc_by(bytes as u64);
 
                     let msg_res = Message::from_protobuf(&msg);
                     if let Some(add_back_permits) = match permits.unwrap().value {
diff --git a/src/stream/src/executor/filter.rs b/src/stream/src/executor/filter.rs
index 4d1ecb098bd8f..b183719a9a677 100644
--- a/src/stream/src/executor/filter.rs
+++ b/src/stream/src/executor/filter.rs
@@ -157,11 +157,8 @@ impl FilterExecutor {
 
 #[cfg(test)]
 mod tests {
-    use futures::StreamExt;
     use risingwave_common::array::stream_chunk::StreamChunkTestExt;
-    use risingwave_common::array::StreamChunk;
-    use risingwave_common::catalog::{Field, Schema};
-    use risingwave_common::types::DataType;
+    use risingwave_common::catalog::Field;
 
     use super::super::test_utils::expr::build_from_pretty;
     use super::super::test_utils::MockSource;
diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs
index 02ad790e53983..2659e2c154c72 100644
--- a/src/stream/src/executor/hash_agg.rs
+++ b/src/stream/src/executor/hash_agg.rs
@@ -31,8 +31,9 @@ use super::aggregation::{
     agg_call_filter_res, iter_table_storage, AggStateStorage, DistinctDeduplicater, GroupKey,
     OnlyOutputIfHasInput,
 };
+use super::monitor::HashAggMetrics;
 use super::sort_buffer::SortBuffer;
-use crate::cache::{cache_may_stale, new_with_hasher, ManagedLruCache};
+use crate::cache::{cache_may_stale, ManagedLruCache};
 use crate::common::metrics::MetricsInfo;
 use crate::executor::aggregation::AggGroup as GenericAggGroup;
 use crate::executor::prelude::*;
@@ -109,7 +110,7 @@ struct ExecutorInner<K: HashKey, S: StateStore> {
     distinct_dedup_tables: HashMap<usize, StateTable<S>>,
 
     /// Watermark epoch.
-    watermark_epoch: AtomicU64Ref,
+    watermark_sequence: AtomicU64Ref,
 
     /// State cache size for extreme agg.
     extreme_cache_size: usize,
@@ -133,6 +134,9 @@ impl<K: HashKey, S: StateStore> ExecutorInner<K, S> {
 }
 
 struct ExecutionVars<K: HashKey, S: StateStore> {
+    metrics: HashAggMetrics,
+
+    // Stats collected during execution, will be flushed to metrics at the end of each barrier.
     stats: ExecutionStats,
 
     /// Cache for [`AggGroup`]s. `HashKey` -> `AggGroup`.
@@ -215,7 +219,7 @@ impl<K: HashKey, S: StateStore> HashAggExecutor<K, S> {
                 storages: args.storages,
                 intermediate_state_table: args.intermediate_state_table,
                 distinct_dedup_tables: args.distinct_dedup_tables,
-                watermark_epoch: args.watermark_epoch,
+                watermark_sequence: args.watermark_epoch,
                 extreme_cache_size: args.extreme_cache_size,
                 chunk_size: args.extra.chunk_size,
                 max_dirty_groups_heap_size: args.extra.max_dirty_groups_heap_size,
@@ -388,18 +392,11 @@ impl<K: HashKey, S: StateStore> HashAggExecutor<K, S> {
         }
 
         // Update the metrics.
-        let actor_id_str = this.actor_ctx.id.to_string();
-        let fragment_id_str = this.actor_ctx.fragment_id.to_string();
-        let table_id_str = this.intermediate_state_table.table_id().to_string();
-        this.actor_ctx
-            .streaming_metrics
+        vars.metrics
             .agg_dirty_groups_count
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
             .set(vars.dirty_groups.len() as i64);
-        this.actor_ctx
-            .streaming_metrics
+        vars.metrics
             .agg_dirty_groups_heap_size
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
             .set(vars.dirty_groups.estimated_heap_size() as i64);
 
         Ok(())
@@ -502,34 +499,21 @@ impl<K: HashKey, S: StateStore> HashAggExecutor<K, S> {
         vars.agg_group_cache.evict();
     }
 
-    fn update_metrics(this: &ExecutorInner<K, S>, vars: &mut ExecutionVars<K, S>) {
-        let actor_id_str = this.actor_ctx.id.to_string();
-        let fragment_id_str = this.actor_ctx.fragment_id.to_string();
-        let table_id_str = this.intermediate_state_table.table_id().to_string();
-        this.actor_ctx
-            .streaming_metrics
+    fn flush_metrics(_this: &ExecutorInner<K, S>, vars: &mut ExecutionVars<K, S>) {
+        vars.metrics
             .agg_lookup_miss_count
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
             .inc_by(std::mem::take(&mut vars.stats.lookup_miss_count));
-        this.actor_ctx
-            .streaming_metrics
+        vars.metrics
             .agg_total_lookup_count
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
             .inc_by(std::mem::take(&mut vars.stats.total_lookup_count));
-        this.actor_ctx
-            .streaming_metrics
+        vars.metrics
             .agg_cached_entry_count
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
             .set(vars.agg_group_cache.len() as i64);
-        this.actor_ctx
-            .streaming_metrics
+        vars.metrics
             .agg_chunk_lookup_miss_count
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
             .inc_by(std::mem::take(&mut vars.stats.chunk_lookup_miss_count));
-        this.actor_ctx
-            .streaming_metrics
+        vars.metrics
             .agg_chunk_total_lookup_count
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
             .inc_by(std::mem::take(&mut vars.stats.chunk_total_lookup_count));
     }
 
@@ -570,20 +554,26 @@ impl<K: HashKey, S: StateStore> HashAggExecutor<K, S> {
             this.actor_ctx.id,
             "agg intermediate state table",
         );
+        let metrics = this.actor_ctx.streaming_metrics.new_hash_agg_metrics(
+            this.intermediate_state_table.table_id(),
+            this.actor_ctx.id,
+            this.actor_ctx.fragment_id,
+        );
 
         let mut vars = ExecutionVars {
+            metrics,
             stats: ExecutionStats::new(),
-            agg_group_cache: new_with_hasher(
-                this.watermark_epoch.clone(),
+            agg_group_cache: ManagedLruCache::unbounded_with_hasher(
+                this.watermark_sequence.clone(),
                 agg_group_cache_metrics_info,
                 PrecomputedBuildHasher,
             ),
             dirty_groups: Default::default(),
             distinct_dedup: DistinctDeduplicater::new(
                 &this.agg_calls,
-                this.watermark_epoch.clone(),
+                this.watermark_sequence.clone(),
                 &this.distinct_dedup_tables,
-                this.actor_ctx.clone(),
+                &this.actor_ctx,
             ),
             buffered_watermarks: vec![None; this.group_key_indices.len()],
             window_watermark: None,
@@ -606,10 +596,6 @@ impl<K: HashKey, S: StateStore> HashAggExecutor<K, S> {
         this.all_state_tables_mut().for_each(|table| {
             table.init_epoch(barrier.epoch);
         });
-        vars.agg_group_cache.update_epoch(barrier.epoch.curr);
-        vars.distinct_dedup.dedup_caches_mut().for_each(|cache| {
-            cache.update_epoch(barrier.epoch.curr);
-        });
 
         yield Message::Barrier(barrier);
 
@@ -642,7 +628,7 @@ impl<K: HashKey, S: StateStore> HashAggExecutor<K, S> {
                     Self::try_flush_data(&mut this).await?;
                 }
                 Message::Barrier(barrier) => {
-                    Self::update_metrics(&this, &mut vars);
+                    Self::flush_metrics(&this, &mut vars);
 
                     #[for_await]
                     for chunk in Self::flush_data(&mut this, &mut vars) {
@@ -681,12 +667,6 @@ impl<K: HashKey, S: StateStore> HashAggExecutor<K, S> {
                         }
                     }
 
-                    // Update the current epoch.
-                    vars.agg_group_cache.update_epoch(barrier.epoch.curr);
-                    vars.distinct_dedup.dedup_caches_mut().for_each(|cache| {
-                        cache.update_epoch(barrier.epoch.curr);
-                    });
-
                     yield Message::Barrier(barrier);
                 }
             }
diff --git a/src/stream/src/executor/hash_join.rs b/src/stream/src/executor/hash_join.rs
index ea0253c9cbbc1..0bf825138d92a 100644
--- a/src/stream/src/executor/hash_join.rs
+++ b/src/stream/src/executor/hash_join.rs
@@ -11,14 +11,17 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 use std::collections::{BTreeMap, HashSet};
+use std::num::NonZeroU32;
+use std::sync::LazyLock;
 use std::time::Duration;
 
+use governor::{Quota, RateLimiter};
 use itertools::Itertools;
 use multimap::MultiMap;
-use risingwave_common::array::{Op, RowRef};
+use risingwave_common::array::Op;
 use risingwave_common::hash::{HashKey, NullBitmap};
+use risingwave_common::log::LogSuppresser;
 use risingwave_common::types::{DefaultOrd, ToOwnedDatum};
 use risingwave_common::util::epoch::EpochPair;
 use risingwave_common::util::iter_util::ZipEqDebug;
@@ -30,7 +33,7 @@ use self::builder::JoinChunkBuilder;
 use super::barrier_align::*;
 use super::join::hash_join::*;
 use super::join::row::JoinRow;
-use super::join::{JoinTypePrimitive, SideTypePrimitive, *};
+use super::join::*;
 use super::watermark::*;
 use crate::executor::join::builder::JoinStreamChunkBuilder;
 use crate::executor::prelude::*;
@@ -157,6 +160,8 @@ pub struct HashJoinExecutor<K: HashKey, S: StateStore, const T: JoinTypePrimitiv
 
     /// watermark column index -> `BufferedWatermarks`
     watermark_buffers: BTreeMap<usize, BufferedWatermarks<SideTypePrimitive>>,
+
+    high_join_amplification_threshold: usize,
 }
 
 impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> std::fmt::Debug
@@ -193,6 +198,7 @@ struct EqJoinArgs<'a, K: HashKey, S: StateStore> {
     append_only_optimize: bool,
     chunk_size: usize,
     cnt_rows_received: &'a mut u32,
+    high_join_amplification_threshold: usize,
 }
 
 impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K, S, T> {
@@ -216,6 +222,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
         is_append_only: bool,
         metrics: Arc<StreamingMetrics>,
         chunk_size: usize,
+        high_join_amplification_threshold: usize,
     ) -> Self {
         let side_l_column_n = input_l.schema().len();
 
@@ -444,6 +451,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
             chunk_size,
             cnt_rows_received: 0,
             watermark_buffers,
+            high_join_amplification_threshold,
         }
     }
 
@@ -457,6 +465,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
             self.ctx.id,
             self.ctx.fragment_id,
             self.metrics.clone(),
+            "Join",
         );
         pin_mut!(aligned_stream);
 
@@ -536,6 +545,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
                         append_only_optimize: self.append_only_optimize,
                         chunk_size: self.chunk_size,
                         cnt_rows_received: &mut self.cnt_rows_received,
+                        high_join_amplification_threshold: self.high_join_amplification_threshold,
                     }) {
                         left_time += left_start_time.elapsed();
                         yield Message::Chunk(chunk?);
@@ -560,6 +570,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
                         append_only_optimize: self.append_only_optimize,
                         chunk_size: self.chunk_size,
                         cnt_rows_received: &mut self.cnt_rows_received,
+                        high_join_amplification_threshold: self.high_join_amplification_threshold,
                     }) {
                         right_time += right_start_time.elapsed();
                         yield Message::Chunk(chunk?);
@@ -584,10 +595,6 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
                         self.side_r.ht.update_vnode_bitmap(vnode_bitmap);
                     }
 
-                    // Update epoch for managed cache.
-                    self.side_l.ht.update_epoch(barrier.epoch.curr);
-                    self.side_r.ht.update_epoch(barrier.epoch.curr);
-
                     // Report metrics of cached join rows/entries
                     for (join_cached_entry_count, ht) in [
                         (&left_join_cached_entry_count, &self.side_l.ht),
@@ -774,6 +781,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
             append_only_optimize,
             chunk_size,
             cnt_rows_received,
+            high_join_amplification_threshold,
             ..
         } = args;
 
@@ -829,6 +837,27 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
 
             if let Some(rows) = &matched_rows {
                 join_matched_join_keys.observe(rows.len() as _);
+                if rows.len() > high_join_amplification_threshold {
+                    static LOG_SUPPERSSER: LazyLock<LogSuppresser> = LazyLock::new(|| {
+                        LogSuppresser::new(RateLimiter::direct(Quota::per_minute(
+                            NonZeroU32::new(1).unwrap(),
+                        )))
+                    });
+                    if let Ok(suppressed_count) = LOG_SUPPERSSER.check() {
+                        let join_key_data_types = side_update.ht.join_key_data_types();
+                        let key = key.deserialize(join_key_data_types)?;
+                        tracing::warn!(target: "high_join_amplification",
+                            suppressed_count,
+                            matched_rows_len = rows.len(),
+                            update_table_id = side_update.ht.table_id(),
+                            match_table_id = side_match.ht.table_id(),
+                            join_key = ?key,
+                            actor_id = ctx.id,
+                            fragment_id = ctx.fragment_id,
+                            "large rows matched for join key"
+                        );
+                    }
+                }
             } else {
                 join_matched_join_keys.observe(0.0)
             }
@@ -1051,20 +1080,16 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive> HashJoinExecutor<K,
 mod tests {
     use std::sync::atomic::AtomicU64;
 
-    use risingwave_common::array::stream_chunk::StreamChunkTestExt;
     use risingwave_common::array::*;
-    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, Schema, TableId};
+    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, TableId};
     use risingwave_common::hash::{Key128, Key64};
-    use risingwave_common::types::ScalarImpl;
     use risingwave_common::util::epoch::test_epoch;
     use risingwave_common::util::sort_util::OrderType;
     use risingwave_storage::memory::MemoryStateStore;
 
     use super::*;
-    use crate::common::table::state_table::StateTable;
     use crate::executor::test_utils::expr::build_from_pretty;
     use crate::executor::test_utils::{MessageSender, MockSource, StreamExecutorTestExt};
-    use crate::executor::{ActorContext, Barrier, EpochPair};
 
     async fn create_in_memory_state_table(
         mem_state: MemoryStateStore,
@@ -1192,6 +1217,7 @@ mod tests {
             false,
             Arc::new(StreamingMetrics::unused()),
             1024,
+            2048,
         );
         (tx_l, tx_r, executor.boxed().execute())
     }
@@ -1284,6 +1310,7 @@ mod tests {
             true,
             Arc::new(StreamingMetrics::unused()),
             1024,
+            2048,
         );
         (tx_l, tx_r, executor.boxed().execute())
     }
diff --git a/src/stream/src/executor/hop_window.rs b/src/stream/src/executor/hop_window.rs
index 28408ad046a4e..cd3c0071a18c7 100644
--- a/src/stream/src/executor/hop_window.rs
+++ b/src/stream/src/executor/hop_window.rs
@@ -226,17 +226,13 @@ impl HopWindowExecutor {
 
 #[cfg(test)]
 mod tests {
-    use futures::StreamExt;
     use risingwave_common::array::stream_chunk::StreamChunkTestExt;
-    use risingwave_common::catalog::{Field, Schema};
+    use risingwave_common::catalog::Field;
     use risingwave_common::types::test_utils::IntervalTestExt;
-    use risingwave_common::types::{DataType, Interval};
     use risingwave_expr::expr::test_utils::make_hop_window_expression;
-    use risingwave_expr::expr::NonStrictExpression;
 
     use super::*;
     use crate::executor::test_utils::MockSource;
-    use crate::executor::{ActorContext, Execute, StreamChunk};
 
     const CHUNK_SIZE: usize = 256;
 
diff --git a/src/stream/src/executor/integration_tests.rs b/src/stream/src/executor/integration_tests.rs
index d29224aa63aa4..8f73fe26ee7de 100644
--- a/src/stream/src/executor/integration_tests.rs
+++ b/src/stream/src/executor/integration_tests.rs
@@ -12,13 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::{Arc, Mutex};
+use std::sync::Mutex;
 
-use futures::StreamExt;
 use futures_async_stream::try_stream;
 use multimap::MultiMap;
 use risingwave_common::array::*;
-use risingwave_common::catalog::{Field, Schema};
+use risingwave_common::catalog::Field;
 use risingwave_common::config;
 use risingwave_common::types::*;
 use risingwave_common::util::epoch::{test_epoch, EpochExt};
@@ -29,15 +28,12 @@ use risingwave_storage::memory::MemoryStateStore;
 
 use super::exchange::permit::channel_for_test;
 use super::*;
-use crate::executor::actor::ActorContext;
 use crate::executor::dispatch::*;
 use crate::executor::exchange::output::{BoxedOutput, LocalOutput};
 use crate::executor::monitor::StreamingMetrics;
-use crate::executor::receiver::ReceiverExecutor;
 use crate::executor::test_utils::agg_executor::{
     generate_agg_schema, new_boxed_simple_agg_executor,
 };
-use crate::executor::{Execute, MergeExecutor, ProjectExecutor, StatelessSimpleAggExecutor};
 use crate::task::{LocalBarrierManager, SharedContext};
 
 /// This test creates a merger-dispatcher pair, and run a sum. Each chunk
@@ -182,6 +178,7 @@ async fn test_merger_sum_aggr() {
         MultiMap::new(),
         vec![],
         0.0,
+        false,
     );
 
     let items = Arc::new(Mutex::new(vec![]));
diff --git a/src/stream/src/executor/join/builder.rs b/src/stream/src/executor/join/builder.rs
index ca34ba9cad9e8..72208aa45ded8 100644
--- a/src/stream/src/executor/join/builder.rs
+++ b/src/stream/src/executor/join/builder.rs
@@ -20,7 +20,6 @@ use risingwave_common::types::{DataType, DatumRef};
 use self::row::JoinRow;
 // Re-export `StreamChunkBuilder`.
 use super::*;
-use super::{JoinTypePrimitive, SideTypePrimitive};
 
 type IndexMappings = Vec<(usize, usize)>;
 
diff --git a/src/stream/src/executor/join/hash_join.rs b/src/stream/src/executor/join/hash_join.rs
index 88b3ede24b619..6398a475a6062 100644
--- a/src/stream/src/executor/join/hash_join.rs
+++ b/src/stream/src/executor/join/hash_join.rs
@@ -13,12 +13,13 @@
 // limitations under the License.
 
 use std::alloc::Global;
+use std::cmp::Ordering;
 use std::ops::{Bound, Deref, DerefMut};
 use std::sync::Arc;
 
 use anyhow::Context;
 use futures::future::{join, try_join};
-use futures::StreamExt;
+use futures::{pin_mut, stream, StreamExt};
 use futures_async_stream::for_await;
 use local_stats_alloc::{SharedStatsAlloc, StatsAlloc};
 use risingwave_common::buffer::Bitmap;
@@ -27,6 +28,7 @@ use risingwave_common::metrics::LabelGuardedIntCounter;
 use risingwave_common::row::{OwnedRow, Row, RowExt};
 use risingwave_common::types::{DataType, ScalarImpl};
 use risingwave_common::util::epoch::EpochPair;
+use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::row_serde::OrderedRowSerde;
 use risingwave_common::util::sort_util::OrderType;
 use risingwave_common_estimate_size::EstimateSize;
@@ -34,10 +36,10 @@ use risingwave_storage::store::PrefetchOptions;
 use risingwave_storage::StateStore;
 
 use super::row::{DegreeType, EncodedJoinRow};
-use crate::cache::{new_with_hasher_in, ManagedLruCache};
+use crate::cache::ManagedLruCache;
 use crate::common::metrics::MetricsInfo;
 use crate::common::table::state_table::StateTable;
-use crate::consistency::{consistency_error, enable_strict_consistency};
+use crate::consistency::{consistency_error, consistency_panic, enable_strict_consistency};
 use crate::executor::error::StreamExecutorResult;
 use crate::executor::join::row::JoinRow;
 use crate::executor::monitor::StreamingMetrics;
@@ -103,9 +105,9 @@ pub struct JoinHashMapMetrics {
     insert_cache_miss_count: usize,
 
     // Metrics
-    join_lookup_total_count_metric: LabelGuardedIntCounter<5>,
-    join_lookup_miss_count_metric: LabelGuardedIntCounter<5>,
-    join_insert_cache_miss_count_metrics: LabelGuardedIntCounter<5>,
+    join_lookup_total_count_metric: LabelGuardedIntCounter<4>,
+    join_lookup_miss_count_metric: LabelGuardedIntCounter<4>,
+    join_insert_cache_miss_count_metrics: LabelGuardedIntCounter<4>,
 }
 
 impl JoinHashMapMetrics {
@@ -115,37 +117,19 @@ impl JoinHashMapMetrics {
         fragment_id: FragmentId,
         side: &'static str,
         join_table_id: u32,
-        degree_table_id: u32,
     ) -> Self {
         let actor_id = actor_id.to_string();
         let fragment_id = fragment_id.to_string();
         let join_table_id = join_table_id.to_string();
-        let degree_table_id = degree_table_id.to_string();
-        let join_lookup_total_count_metric =
-            metrics.join_lookup_total_count.with_guarded_label_values(&[
-                (side),
-                &join_table_id,
-                &degree_table_id,
-                &actor_id,
-                &fragment_id,
-            ]);
-        let join_lookup_miss_count_metric =
-            metrics.join_lookup_miss_count.with_guarded_label_values(&[
-                (side),
-                &join_table_id,
-                &degree_table_id,
-                &actor_id,
-                &fragment_id,
-            ]);
+        let join_lookup_total_count_metric = metrics
+            .join_lookup_total_count
+            .with_guarded_label_values(&[(side), &join_table_id, &actor_id, &fragment_id]);
+        let join_lookup_miss_count_metric = metrics
+            .join_lookup_miss_count
+            .with_guarded_label_values(&[(side), &join_table_id, &actor_id, &fragment_id]);
         let join_insert_cache_miss_count_metrics = metrics
             .join_insert_cache_miss_count
-            .with_guarded_label_values(&[
-                (side),
-                &join_table_id,
-                &degree_table_id,
-                &actor_id,
-                &fragment_id,
-            ]);
+            .with_guarded_label_values(&[(side), &join_table_id, &actor_id, &fragment_id]);
 
         Self {
             lookup_miss_count: 0,
@@ -233,7 +217,7 @@ impl<K: HashKey, S: StateStore> JoinHashMap<K, S> {
     /// Create a [`JoinHashMap`] with the given LRU capacity.
     #[allow(clippy::too_many_arguments)]
     pub fn new(
-        watermark_epoch: AtomicU64Ref,
+        watermark_sequence: AtomicU64Ref,
         join_key_data_types: Vec<DataType>,
         state_join_key_indices: Vec<usize>,
         state_all_data_types: Vec<DataType>,
@@ -263,7 +247,6 @@ impl<K: HashKey, S: StateStore> JoinHashMap<K, S> {
         );
 
         let join_table_id = state_table.table_id();
-        let degree_table_id = degree_table.table_id();
         let state = TableInner {
             pk_indices: state_pk_indices,
             join_key_indices: state_join_key_indices,
@@ -284,11 +267,15 @@ impl<K: HashKey, S: StateStore> JoinHashMap<K, S> {
             metrics.clone(),
             join_table_id,
             actor_id,
-            &format!("hash join {}", side),
+            format!("hash join {}", side),
         );
 
-        let cache =
-            new_with_hasher_in(watermark_epoch, metrics_info, PrecomputedBuildHasher, alloc);
+        let cache = ManagedLruCache::unbounded_with_hasher_in(
+            watermark_sequence,
+            metrics_info,
+            PrecomputedBuildHasher,
+            alloc,
+        );
 
         Self {
             inner: cache,
@@ -299,28 +286,15 @@ impl<K: HashKey, S: StateStore> JoinHashMap<K, S> {
             degree_state,
             need_degree_table,
             pk_contained_in_jk,
-            metrics: JoinHashMapMetrics::new(
-                &metrics,
-                actor_id,
-                fragment_id,
-                side,
-                join_table_id,
-                degree_table_id,
-            ),
+            metrics: JoinHashMapMetrics::new(&metrics, actor_id, fragment_id, side, join_table_id),
         }
     }
 
     pub fn init(&mut self, epoch: EpochPair) {
-        self.update_epoch(epoch.curr);
         self.state.table.init_epoch(epoch);
         self.degree_state.table.init_epoch(epoch);
     }
 
-    pub fn update_epoch(&mut self, epoch: u64) {
-        // Update the current epoch in `ManagedLruCache`
-        self.inner.update_epoch(epoch)
-    }
-
     /// Update the vnode bitmap and manipulate the cache if necessary.
     pub fn update_vnode_bitmap(&mut self, vnode_bitmap: Arc<Bitmap>) -> bool {
         let (_previous_vnode_bitmap, cache_may_stale) =
@@ -387,39 +361,114 @@ impl<K: HashKey, S: StateStore> JoinHashMap<K, S> {
 
             let mut pinned_table_iter = std::pin::pin!(table_iter);
             let mut pinned_degree_table_iter = std::pin::pin!(degree_table_iter);
+
+            // For better tolerating inconsistent stream, we have to first buffer all rows and
+            // degree rows, and check the number of them, then iterate on them.
+            let mut rows = vec![];
+            let mut degree_rows = vec![];
+            let mut inconsistency_happened = false;
             loop {
-                // Iterate on both iterators and ensure they have same size. Basically `zip_eq()`.
-                let (row, degree) =
+                let (row, degree_row) =
                     join(pinned_table_iter.next(), pinned_degree_table_iter.next()).await;
-                let (row, degree) = match (row, degree) {
+                let (row, degree_row) = match (row, degree_row) {
                     (None, None) => break,
-                    (None, Some(_)) | (Some(_), None) => {
-                        panic!("mismatched row and degree table of join key: {:?}", &key)
+                    (None, Some(_)) => {
+                        inconsistency_happened = true;
+                        consistency_panic!(
+                            "mismatched row and degree table of join key: {:?}, degree table has more rows",
+                            &key
+                        );
+                        break;
+                    }
+                    (Some(_), None) => {
+                        inconsistency_happened = true;
+                        consistency_panic!(
+                            "mismatched row and degree table of join key: {:?}, input table has more rows",
+                            &key
+                        );
+                        break;
                     }
                     (Some(r), Some(d)) => (r, d),
                 };
 
                 let row = row?;
-                let degree_row = degree?;
-                let pk1 = row.key();
-                let pk2 = degree_row.key();
-                debug_assert_eq!(
-                    pk1, pk2,
-                    "mismatched pk in degree table: pk1: {pk1:?}, pk2: {pk2:?}",
-                );
-                let pk = row
-                    .as_ref()
-                    .project(&self.state.pk_indices)
-                    .memcmp_serialize(&self.pk_serializer);
-                let degree_i64 = degree_row
-                    .datum_at(degree_row.len() - 1)
-                    .expect("degree should not be NULL");
-                entry_state
-                    .insert(
-                        pk,
-                        JoinRow::new(row.row(), degree_i64.into_int64() as u64).encode(),
-                    )
-                    .with_context(|| self.state.error_context(row.row()))?;
+                let degree_row = degree_row?;
+                rows.push(row);
+                degree_rows.push(degree_row);
+            }
+
+            if inconsistency_happened {
+                // Pk-based row-degree pairing.
+                assert_ne!(rows.len(), degree_rows.len());
+
+                let row_iter = stream::iter(rows.into_iter()).peekable();
+                let degree_row_iter = stream::iter(degree_rows.into_iter()).peekable();
+                pin_mut!(row_iter);
+                pin_mut!(degree_row_iter);
+
+                loop {
+                    match join(row_iter.as_mut().peek(), degree_row_iter.as_mut().peek()).await {
+                        (None, _) | (_, None) => break,
+                        (Some(row), Some(degree_row)) => match row.key().cmp(degree_row.key()) {
+                            Ordering::Greater => {
+                                degree_row_iter.next().await;
+                            }
+                            Ordering::Less => {
+                                row_iter.next().await;
+                            }
+                            Ordering::Equal => {
+                                let row = row_iter.next().await.unwrap();
+                                let degree_row = degree_row_iter.next().await.unwrap();
+
+                                let pk = row
+                                    .as_ref()
+                                    .project(&self.state.pk_indices)
+                                    .memcmp_serialize(&self.pk_serializer);
+                                let degree_i64 = degree_row
+                                    .datum_at(degree_row.len() - 1)
+                                    .expect("degree should not be NULL");
+                                entry_state
+                                    .insert(
+                                        pk,
+                                        JoinRow::new(row.row(), degree_i64.into_int64() as u64)
+                                            .encode(),
+                                    )
+                                    .with_context(|| self.state.error_context(row.row()))?;
+                            }
+                        },
+                    }
+                }
+            } else {
+                // 1 to 1 row-degree pairing.
+                // Actually it's possible that both the input data table and the degree table missed
+                // some equal number of rows, but let's ignore this case because it should be rare.
+
+                assert_eq!(rows.len(), degree_rows.len());
+
+                #[for_await]
+                for (row, degree_row) in
+                    stream::iter(rows.into_iter().zip_eq_fast(degree_rows.into_iter()))
+                {
+                    let pk1 = row.key();
+                    let pk2 = degree_row.key();
+                    debug_assert_eq!(
+                        pk1, pk2,
+                        "mismatched pk in degree table: pk1: {pk1:?}, pk2: {pk2:?}",
+                    );
+                    let pk = row
+                        .as_ref()
+                        .project(&self.state.pk_indices)
+                        .memcmp_serialize(&self.pk_serializer);
+                    let degree_i64 = degree_row
+                        .datum_at(degree_row.len() - 1)
+                        .expect("degree should not be NULL");
+                    entry_state
+                        .insert(
+                            pk,
+                            JoinRow::new(row.row(), degree_i64.into_int64() as u64).encode(),
+                        )
+                        .with_context(|| self.state.error_context(row.row()))?;
+                }
             }
         } else {
             let sub_range: &(Bound<OwnedRow>, Bound<OwnedRow>) =
@@ -603,9 +652,10 @@ impl<K: HashKey, S: StateStore> JoinHashMap<K, S> {
         join_row: &mut JoinRow<OwnedRow>,
     ) {
         self.manipulate_degree(join_row_ref, join_row, |d| {
-            *d = d
-                .checked_sub(1)
-                .expect("Tried to decrement zero join row degree")
+            *d = d.checked_sub(1).unwrap_or_else(|| {
+                consistency_panic!("Tried to decrement zero join row degree");
+                0
+            });
         })
     }
 
@@ -626,6 +676,10 @@ impl<K: HashKey, S: StateStore> JoinHashMap<K, S> {
     pub fn table_id(&self) -> u32 {
         self.state.table.table_id()
     }
+
+    pub fn join_key_data_types(&self) -> &[DataType] {
+        &self.join_key_data_types
+    }
 }
 
 use risingwave_common_estimate_size::KvSize;
@@ -734,7 +788,6 @@ impl JoinEntryState {
 mod tests {
     use itertools::Itertools;
     use risingwave_common::array::*;
-    use risingwave_common::types::{DataType, ScalarImpl};
     use risingwave_common::util::iter_util::ZipEqDebug;
 
     use super::*;
diff --git a/src/stream/src/executor/lookup.rs b/src/stream/src/executor/lookup.rs
index 8d62d27e37f06..6dc4183fe27da 100644
--- a/src/stream/src/executor/lookup.rs
+++ b/src/stream/src/executor/lookup.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use async_trait::async_trait;
-use futures::StreamExt;
 
 mod cache;
 mod sides;
diff --git a/src/stream/src/executor/lookup/cache.rs b/src/stream/src/executor/lookup/cache.rs
index 287f11c2af608..84bab3d737447 100644
--- a/src/stream/src/executor/lookup/cache.rs
+++ b/src/stream/src/executor/lookup/cache.rs
@@ -16,7 +16,7 @@ use risingwave_common::array::Op;
 use risingwave_common::row::RowExt;
 use risingwave_common_estimate_size::collections::{EstimatedHashSet, EstimatedVec};
 
-use crate::cache::{new_unbounded, ManagedLruCache};
+use crate::cache::ManagedLruCache;
 use crate::common::metrics::MetricsInfo;
 use crate::consistency::consistency_panic;
 use crate::executor::prelude::*;
@@ -70,18 +70,13 @@ impl LookupCache {
         self.data.len()
     }
 
-    /// Update the current epoch.
-    pub fn update_epoch(&mut self, epoch: u64) {
-        self.data.update_epoch(epoch);
-    }
-
     /// Clear the cache.
     pub fn clear(&mut self) {
         self.data.clear();
     }
 
-    pub fn new(watermark_epoch: AtomicU64Ref, metrics_info: MetricsInfo) -> Self {
-        let cache = new_unbounded(watermark_epoch, metrics_info);
+    pub fn new(watermark_sequence: AtomicU64Ref, metrics_info: MetricsInfo) -> Self {
+        let cache = ManagedLruCache::unbounded(watermark_sequence, metrics_info);
         Self { data: cache }
     }
 }
diff --git a/src/stream/src/executor/lookup/impl_.rs b/src/stream/src/executor/lookup/impl_.rs
index 8ce6688540e3f..2ec47ca3fc500 100644
--- a/src/stream/src/executor/lookup/impl_.rs
+++ b/src/stream/src/executor/lookup/impl_.rs
@@ -31,6 +31,7 @@ use crate::executor::join::builder::JoinStreamChunkBuilder;
 use crate::executor::lookup::cache::LookupCache;
 use crate::executor::lookup::sides::{ArrangeJoinSide, ArrangeMessage, StreamJoinSide};
 use crate::executor::lookup::LookupExecutor;
+use crate::executor::monitor::LookupExecutorMetrics;
 use crate::executor::prelude::*;
 
 /// Parameters for [`LookupExecutor`].
@@ -238,6 +239,12 @@ impl<S: StateStore> LookupExecutor<S> {
             .boxed()
         };
 
+        let metrics = self.ctx.streaming_metrics.new_lookup_executor_metrics(
+            self.arrangement.storage_table.table_id(),
+            self.ctx.id,
+            self.ctx.fragment_id,
+        );
+
         let (stream_to_output, arrange_to_output) = JoinStreamChunkBuilder::get_i2o_mapping(
             &self.column_mapping,
             self.stream.col_types.len(),
@@ -293,7 +300,11 @@ impl<S: StateStore> LookupExecutor<S> {
 
                     for (op, row) in ops.iter().zip_eq_debug(chunk.rows()) {
                         for matched_row in self
-                            .lookup_one_row(&row, self.last_barrier.as_ref().unwrap().epoch)
+                            .lookup_one_row(
+                                &row,
+                                self.last_barrier.as_ref().unwrap().epoch,
+                                &metrics,
+                            )
                             .await?
                         {
                             tracing::debug!(target: "events::stream::lookup::put", "{:?} {:?}", row, matched_row);
@@ -328,7 +339,6 @@ impl<S: StateStore> LookupExecutor<S> {
         }
 
         // Use the new stream barrier epoch as new cache epoch
-        self.lookup_cache.update_epoch(barrier.epoch.curr);
         self.last_barrier = Some(barrier.clone());
     }
 
@@ -337,30 +347,21 @@ impl<S: StateStore> LookupExecutor<S> {
         &mut self,
         stream_row: &RowRef<'_>,
         epoch_pair: EpochPair,
+        metrics: &LookupExecutorMetrics,
     ) -> StreamExecutorResult<Vec<OwnedRow>> {
         // stream_row is the row from stream side, we need to transform into the correct order of
         // the arrangement side.
         let lookup_row = stream_row
             .project(&self.key_indices_mapping)
             .into_owned_row();
-        let table_id_str = self.arrangement.storage_table.table_id().to_string();
-        let actor_id_str = self.ctx.id.to_string();
-        let fragment_id_str = self.ctx.fragment_id.to_string();
-        self.ctx
-            .streaming_metrics
-            .lookup_total_query_cache_count
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-            .inc();
+
+        metrics.lookup_total_query_cache_count.inc();
         if let Some(result) = self.lookup_cache.lookup(&lookup_row) {
             return Ok(result.iter().cloned().collect_vec());
         }
 
         // cache miss
-        self.ctx
-            .streaming_metrics
-            .lookup_cache_miss_count
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-            .inc();
+        metrics.lookup_cache_miss_count.inc();
 
         tracing::debug!(target: "events::stream::lookup::lookup_row", "{:?}", lookup_row);
 
@@ -405,10 +406,8 @@ impl<S: StateStore> LookupExecutor<S> {
 
         self.lookup_cache.batch_update(lookup_row, all_rows.clone());
 
-        self.ctx
-            .streaming_metrics
+        metrics
             .lookup_cached_entry_count
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
             .set(self.lookup_cache.len() as i64);
 
         Ok(all_rows.into_inner())
diff --git a/src/stream/src/executor/lookup_union.rs b/src/stream/src/executor/lookup_union.rs
index 2189a821fab11..8c0e4103b5b2c 100644
--- a/src/stream/src/executor/lookup_union.rs
+++ b/src/stream/src/executor/lookup_union.rs
@@ -15,7 +15,7 @@
 use async_trait::async_trait;
 use futures::channel::mpsc;
 use futures::future::{join_all, select, Either};
-use futures::{FutureExt, SinkExt, StreamExt};
+use futures::{FutureExt, SinkExt};
 use itertools::Itertools;
 
 use crate::executor::prelude::*;
@@ -120,10 +120,8 @@ impl LookupUnionExecutor {
 #[cfg(test)]
 mod tests {
     use futures::TryStreamExt;
-    use risingwave_common::array::StreamChunk;
-    use risingwave_common::catalog::{Field, Schema};
+    use risingwave_common::catalog::Field;
     use risingwave_common::test_prelude::StreamChunkTestExt;
-    use risingwave_common::types::DataType;
     use risingwave_common::util::epoch::test_epoch;
 
     use super::*;
diff --git a/src/stream/src/executor/managed_state/join/join_entry_state.rs b/src/stream/src/executor/managed_state/join/join_entry_state.rs
deleted file mode 100644
index a4dab83240147..0000000000000
--- a/src/stream/src/executor/managed_state/join/join_entry_state.rs
+++ /dev/null
@@ -1,190 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use risingwave_common_estimate_size::KvSize;
-use thiserror::Error;
-
-use super::*;
-use crate::consistency::{consistency_error, enable_strict_consistency};
-
-/// We manages a `HashMap` in memory for all entries belonging to a join key.
-/// When evicted, `cached` does not hold any entries.
-///
-/// If a `JoinEntryState` exists for a join key, the all records under this
-/// join key will be presented in the cache.
-#[derive(Default)]
-pub struct JoinEntryState {
-    /// The full copy of the state.
-    cached: join_row_set::JoinRowSet<PkType, StateValueType>,
-    kv_heap_size: KvSize,
-}
-
-impl EstimateSize for JoinEntryState {
-    fn estimated_heap_size(&self) -> usize {
-        // TODO: Add btreemap internal size.
-        // https://github.com/risingwavelabs/risingwave/issues/9713
-        self.kv_heap_size.size()
-    }
-}
-
-#[derive(Error, Debug)]
-pub enum JoinEntryError {
-    #[error("double inserting a join state entry")]
-    OccupiedError,
-    #[error("removing a join state entry but it is not in the cache")]
-    RemoveError,
-}
-
-impl JoinEntryState {
-    /// Insert into the cache.
-    pub fn insert(
-        &mut self,
-        key: PkType,
-        value: StateValueType,
-    ) -> Result<&mut StateValueType, JoinEntryError> {
-        let mut removed = false;
-        if !enable_strict_consistency() {
-            // strict consistency is off, let's remove existing (if any) first
-            if let Some(old_value) = self.cached.remove(&key) {
-                self.kv_heap_size.sub(&key, &old_value);
-                removed = true;
-            }
-        }
-
-        self.kv_heap_size.add(&key, &value);
-
-        let ret = self.cached.try_insert(key.clone(), value);
-
-        if !enable_strict_consistency() {
-            assert!(ret.is_ok(), "we have removed existing entry, if any");
-            if removed {
-                // if not silent, we should log the error
-                consistency_error!(?key, "double inserting a join state entry");
-            }
-        }
-
-        ret.map_err(|_| JoinEntryError::OccupiedError)
-    }
-
-    /// Delete from the cache.
-    pub fn remove(&mut self, pk: PkType) -> Result<(), JoinEntryError> {
-        if let Some(value) = self.cached.remove(&pk) {
-            self.kv_heap_size.sub(&pk, &value);
-            Ok(())
-        } else if enable_strict_consistency() {
-            Err(JoinEntryError::RemoveError)
-        } else {
-            consistency_error!(?pk, "removing a join state entry but it's not in the cache");
-            Ok(())
-        }
-    }
-
-    /// Note: the first item in the tuple is the mutable reference to the value in this entry, while
-    /// the second item is the decoded value. To mutate the degree, one **must not** forget to apply
-    /// the changes to the first item.
-    ///
-    /// WARNING: Should not change the heap size of `StateValueType` with the mutable reference.
-    pub fn values_mut<'a>(
-        &'a mut self,
-        data_types: &'a [DataType],
-    ) -> impl Iterator<
-        Item = (
-            &'a mut StateValueType,
-            StreamExecutorResult<JoinRow<OwnedRow>>,
-        ),
-    > + 'a {
-        self.cached.values_mut().map(|encoded| {
-            let decoded = encoded.decode(data_types);
-            (encoded, decoded)
-        })
-    }
-
-    pub fn len(&self) -> usize {
-        self.cached.len()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use itertools::Itertools;
-    use risingwave_common::array::*;
-    use risingwave_common::types::ScalarImpl;
-    use risingwave_common::util::iter_util::ZipEqDebug;
-
-    use super::*;
-
-    fn insert_chunk(
-        managed_state: &mut JoinEntryState,
-        pk_indices: &[usize],
-        data_chunk: &DataChunk,
-    ) {
-        for row_ref in data_chunk.rows() {
-            let row: OwnedRow = row_ref.into_owned_row();
-            let value_indices = (0..row.len() - 1).collect_vec();
-            let pk = pk_indices.iter().map(|idx| row[*idx].clone()).collect_vec();
-            // Pk is only a `i64` here, so encoding method does not matter.
-            let pk = OwnedRow::new(pk).project(&value_indices).value_serialize();
-            let join_row = JoinRow { row, degree: 0 };
-            managed_state.insert(pk, join_row.encode()).unwrap();
-        }
-    }
-
-    fn check(
-        managed_state: &mut JoinEntryState,
-        col_types: &[DataType],
-        col1: &[i64],
-        col2: &[i64],
-    ) {
-        for ((_, matched_row), (d1, d2)) in managed_state
-            .values_mut(col_types)
-            .zip_eq_debug(col1.iter().zip_eq_debug(col2.iter()))
-        {
-            let matched_row = matched_row.unwrap();
-            assert_eq!(matched_row.row[0], Some(ScalarImpl::Int64(*d1)));
-            assert_eq!(matched_row.row[1], Some(ScalarImpl::Int64(*d2)));
-            assert_eq!(matched_row.degree, 0);
-        }
-    }
-
-    #[tokio::test]
-    async fn test_managed_all_or_none_state() {
-        let mut managed_state = JoinEntryState::default();
-        let col_types = vec![DataType::Int64, DataType::Int64];
-        let pk_indices = [0];
-
-        let col1 = [3, 2, 1];
-        let col2 = [4, 5, 6];
-        let data_chunk1 = DataChunk::from_pretty(
-            "I I
-             3 4
-             2 5
-             1 6",
-        );
-
-        // `Vec` in state
-        insert_chunk(&mut managed_state, &pk_indices, &data_chunk1);
-        check(&mut managed_state, &col_types, &col1, &col2);
-
-        // `BtreeMap` in state
-        let col1 = [1, 2, 3, 4, 5];
-        let col2 = [6, 5, 4, 9, 8];
-        let data_chunk2 = DataChunk::from_pretty(
-            "I I
-             5 8
-             4 9",
-        );
-        insert_chunk(&mut managed_state, &pk_indices, &data_chunk2);
-        check(&mut managed_state, &col_types, &col1, &col2);
-    }
-}
diff --git a/src/stream/src/executor/managed_state/join/join_row_set.rs b/src/stream/src/executor/managed_state/join/join_row_set.rs
deleted file mode 100644
index de6f5ce2f0279..0000000000000
--- a/src/stream/src/executor/managed_state/join/join_row_set.rs
+++ /dev/null
@@ -1,120 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::btree_map::OccupiedError as BTreeMapOccupiedError;
-use std::collections::BTreeMap;
-use std::fmt::Debug;
-use std::mem;
-
-use auto_enums::auto_enum;
-use enum_as_inner::EnumAsInner;
-
-const MAX_VEC_SIZE: usize = 4;
-
-#[derive(Debug, EnumAsInner)]
-pub enum JoinRowSet<K, V> {
-    BTree(BTreeMap<K, V>),
-    Vec(Vec<(K, V)>),
-}
-
-impl<K, V> Default for JoinRowSet<K, V> {
-    fn default() -> Self {
-        Self::Vec(Vec::new())
-    }
-}
-
-#[derive(Debug)]
-#[allow(dead_code)]
-pub struct VecOccupiedError<'a, K, V> {
-    key: &'a K,
-    old_value: &'a V,
-    new_value: V,
-}
-
-#[derive(Debug)]
-pub enum JoinRowSetOccupiedError<'a, K: Ord, V> {
-    BTree(BTreeMapOccupiedError<'a, K, V>),
-    Vec(VecOccupiedError<'a, K, V>),
-}
-
-impl<K: Ord, V> JoinRowSet<K, V> {
-    pub fn try_insert(
-        &mut self,
-        key: K,
-        value: V,
-    ) -> Result<&'_ mut V, JoinRowSetOccupiedError<'_, K, V>> {
-        if let Self::Vec(inner) = self
-            && inner.len() >= MAX_VEC_SIZE
-        {
-            let btree = BTreeMap::from_iter(inner.drain(..));
-            mem::swap(self, &mut Self::BTree(btree));
-        }
-
-        match self {
-            Self::BTree(inner) => inner
-                .try_insert(key, value)
-                .map_err(JoinRowSetOccupiedError::BTree),
-            Self::Vec(inner) => {
-                if let Some(pos) = inner.iter().position(|elem| elem.0 == key) {
-                    Err(JoinRowSetOccupiedError::Vec(VecOccupiedError {
-                        key: &inner[pos].0,
-                        old_value: &inner[pos].1,
-                        new_value: value,
-                    }))
-                } else {
-                    if inner.capacity() == 0 {
-                        // `Vec` will give capacity 4 when `1 < mem::size_of::<T> <= 1024`
-                        // We only give one for memory optimization
-                        inner.reserve_exact(1);
-                    }
-                    inner.push((key, value));
-                    Ok(&mut inner.last_mut().unwrap().1)
-                }
-            }
-        }
-    }
-
-    pub fn remove(&mut self, key: &K) -> Option<V> {
-        let ret = match self {
-            Self::BTree(inner) => inner.remove(key),
-            Self::Vec(inner) => inner
-                .iter()
-                .position(|elem| &elem.0 == key)
-                .map(|pos| inner.swap_remove(pos).1),
-        };
-        if let Self::BTree(inner) = self
-            && inner.len() <= MAX_VEC_SIZE / 2
-        {
-            let btree = mem::take(inner);
-            let vec = Vec::from_iter(btree);
-            mem::swap(self, &mut Self::Vec(vec));
-        }
-        ret
-    }
-
-    pub fn len(&self) -> usize {
-        match self {
-            Self::BTree(inner) => inner.len(),
-            Self::Vec(inner) => inner.len(),
-        }
-    }
-
-    #[auto_enum(Iterator)]
-    pub fn values_mut(&mut self) -> impl Iterator<Item = &'_ mut V> {
-        match self {
-            Self::BTree(inner) => inner.values_mut(),
-            Self::Vec(inner) => inner.iter_mut().map(|(_, v)| v),
-        }
-    }
-}
diff --git a/src/stream/src/executor/managed_state/join/mod.rs b/src/stream/src/executor/managed_state/join/mod.rs
deleted file mode 100644
index dced3899c404b..0000000000000
--- a/src/stream/src/executor/managed_state/join/mod.rs
+++ /dev/null
@@ -1,694 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-mod join_entry_state;
-mod join_row_set;
-
-use std::alloc::Global;
-use std::ops::{Bound, Deref, DerefMut};
-use std::sync::Arc;
-
-use anyhow::Context;
-use futures::future::{join, try_join};
-use futures::StreamExt;
-use futures_async_stream::for_await;
-pub(super) use join_entry_state::JoinEntryState;
-use local_stats_alloc::{SharedStatsAlloc, StatsAlloc};
-use risingwave_common::buffer::Bitmap;
-use risingwave_common::hash::{HashKey, PrecomputedBuildHasher};
-use risingwave_common::metrics::LabelGuardedIntCounter;
-use risingwave_common::row;
-use risingwave_common::row::{CompactedRow, OwnedRow, Row, RowExt};
-use risingwave_common::types::{DataType, ScalarImpl};
-use risingwave_common::util::epoch::EpochPair;
-use risingwave_common::util::row_serde::OrderedRowSerde;
-use risingwave_common::util::sort_util::OrderType;
-use risingwave_common_estimate_size::EstimateSize;
-use risingwave_storage::store::PrefetchOptions;
-use risingwave_storage::StateStore;
-
-use crate::cache::{new_with_hasher_in, ManagedLruCache};
-use crate::common::metrics::MetricsInfo;
-use crate::common::table::state_table::StateTable;
-use crate::executor::error::StreamExecutorResult;
-use crate::executor::monitor::StreamingMetrics;
-use crate::task::{ActorId, AtomicU64Ref, FragmentId};
-
-type DegreeType = u64;
-
-fn build_degree_row(order_key: impl Row, degree: DegreeType) -> impl Row {
-    order_key.chain(row::once(Some(ScalarImpl::Int64(degree as i64))))
-}
-
-/// This is a row with a match degree
-#[derive(Clone, Debug)]
-pub struct JoinRow<R: Row> {
-    pub row: R,
-    degree: DegreeType,
-}
-
-impl<R: Row> JoinRow<R> {
-    pub fn new(row: R, degree: DegreeType) -> Self {
-        Self { row, degree }
-    }
-
-    pub fn is_zero_degree(&self) -> bool {
-        self.degree == 0
-    }
-
-    /// Return row and degree in `Row` format. The degree part will be inserted in degree table
-    /// later, so a pk prefix will be added.
-    ///
-    /// * `state_order_key_indices` - the order key of `row`
-    pub fn to_table_rows<'a>(
-        &'a self,
-        state_order_key_indices: &'a [usize],
-    ) -> (&'a R, impl Row + 'a) {
-        let order_key = (&self.row).project(state_order_key_indices);
-        let degree = build_degree_row(order_key, self.degree);
-        (&self.row, degree)
-    }
-
-    pub fn encode(&self) -> EncodedJoinRow {
-        EncodedJoinRow {
-            compacted_row: (&self.row).into(),
-            degree: self.degree,
-        }
-    }
-}
-
-#[derive(Clone, Debug, EstimateSize)]
-pub struct EncodedJoinRow {
-    pub compacted_row: CompactedRow,
-    degree: DegreeType,
-}
-
-impl EncodedJoinRow {
-    fn decode(&self, data_types: &[DataType]) -> StreamExecutorResult<JoinRow<OwnedRow>> {
-        Ok(JoinRow {
-            row: self.decode_row(data_types)?,
-            degree: self.degree,
-        })
-    }
-
-    fn decode_row(&self, data_types: &[DataType]) -> StreamExecutorResult<OwnedRow> {
-        let row = self.compacted_row.deserialize(data_types)?;
-        Ok(row)
-    }
-}
-
-/// Memcomparable encoding.
-type PkType = Vec<u8>;
-
-pub type StateValueType = EncodedJoinRow;
-pub type HashValueType = Box<JoinEntryState>;
-
-impl EstimateSize for HashValueType {
-    fn estimated_heap_size(&self) -> usize {
-        self.as_ref().estimated_heap_size()
-    }
-}
-
-/// The wrapper for [`JoinEntryState`] which should be `Some` most of the time in the hash table.
-///
-/// When the executor is operating on the specific entry of the map, it can hold the ownership of
-/// the entry by taking the value out of the `Option`, instead of holding a mutable reference to the
-/// map, which can make the compiler happy.
-struct HashValueWrapper(Option<HashValueType>);
-
-impl EstimateSize for HashValueWrapper {
-    fn estimated_heap_size(&self) -> usize {
-        self.0.estimated_heap_size()
-    }
-}
-
-impl HashValueWrapper {
-    const MESSAGE: &'static str = "the state should always be `Some`";
-
-    /// Take the value out of the wrapper. Panic if the value is `None`.
-    pub fn take(&mut self) -> HashValueType {
-        self.0.take().expect(Self::MESSAGE)
-    }
-}
-
-impl Deref for HashValueWrapper {
-    type Target = HashValueType;
-
-    fn deref(&self) -> &Self::Target {
-        self.0.as_ref().expect(Self::MESSAGE)
-    }
-}
-
-impl DerefMut for HashValueWrapper {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self.0.as_mut().expect(Self::MESSAGE)
-    }
-}
-
-type JoinHashMapInner<K> =
-    ManagedLruCache<K, HashValueWrapper, PrecomputedBuildHasher, SharedStatsAlloc<Global>>;
-
-pub struct JoinHashMapMetrics {
-    /// Basic information
-    /// How many times have we hit the cache of join executor
-    lookup_miss_count: usize,
-    total_lookup_count: usize,
-    /// How many times have we miss the cache when insert row
-    insert_cache_miss_count: usize,
-
-    // Metrics
-    join_lookup_total_count_metric: LabelGuardedIntCounter<5>,
-    join_lookup_miss_count_metric: LabelGuardedIntCounter<5>,
-    join_insert_cache_miss_count_metrics: LabelGuardedIntCounter<5>,
-}
-
-impl JoinHashMapMetrics {
-    pub fn new(
-        metrics: &StreamingMetrics,
-        actor_id: ActorId,
-        fragment_id: FragmentId,
-        side: &'static str,
-        join_table_id: u32,
-        degree_table_id: u32,
-    ) -> Self {
-        let actor_id = actor_id.to_string();
-        let fragment_id = fragment_id.to_string();
-        let join_table_id = join_table_id.to_string();
-        let degree_table_id = degree_table_id.to_string();
-        let join_lookup_total_count_metric =
-            metrics.join_lookup_total_count.with_guarded_label_values(&[
-                (side),
-                &join_table_id,
-                &degree_table_id,
-                &actor_id,
-                &fragment_id,
-            ]);
-        let join_lookup_miss_count_metric =
-            metrics.join_lookup_miss_count.with_guarded_label_values(&[
-                (side),
-                &join_table_id,
-                &degree_table_id,
-                &actor_id,
-                &fragment_id,
-            ]);
-        let join_insert_cache_miss_count_metrics = metrics
-            .join_insert_cache_miss_count
-            .with_guarded_label_values(&[
-                (side),
-                &join_table_id,
-                &degree_table_id,
-                &actor_id,
-                &fragment_id,
-            ]);
-
-        Self {
-            lookup_miss_count: 0,
-            total_lookup_count: 0,
-            insert_cache_miss_count: 0,
-            join_lookup_total_count_metric,
-            join_lookup_miss_count_metric,
-            join_insert_cache_miss_count_metrics,
-        }
-    }
-
-    pub fn flush(&mut self) {
-        self.join_lookup_total_count_metric
-            .inc_by(self.total_lookup_count as u64);
-        self.join_lookup_miss_count_metric
-            .inc_by(self.lookup_miss_count as u64);
-        self.join_insert_cache_miss_count_metrics
-            .inc_by(self.insert_cache_miss_count as u64);
-        self.total_lookup_count = 0;
-        self.lookup_miss_count = 0;
-        self.insert_cache_miss_count = 0;
-    }
-}
-
-pub struct JoinHashMap<K: HashKey, S: StateStore> {
-    /// Store the join states.
-    inner: JoinHashMapInner<K>,
-    /// Data types of the join key columns
-    join_key_data_types: Vec<DataType>,
-    /// Null safe bitmap for each join pair
-    null_matched: K::Bitmap,
-    /// The memcomparable serializer of primary key.
-    pk_serializer: OrderedRowSerde,
-    /// State table. Contains the data from upstream.
-    state: TableInner<S>,
-    /// Degree table.
-    ///
-    /// The degree is generated from the hash join executor.
-    /// Each row in `state` has a corresponding degree in `degree state`.
-    /// A degree value `d` in for a row means the row has `d` matched row in the other join side.
-    ///
-    /// It will only be used when needed in a side.
-    ///
-    /// - Full Outer: both side
-    /// - Left Outer/Semi/Anti: left side
-    /// - Right Outer/Semi/Anti: right side
-    /// - Inner: None.
-    degree_state: TableInner<S>,
-    /// If degree table is need
-    need_degree_table: bool,
-    /// Pk is part of the join key.
-    pk_contained_in_jk: bool,
-    /// Metrics of the hash map
-    metrics: JoinHashMapMetrics,
-}
-
-struct TableInner<S: StateStore> {
-    /// Indices of the (cache) pk in a state row
-    pk_indices: Vec<usize>,
-    /// Indices of the join key in a state row
-    join_key_indices: Vec<usize>,
-    // This should be identical to the pk in state table.
-    order_key_indices: Vec<usize>,
-    // This should be identical to the data types in table schema.
-    #[expect(dead_code)]
-    all_data_types: Vec<DataType>,
-    pub(crate) table: StateTable<S>,
-}
-
-impl<S: StateStore> TableInner<S> {
-    fn error_context(&self, row: &impl Row) -> String {
-        let pk = row.project(&self.pk_indices);
-        let jk = row.project(&self.join_key_indices);
-        format!(
-            "join key: {}, pk: {}, row: {}, state_table_id: {}",
-            jk.display(),
-            pk.display(),
-            row.display(),
-            self.table.table_id()
-        )
-    }
-}
-
-impl<K: HashKey, S: StateStore> JoinHashMap<K, S> {
-    /// Create a [`JoinHashMap`] with the given LRU capacity.
-    #[allow(clippy::too_many_arguments)]
-    pub fn new(
-        watermark_epoch: AtomicU64Ref,
-        join_key_data_types: Vec<DataType>,
-        state_join_key_indices: Vec<usize>,
-        state_all_data_types: Vec<DataType>,
-        state_table: StateTable<S>,
-        state_pk_indices: Vec<usize>,
-        degree_join_key_indices: Vec<usize>,
-        degree_all_data_types: Vec<DataType>,
-        degree_table: StateTable<S>,
-        degree_pk_indices: Vec<usize>,
-        null_matched: K::Bitmap,
-        need_degree_table: bool,
-        pk_contained_in_jk: bool,
-        metrics: Arc<StreamingMetrics>,
-        actor_id: ActorId,
-        fragment_id: FragmentId,
-        side: &'static str,
-    ) -> Self {
-        let alloc = StatsAlloc::new(Global).shared();
-        // TODO: unify pk encoding with state table.
-        let pk_data_types = state_pk_indices
-            .iter()
-            .map(|i| state_all_data_types[*i].clone())
-            .collect();
-        let pk_serializer = OrderedRowSerde::new(
-            pk_data_types,
-            vec![OrderType::ascending(); state_pk_indices.len()],
-        );
-
-        let join_table_id = state_table.table_id();
-        let degree_table_id = degree_table.table_id();
-        let state = TableInner {
-            pk_indices: state_pk_indices,
-            join_key_indices: state_join_key_indices,
-            order_key_indices: state_table.pk_indices().to_vec(),
-            all_data_types: state_all_data_types,
-            table: state_table,
-        };
-
-        let degree_state = TableInner {
-            pk_indices: degree_pk_indices,
-            join_key_indices: degree_join_key_indices,
-            order_key_indices: degree_table.pk_indices().to_vec(),
-            all_data_types: degree_all_data_types,
-            table: degree_table,
-        };
-
-        let metrics_info = MetricsInfo::new(
-            metrics.clone(),
-            join_table_id,
-            actor_id,
-            &format!("hash join {}", side),
-        );
-
-        let cache =
-            new_with_hasher_in(watermark_epoch, metrics_info, PrecomputedBuildHasher, alloc);
-
-        Self {
-            inner: cache,
-            join_key_data_types,
-            null_matched,
-            pk_serializer,
-            state,
-            degree_state,
-            need_degree_table,
-            pk_contained_in_jk,
-            metrics: JoinHashMapMetrics::new(
-                &metrics,
-                actor_id,
-                fragment_id,
-                side,
-                join_table_id,
-                degree_table_id,
-            ),
-        }
-    }
-
-    pub fn init(&mut self, epoch: EpochPair) {
-        self.update_epoch(epoch.curr);
-        self.state.table.init_epoch(epoch);
-        self.degree_state.table.init_epoch(epoch);
-    }
-
-    pub fn update_epoch(&mut self, epoch: u64) {
-        // Update the current epoch in `ManagedLruCache`
-        self.inner.update_epoch(epoch)
-    }
-
-    /// Update the vnode bitmap and manipulate the cache if necessary.
-    pub fn update_vnode_bitmap(&mut self, vnode_bitmap: Arc<Bitmap>) -> bool {
-        let (_previous_vnode_bitmap, cache_may_stale) =
-            self.state.table.update_vnode_bitmap(vnode_bitmap.clone());
-        let _ = self.degree_state.table.update_vnode_bitmap(vnode_bitmap);
-
-        if cache_may_stale {
-            self.inner.clear();
-        }
-
-        cache_may_stale
-    }
-
-    pub fn update_watermark(&mut self, watermark: ScalarImpl) {
-        // TODO: remove data in cache.
-        self.state.table.update_watermark(watermark.clone(), false);
-        self.degree_state.table.update_watermark(watermark, false);
-    }
-
-    /// Take the state for the given `key` out of the hash table and return it. One **MUST** call
-    /// `update_state` after some operations to put the state back.
-    ///
-    /// If the state does not exist in the cache, fetch the remote storage and return. If it still
-    /// does not exist in the remote storage, a [`JoinEntryState`] with empty cache will be
-    /// returned.
-    ///
-    /// Note: This will NOT remove anything from remote storage.
-    pub async fn take_state<'a>(&mut self, key: &K) -> StreamExecutorResult<HashValueType> {
-        self.metrics.total_lookup_count += 1;
-        let state = if self.inner.contains(key) {
-            // Do not update the LRU statistics here with `peek_mut` since we will put the state
-            // back.
-            let mut state = self.inner.peek_mut(key).unwrap();
-            state.take()
-        } else {
-            self.metrics.lookup_miss_count += 1;
-            self.fetch_cached_state(key).await?.into()
-        };
-        Ok(state)
-    }
-
-    /// Fetch cache from the state store. Should only be called if the key does not exist in memory.
-    /// Will return a empty `JoinEntryState` even when state does not exist in remote.
-    async fn fetch_cached_state(&self, key: &K) -> StreamExecutorResult<JoinEntryState> {
-        let key = key.deserialize(&self.join_key_data_types)?;
-
-        let mut entry_state = JoinEntryState::default();
-
-        if self.need_degree_table {
-            let sub_range: &(Bound<OwnedRow>, Bound<OwnedRow>) =
-                &(Bound::Unbounded, Bound::Unbounded);
-            let table_iter_fut =
-                self.state
-                    .table
-                    .iter_with_prefix(&key, sub_range, PrefetchOptions::default());
-            let degree_table_iter_fut = self.degree_state.table.iter_with_prefix(
-                &key,
-                sub_range,
-                PrefetchOptions::default(),
-            );
-
-            let (table_iter, degree_table_iter) =
-                try_join(table_iter_fut, degree_table_iter_fut).await?;
-
-            let mut pinned_table_iter = std::pin::pin!(table_iter);
-            let mut pinned_degree_table_iter = std::pin::pin!(degree_table_iter);
-            loop {
-                // Iterate on both iterators and ensure they have same size. Basically `zip_eq()`.
-                let (row, degree) =
-                    join(pinned_table_iter.next(), pinned_degree_table_iter.next()).await;
-                let (row, degree) = match (row, degree) {
-                    (None, None) => break,
-                    (None, Some(_)) | (Some(_), None) => {
-                        panic!("mismatched row and degree table of join key: {:?}", &key)
-                    }
-                    (Some(r), Some(d)) => (r, d),
-                };
-
-                let row = row?;
-                let degree_row = degree?;
-                let pk1 = row.key();
-                let pk2 = degree_row.key();
-                debug_assert_eq!(
-                    pk1, pk2,
-                    "mismatched pk in degree table: pk1: {pk1:?}, pk2: {pk2:?}",
-                );
-                let pk = row
-                    .as_ref()
-                    .project(&self.state.pk_indices)
-                    .memcmp_serialize(&self.pk_serializer);
-                let degree_i64 = degree_row
-                    .datum_at(degree_row.len() - 1)
-                    .expect("degree should not be NULL");
-                entry_state
-                    .insert(
-                        pk,
-                        JoinRow::new(row.row(), degree_i64.into_int64() as u64).encode(),
-                    )
-                    .with_context(|| self.state.error_context(row.row()))?;
-            }
-        } else {
-            let sub_range: &(Bound<OwnedRow>, Bound<OwnedRow>) =
-                &(Bound::Unbounded, Bound::Unbounded);
-            let table_iter = self
-                .state
-                .table
-                .iter_with_prefix(&key, sub_range, PrefetchOptions::default())
-                .await?;
-
-            #[for_await]
-            for entry in table_iter {
-                let row = entry?;
-                let pk = row
-                    .as_ref()
-                    .project(&self.state.pk_indices)
-                    .memcmp_serialize(&self.pk_serializer);
-                entry_state
-                    .insert(pk, JoinRow::new(row.row(), 0).encode())
-                    .with_context(|| self.state.error_context(row.row()))?;
-            }
-        };
-
-        Ok(entry_state)
-    }
-
-    pub async fn flush(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> {
-        self.metrics.flush();
-        self.state.table.commit(epoch).await?;
-        self.degree_state.table.commit(epoch).await?;
-        Ok(())
-    }
-
-    pub async fn try_flush(&mut self) -> StreamExecutorResult<()> {
-        self.state.table.try_flush().await?;
-        self.degree_state.table.try_flush().await?;
-        Ok(())
-    }
-
-    /// Insert a join row
-    #[allow(clippy::unused_async)]
-    pub async fn insert(&mut self, key: &K, value: JoinRow<impl Row>) -> StreamExecutorResult<()> {
-        let pk = (&value.row)
-            .project(&self.state.pk_indices)
-            .memcmp_serialize(&self.pk_serializer);
-
-        // TODO(yuhao): avoid this `contains`.
-        // https://github.com/risingwavelabs/risingwave/issues/9233
-        if self.inner.contains(key) {
-            // Update cache
-            let mut entry = self.inner.get_mut(key).unwrap();
-            entry
-                .insert(pk, value.encode())
-                .with_context(|| self.state.error_context(&value.row))?;
-        } else if self.pk_contained_in_jk {
-            // Refill cache when the join key exist in neither cache or storage.
-            self.metrics.insert_cache_miss_count += 1;
-            let mut state = JoinEntryState::default();
-            state
-                .insert(pk, value.encode())
-                .with_context(|| self.state.error_context(&value.row))?;
-            self.update_state(key, state.into());
-        }
-
-        // Update the flush buffer.
-        let (row, degree) = value.to_table_rows(&self.state.order_key_indices);
-        self.state.table.insert(row);
-        self.degree_state.table.insert(degree);
-        Ok(())
-    }
-
-    /// Insert a row.
-    /// Used when the side does not need to update degree.
-    #[allow(clippy::unused_async)]
-    pub async fn insert_row(&mut self, key: &K, value: impl Row) -> StreamExecutorResult<()> {
-        let join_row = JoinRow::new(&value, 0);
-        let pk = (&value)
-            .project(&self.state.pk_indices)
-            .memcmp_serialize(&self.pk_serializer);
-
-        // TODO(yuhao): avoid this `contains`.
-        // https://github.com/risingwavelabs/risingwave/issues/9233
-        if self.inner.contains(key) {
-            // Update cache
-            let mut entry = self.inner.get_mut(key).unwrap();
-            entry
-                .insert(pk, join_row.encode())
-                .with_context(|| self.state.error_context(&value))?;
-        } else if self.pk_contained_in_jk {
-            // Refill cache when the join key exist in neither cache or storage.
-            self.metrics.insert_cache_miss_count += 1;
-            let mut state = JoinEntryState::default();
-            state
-                .insert(pk, join_row.encode())
-                .with_context(|| self.state.error_context(&value))?;
-            self.update_state(key, state.into());
-        }
-
-        // Update the flush buffer.
-        self.state.table.insert(value);
-        Ok(())
-    }
-
-    /// Delete a join row
-    pub fn delete(&mut self, key: &K, value: JoinRow<impl Row>) -> StreamExecutorResult<()> {
-        if let Some(mut entry) = self.inner.get_mut(key) {
-            let pk = (&value.row)
-                .project(&self.state.pk_indices)
-                .memcmp_serialize(&self.pk_serializer);
-            entry
-                .remove(pk)
-                .with_context(|| self.state.error_context(&value.row))?;
-        }
-
-        // If no cache maintained, only update the state table.
-        let (row, degree) = value.to_table_rows(&self.state.order_key_indices);
-        self.state.table.delete(row);
-        self.degree_state.table.delete(degree);
-        Ok(())
-    }
-
-    /// Delete a row
-    /// Used when the side does not need to update degree.
-    pub fn delete_row(&mut self, key: &K, value: impl Row) -> StreamExecutorResult<()> {
-        if let Some(mut entry) = self.inner.get_mut(key) {
-            let pk = (&value)
-                .project(&self.state.pk_indices)
-                .memcmp_serialize(&self.pk_serializer);
-            entry
-                .remove(pk)
-                .with_context(|| self.state.error_context(&value))?;
-        }
-
-        // If no cache maintained, only update the state table.
-        self.state.table.delete(value);
-        Ok(())
-    }
-
-    /// Update a [`JoinEntryState`] into the hash table.
-    pub fn update_state(&mut self, key: &K, state: HashValueType) {
-        self.inner.put(key.clone(), HashValueWrapper(Some(state)));
-    }
-
-    /// Manipulate the degree of the given [`JoinRow`] and [`EncodedJoinRow`] with `action`, both in
-    /// memory and in the degree table.
-    fn manipulate_degree(
-        &mut self,
-        join_row_ref: &mut StateValueType,
-        join_row: &mut JoinRow<OwnedRow>,
-        action: impl Fn(&mut DegreeType),
-    ) {
-        // TODO: no need to `into_owned_row` here due to partial borrow.
-        let old_degree = join_row
-            .to_table_rows(&self.state.order_key_indices)
-            .1
-            .into_owned_row();
-
-        action(&mut join_row_ref.degree);
-        action(&mut join_row.degree);
-
-        let new_degree = join_row.to_table_rows(&self.state.order_key_indices).1;
-
-        self.degree_state.table.update(old_degree, new_degree);
-    }
-
-    /// Increment the degree of the given [`JoinRow`] and [`EncodedJoinRow`] with `action`, both in
-    /// memory and in the degree table.
-    pub fn inc_degree(
-        &mut self,
-        join_row_ref: &mut StateValueType,
-        join_row: &mut JoinRow<OwnedRow>,
-    ) {
-        self.manipulate_degree(join_row_ref, join_row, |d| *d += 1)
-    }
-
-    /// Decrement the degree of the given [`JoinRow`] and [`EncodedJoinRow`] with `action`, both in
-    /// memory and in the degree table.
-    pub fn dec_degree(
-        &mut self,
-        join_row_ref: &mut StateValueType,
-        join_row: &mut JoinRow<OwnedRow>,
-    ) {
-        self.manipulate_degree(join_row_ref, join_row, |d| {
-            *d = d
-                .checked_sub(1)
-                .expect("Tried to decrement zero join row degree")
-        })
-    }
-
-    /// Evict the cache.
-    pub fn evict(&mut self) {
-        self.inner.evict();
-    }
-
-    /// Cached entry count for this hash table.
-    pub fn entry_count(&self) -> usize {
-        self.inner.len()
-    }
-
-    pub fn null_matched(&self) -> &K::Bitmap {
-        &self.null_matched
-    }
-
-    pub fn table_id(&self) -> u32 {
-        self.state.table.table_id()
-    }
-}
diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs
index 7a5450716f9ac..19124fe8c22d4 100644
--- a/src/stream/src/executor/merge.rs
+++ b/src/stream/src/executor/merge.rs
@@ -18,7 +18,9 @@ use std::task::{Context, Poll};
 
 use anyhow::Context as _;
 use futures::stream::{FusedStream, FuturesUnordered, StreamFuture};
-use futures::StreamExt;
+use prometheus::Histogram;
+use risingwave_common::config::MetricLevel;
+use risingwave_common::metrics::LabelGuardedMetric;
 use tokio::time::Instant;
 
 use super::exchange::input::BoxedInput;
@@ -26,8 +28,7 @@ use super::watermark::*;
 use super::*;
 use crate::executor::exchange::input::new_input;
 use crate::executor::prelude::*;
-use crate::executor::utils::ActorInputMetrics;
-use crate::task::{FragmentId, SharedContext};
+use crate::task::SharedContext;
 
 /// `MergeExecutor` merges data from multiple channels. Dataflow from one channel
 /// will be stopped on barrier.
@@ -94,12 +95,28 @@ impl MergeExecutor {
 
     #[try_stream(ok = Message, error = StreamExecutorError)]
     async fn execute_inner(mut self: Box<Self>) {
+        let merge_barrier_align_duration = if self.metrics.level >= MetricLevel::Debug {
+            Some(
+                self.metrics
+                    .merge_barrier_align_duration
+                    .with_label_values(&[
+                        &self.actor_context.id.to_string(),
+                        &self.actor_context.fragment_id.to_string(),
+                    ]),
+            )
+        } else {
+            None
+        };
+
         // Futures of all active upstreams.
-        let select_all = SelectReceivers::new(self.actor_context.id, self.upstreams);
+        let select_all = SelectReceivers::new(
+            self.actor_context.id,
+            self.upstreams,
+            merge_barrier_align_duration.clone(),
+        );
         let actor_id = self.actor_context.id;
 
-        let mut metrics = ActorInputMetrics::new(
-            &self.metrics,
+        let mut metrics = self.metrics.new_actor_input_metrics(
             actor_id,
             self.fragment_id,
             self.upstream_fragment_id,
@@ -185,8 +202,11 @@ impl MergeExecutor {
 
                             // Poll the first barrier from the new upstreams. It must be the same as
                             // the one we polled from original upstreams.
-                            let mut select_new =
-                                SelectReceivers::new(self.actor_context.id, new_upstreams);
+                            let mut select_new = SelectReceivers::new(
+                                self.actor_context.id,
+                                new_upstreams,
+                                merge_barrier_align_duration.clone(),
+                            );
                             let new_barrier = expect_first_barrier(&mut select_new).await?;
                             assert_eq!(barrier, &new_barrier);
 
@@ -214,8 +234,7 @@ impl MergeExecutor {
                         }
 
                         self.upstream_fragment_id = new_upstream_fragment_id;
-                        metrics = ActorInputMetrics::new(
-                            &self.metrics,
+                        metrics = self.metrics.new_actor_input_metrics(
                             actor_id,
                             self.fragment_id,
                             self.upstream_fragment_id,
@@ -253,6 +272,8 @@ pub struct SelectReceivers {
     actor_id: u32,
     /// watermark column index -> `BufferedWatermarks`
     buffered_watermarks: BTreeMap<usize, BufferedWatermarks<ActorId>>,
+    /// If None, then we don't take `Instant::now()` and `observe` during `poll_next`
+    merge_barrier_align_duration: Option<LabelGuardedMetric<Histogram, 2>>,
 }
 
 impl Stream for SelectReceivers {
@@ -265,6 +286,7 @@ impl Stream for SelectReceivers {
             return Poll::Ready(None);
         }
 
+        let mut start = None;
         loop {
             match futures::ready!(self.active.poll_next_unpin(cx)) {
                 // Directly forward the error.
@@ -289,6 +311,11 @@ impl Stream for SelectReceivers {
                         }
                         Message::Barrier(barrier) => {
                             // Block this upstream by pushing it to `blocked`.
+                            if self.blocked.is_empty()
+                                && self.merge_barrier_align_duration.is_some()
+                            {
+                                start = Some(Instant::now());
+                            }
                             self.blocked.push(remaining);
                             if let Some(current_barrier) = self.barrier.as_ref() {
                                 if current_barrier.epoch != barrier.epoch {
@@ -314,7 +341,16 @@ impl Stream for SelectReceivers {
                 // So this branch will never be reached in all cases.
                 Some((None, _)) => unreachable!(),
                 // There's no active upstreams. Process the barrier and resume the blocked ones.
-                None => break,
+                None => {
+                    if let Some(start) = start
+                        && let Some(merge_barrier_align_duration) =
+                            &self.merge_barrier_align_duration
+                    {
+                        // Observe did a few atomic operation inside, we want to avoid the overhead.
+                        merge_barrier_align_duration.observe(start.elapsed().as_secs_f64())
+                    }
+                    break;
+                }
             }
         }
 
@@ -336,7 +372,11 @@ impl Stream for SelectReceivers {
 }
 
 impl SelectReceivers {
-    fn new(actor_id: u32, upstreams: Vec<BoxedInput>) -> Self {
+    fn new(
+        actor_id: u32,
+        upstreams: Vec<BoxedInput>,
+        merge_barrier_align_duration: Option<LabelGuardedMetric<Histogram, 2>>,
+    ) -> Self {
         assert!(!upstreams.is_empty());
         let upstream_actor_ids = upstreams.iter().map(|input| input.actor_id()).collect();
         let mut this = Self {
@@ -346,6 +386,7 @@ impl SelectReceivers {
             barrier: None,
             upstream_actor_ids,
             buffered_watermarks: Default::default(),
+            merge_barrier_align_duration,
         };
         this.extend_active(upstreams);
         this
@@ -413,16 +454,12 @@ impl SelectReceivers {
 
 #[cfg(test)]
 mod tests {
-    use std::collections::HashSet;
     use std::sync::atomic::{AtomicBool, Ordering};
-    use std::sync::Arc;
     use std::time::Duration;
 
     use assert_matches::assert_matches;
     use futures::FutureExt;
-    use itertools::Itertools;
-    use risingwave_common::array::{Op, StreamChunk};
-    use risingwave_common::types::ScalarImpl;
+    use risingwave_common::array::Op;
     use risingwave_common::util::epoch::test_epoch;
     use risingwave_pb::stream_plan::StreamMessage;
     use risingwave_pb::task_service::exchange_service_server::{
@@ -439,7 +476,6 @@ mod tests {
     use super::*;
     use crate::executor::exchange::input::RemoteInput;
     use crate::executor::exchange::permit::channel_for_test;
-    use crate::executor::{Barrier, Execute, Mutation};
     use crate::task::test_utils::helper_make_local_actor;
     use crate::task::LocalBarrierManager;
 
diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs
index 0748fac0a6569..6d5c5867060de 100644
--- a/src/stream/src/executor/mod.rs
+++ b/src/stream/src/executor/mod.rs
@@ -25,7 +25,7 @@ use futures::{Stream, StreamExt};
 use itertools::Itertools;
 use risingwave_common::array::StreamChunk;
 use risingwave_common::buffer::Bitmap;
-use risingwave_common::catalog::Schema;
+use risingwave_common::catalog::{Schema, TableId};
 use risingwave_common::row::OwnedRow;
 use risingwave_common::types::{DataType, Datum, DefaultOrd, ScalarImpl};
 use risingwave_common::util::epoch::{Epoch, EpochPair};
@@ -36,13 +36,14 @@ use risingwave_expr::expr::{Expression, NonStrictExpression};
 use risingwave_pb::data::PbEpoch;
 use risingwave_pb::expr::PbInputRef;
 use risingwave_pb::stream_plan::barrier::BarrierKind;
-use risingwave_pb::stream_plan::barrier_mutation::PbMutation;
+use risingwave_pb::stream_plan::barrier_mutation::Mutation as PbMutation;
 use risingwave_pb::stream_plan::stream_message::StreamMessage;
 use risingwave_pb::stream_plan::update_mutation::{DispatcherUpdate, MergeUpdate};
 use risingwave_pb::stream_plan::{
-    BarrierMutation, CombinedMutation, Dispatchers, PauseMutation, PbAddMutation, PbBarrier,
-    PbDispatcher, PbStreamMessage, PbUpdateMutation, PbWatermark, ResumeMutation,
-    SourceChangeSplitMutation, StopMutation, ThrottleMutation,
+    BarrierMutation, CombinedMutation, CreateSubscriptionMutation, Dispatchers,
+    DropSubscriptionMutation, PauseMutation, PbAddMutation, PbBarrier, PbDispatcher,
+    PbStreamMessage, PbUpdateMutation, PbWatermark, ResumeMutation, SourceChangeSplitMutation,
+    StopMutation, ThrottleMutation,
 };
 use smallvec::SmallVec;
 
@@ -90,7 +91,6 @@ mod sort_buffer;
 pub mod source;
 mod stateless_simple_agg;
 mod stream_reader;
-mod subscription;
 pub mod subtask;
 mod temporal_join;
 mod top_n;
@@ -109,7 +109,7 @@ mod utils;
 pub use actor::{Actor, ActorContext, ActorContextRef};
 use anyhow::Context;
 pub use backfill::arrangement_backfill::*;
-pub use backfill::cdc::{CdcBackfillExecutor, ExternalStorageTable};
+pub use backfill::cdc::{CdcBackfillExecutor, CdcScanOptions, ExternalStorageTable};
 pub use backfill::no_shuffle_backfill::*;
 pub use barrier_recv::BarrierRecvExecutor;
 pub use batch_query::BatchQueryExecutor;
@@ -140,7 +140,6 @@ pub use simple_agg::SimpleAggExecutor;
 pub use sink::SinkExecutor;
 pub use sort::*;
 pub use stateless_simple_agg::StatelessSimpleAggExecutor;
-pub use subscription::SubscriptionExecutor;
 pub use temporal_join::*;
 pub use top_n::{
     AppendOnlyGroupTopNExecutor, AppendOnlyTopNExecutor, GroupTopNExecutor, TopNExecutor,
@@ -285,6 +284,14 @@ pub enum Mutation {
     Resume,
     Throttle(HashMap<ActorId, Option<u32>>),
     AddAndUpdate(AddMutation, UpdateMutation),
+    CreateSubscription {
+        subscription_id: u32,
+        upstream_mv_table_id: TableId,
+    },
+    DropSubscription {
+        subscription_id: u32,
+        upstream_mv_table_id: TableId,
+    },
 }
 
 #[derive(Debug, Clone)]
@@ -373,6 +380,54 @@ impl Barrier {
         }
     }
 
+    /// Whether this barrier adds new downstream fragment for the actor with `upstream_actor_id`.
+    ///
+    /// # Use case
+    /// Some optimizations are applied when an actor doesn't have any downstreams ("standalone" actors).
+    /// * Pause a standalone shared `SourceExecutor`.
+    /// * Disable a standalone `MaterializeExecutor`'s conflict check.
+    ///
+    /// This is implemented by checking `actor_context.initial_dispatch_num` on startup, and
+    /// check `has_more_downstream_fragments` on barrier to see whether the optimization
+    /// needs to be turned off.
+    ///
+    /// ## Some special cases not included
+    ///
+    /// Note that this is not `has_new_downstream_actor/fragment`. For our use case, we only
+    /// care about **number of downstream fragments** (more precisely, existence).
+    /// - When scaling, the number of downstream actors is changed, and they are "new", but downstream fragments is not changed.
+    /// - When `ALTER TABLE sink_into_table`, the fragment is replaced with a "new" one, but the number is not changed.
+    pub fn has_more_downstream_fragments(&self, upstream_actor_id: ActorId) -> bool {
+        let Some(mutation) = self.mutation.as_deref() else {
+            return false;
+        };
+        match mutation {
+            // Add is for mv, index and sink creation.
+            Mutation::Add(AddMutation { adds, .. }) => adds.get(&upstream_actor_id).is_some(),
+            // AddAndUpdate is for sink-into-table.
+            Mutation::AddAndUpdate(
+                AddMutation { adds, .. },
+                UpdateMutation {
+                    dispatchers,
+                    actor_new_dispatchers,
+                    ..
+                },
+            ) => {
+                adds.get(&upstream_actor_id).is_some()
+                    || actor_new_dispatchers.get(&upstream_actor_id).is_some()
+                    || dispatchers.get(&upstream_actor_id).is_some()
+            }
+            Mutation::Update(_)
+            | Mutation::Stop(_)
+            | Mutation::Pause
+            | Mutation::Resume
+            | Mutation::SourceChangeSplit(_)
+            | Mutation::Throttle(_)
+            | Mutation::CreateSubscription { .. }
+            | Mutation::DropSubscription { .. } => false,
+        }
+    }
+
     /// Whether this barrier requires the executor to pause its data stream on startup.
     pub fn is_pause_on_startup(&self) -> bool {
         match self.mutation.as_deref() {
@@ -550,6 +605,20 @@ impl Mutation {
                     },
                 ],
             }),
+            Mutation::CreateSubscription {
+                upstream_mv_table_id,
+                subscription_id,
+            } => PbMutation::CreateSubscription(CreateSubscriptionMutation {
+                upstream_mv_table_id: upstream_mv_table_id.table_id,
+                subscription_id: *subscription_id,
+            }),
+            Mutation::DropSubscription {
+                upstream_mv_table_id,
+                subscription_id,
+            } => PbMutation::DropSubscription(DropSubscriptionMutation {
+                upstream_mv_table_id: upstream_mv_table_id.table_id,
+                subscription_id: *subscription_id,
+            }),
         }
     }
 
@@ -647,7 +716,14 @@ impl Mutation {
                     .map(|(actor_id, limit)| (*actor_id, limit.rate_limit))
                     .collect(),
             ),
-
+            PbMutation::CreateSubscription(create) => Mutation::CreateSubscription {
+                upstream_mv_table_id: TableId::new(create.upstream_mv_table_id),
+                subscription_id: create.subscription_id,
+            },
+            PbMutation::DropSubscription(drop) => Mutation::DropSubscription {
+                upstream_mv_table_id: TableId::new(drop.upstream_mv_table_id),
+                subscription_id: drop.subscription_id,
+            },
             PbMutation::Combined(CombinedMutation { mutations }) => match &mutations[..] {
                 [BarrierMutation {
                     mutation: Some(add),
diff --git a/src/stream/src/executor/monitor/streaming_stats.rs b/src/stream/src/executor/monitor/streaming_stats.rs
index 5aba22aa4edc6..379bdf49a5722 100644
--- a/src/stream/src/executor/monitor/streaming_stats.rs
+++ b/src/stream/src/executor/monitor/streaming_stats.rs
@@ -14,126 +14,137 @@
 
 use std::sync::OnceLock;
 
-use prometheus::core::{AtomicF64, AtomicI64, AtomicU64, GenericCounterVec, GenericGaugeVec};
 use prometheus::{
-    exponential_buckets, histogram_opts, register_gauge_vec_with_registry,
-    register_histogram_with_registry, register_int_counter_vec_with_registry,
-    register_int_counter_with_registry, register_int_gauge_vec_with_registry,
-    register_int_gauge_with_registry, Histogram, IntCounter, IntCounterVec, IntGauge, Registry,
+    exponential_buckets, histogram_opts, register_histogram_with_registry,
+    register_int_counter_with_registry, register_int_gauge_with_registry, Histogram, IntCounter,
+    IntGauge, Registry,
 };
+use risingwave_common::catalog::TableId;
 use risingwave_common::config::MetricLevel;
 use risingwave_common::metrics::{
-    LabelGuardedHistogramVec, LabelGuardedIntCounterVec, LabelGuardedIntGaugeVec,
+    LabelGuardedGauge, LabelGuardedGaugeVec, LabelGuardedHistogramVec, LabelGuardedIntCounter,
+    LabelGuardedIntCounterVec, LabelGuardedIntGauge, LabelGuardedIntGaugeVec,
     RelabeledGuardedHistogramVec,
 };
 use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY;
 use risingwave_common::{
-    register_guarded_histogram_vec_with_registry, register_guarded_int_counter_vec_with_registry,
-    register_guarded_int_gauge_vec_with_registry,
+    register_guarded_gauge_vec_with_registry, register_guarded_histogram_vec_with_registry,
+    register_guarded_int_counter_vec_with_registry, register_guarded_int_gauge_vec_with_registry,
 };
+use risingwave_connector::sink::catalog::SinkId;
 use risingwave_connector::sink::SinkMetrics;
 
 use crate::common::log_store_impl::kv_log_store::{
     REWIND_BACKOFF_FACTOR, REWIND_BASE_DELAY, REWIND_MAX_DELAY,
 };
+use crate::executor::prelude::ActorId;
+use crate::task::FragmentId;
 
 #[derive(Clone)]
 pub struct StreamingMetrics {
     pub level: MetricLevel,
 
     // Executor metrics (disabled by default)
-    pub executor_row_count: GenericCounterVec<AtomicU64>,
+    pub executor_row_count: LabelGuardedIntCounterVec<3>,
 
     // Streaming actor metrics from tokio (disabled by default)
-    pub actor_execution_time: GenericGaugeVec<AtomicF64>,
-    pub actor_scheduled_duration: GenericGaugeVec<AtomicF64>,
-    pub actor_scheduled_cnt: GenericGaugeVec<AtomicI64>,
-    pub actor_fast_poll_duration: GenericGaugeVec<AtomicF64>,
-    pub actor_fast_poll_cnt: GenericGaugeVec<AtomicI64>,
-    pub actor_slow_poll_duration: GenericGaugeVec<AtomicF64>,
-    pub actor_slow_poll_cnt: GenericGaugeVec<AtomicI64>,
-    pub actor_poll_duration: GenericGaugeVec<AtomicF64>,
-    pub actor_poll_cnt: GenericGaugeVec<AtomicI64>,
-    pub actor_idle_duration: GenericGaugeVec<AtomicF64>,
-    pub actor_idle_cnt: GenericGaugeVec<AtomicI64>,
+    actor_execution_time: LabelGuardedGaugeVec<1>,
+    actor_scheduled_duration: LabelGuardedGaugeVec<1>,
+    actor_scheduled_cnt: LabelGuardedIntGaugeVec<1>,
+    actor_fast_poll_duration: LabelGuardedGaugeVec<1>,
+    actor_fast_poll_cnt: LabelGuardedIntGaugeVec<1>,
+    actor_slow_poll_duration: LabelGuardedGaugeVec<1>,
+    actor_slow_poll_cnt: LabelGuardedIntGaugeVec<1>,
+    actor_poll_duration: LabelGuardedGaugeVec<1>,
+    actor_poll_cnt: LabelGuardedIntGaugeVec<1>,
+    actor_idle_duration: LabelGuardedGaugeVec<1>,
+    actor_idle_cnt: LabelGuardedIntGaugeVec<1>,
 
     // Streaming actor
-    pub actor_memory_usage: GenericGaugeVec<AtomicI64>,
-    pub actor_in_record_cnt: LabelGuardedIntCounterVec<3>,
+    #[expect(dead_code)]
+    actor_memory_usage: LabelGuardedIntGaugeVec<2>,
+    actor_in_record_cnt: LabelGuardedIntCounterVec<3>,
     pub actor_out_record_cnt: LabelGuardedIntCounterVec<2>,
 
     // Source
-    pub source_output_row_count: GenericCounterVec<AtomicU64>,
-    pub source_split_change_count: GenericCounterVec<AtomicU64>,
+    pub source_output_row_count: LabelGuardedIntCounterVec<4>,
+    pub source_split_change_count: LabelGuardedIntCounterVec<4>,
     pub source_backfill_row_count: LabelGuardedIntCounterVec<4>,
 
-    // Sink & materialized view
-    pub sink_input_row_count: LabelGuardedIntCounterVec<3>,
-    pub mview_input_row_count: IntCounterVec,
+    // Sink
+    sink_input_row_count: LabelGuardedIntCounterVec<3>,
+    sink_chunk_buffer_size: LabelGuardedIntGaugeVec<3>,
 
     // Exchange (see also `compute::ExchangeServiceMetrics`)
-    pub exchange_frag_recv_size: GenericCounterVec<AtomicU64>,
+    pub exchange_frag_recv_size: LabelGuardedIntCounterVec<2>,
+
+    // Streaming Merge (We break out this metric from `barrier_align_duration` because
+    // the alignment happens on different levels)
+    pub merge_barrier_align_duration: RelabeledGuardedHistogramVec<2>,
 
     // Backpressure
     pub actor_output_buffer_blocking_duration_ns: LabelGuardedIntCounterVec<3>,
-    pub actor_input_buffer_blocking_duration_ns: LabelGuardedIntCounterVec<3>,
+    actor_input_buffer_blocking_duration_ns: LabelGuardedIntCounterVec<3>,
 
     // Streaming Join
-    pub join_lookup_miss_count: LabelGuardedIntCounterVec<5>,
-    pub join_lookup_total_count: LabelGuardedIntCounterVec<5>,
-    pub join_insert_cache_miss_count: LabelGuardedIntCounterVec<5>,
+    pub join_lookup_miss_count: LabelGuardedIntCounterVec<4>,
+    pub join_lookup_total_count: LabelGuardedIntCounterVec<4>,
+    pub join_insert_cache_miss_count: LabelGuardedIntCounterVec<4>,
     pub join_actor_input_waiting_duration_ns: LabelGuardedIntCounterVec<2>,
     pub join_match_duration_ns: LabelGuardedIntCounterVec<3>,
-    pub join_barrier_align_duration: RelabeledGuardedHistogramVec<3>,
     pub join_cached_entry_count: LabelGuardedIntGaugeVec<3>,
     pub join_matched_join_keys: RelabeledGuardedHistogramVec<3>,
 
+    // Streaming Join, Streaming Dynamic Filter and Streaming Union
+    pub barrier_align_duration: RelabeledGuardedHistogramVec<4>,
+
     // Streaming Aggregation
-    pub agg_lookup_miss_count: GenericCounterVec<AtomicU64>,
-    pub agg_total_lookup_count: GenericCounterVec<AtomicU64>,
-    pub agg_cached_entry_count: GenericGaugeVec<AtomicI64>,
-    pub agg_chunk_lookup_miss_count: GenericCounterVec<AtomicU64>,
-    pub agg_chunk_total_lookup_count: GenericCounterVec<AtomicU64>,
-    pub agg_distinct_cache_miss_count: GenericCounterVec<AtomicU64>,
-    pub agg_distinct_total_cache_count: GenericCounterVec<AtomicU64>,
-    pub agg_distinct_cached_entry_count: GenericGaugeVec<AtomicI64>,
-    pub agg_dirty_groups_count: GenericGaugeVec<AtomicI64>,
-    pub agg_dirty_groups_heap_size: GenericGaugeVec<AtomicI64>,
+    agg_lookup_miss_count: LabelGuardedIntCounterVec<3>,
+    agg_total_lookup_count: LabelGuardedIntCounterVec<3>,
+    agg_cached_entry_count: LabelGuardedIntGaugeVec<3>,
+    agg_chunk_lookup_miss_count: LabelGuardedIntCounterVec<3>,
+    agg_chunk_total_lookup_count: LabelGuardedIntCounterVec<3>,
+    agg_dirty_groups_count: LabelGuardedIntGaugeVec<3>,
+    agg_dirty_groups_heap_size: LabelGuardedIntGaugeVec<3>,
+    agg_distinct_cache_miss_count: LabelGuardedIntCounterVec<3>,
+    agg_distinct_total_cache_count: LabelGuardedIntCounterVec<3>,
+    agg_distinct_cached_entry_count: LabelGuardedIntGaugeVec<3>,
 
     // Streaming TopN
-    pub group_top_n_cache_miss_count: GenericCounterVec<AtomicU64>,
-    pub group_top_n_total_query_cache_count: GenericCounterVec<AtomicU64>,
-    pub group_top_n_cached_entry_count: GenericGaugeVec<AtomicI64>,
-    pub group_top_n_appendonly_cache_miss_count: GenericCounterVec<AtomicU64>,
-    pub group_top_n_appendonly_total_query_cache_count: GenericCounterVec<AtomicU64>,
-    pub group_top_n_appendonly_cached_entry_count: GenericGaugeVec<AtomicI64>,
+    group_top_n_cache_miss_count: LabelGuardedIntCounterVec<3>,
+    group_top_n_total_query_cache_count: LabelGuardedIntCounterVec<3>,
+    group_top_n_cached_entry_count: LabelGuardedIntGaugeVec<3>,
+    // TODO(rc): why not just use the above three?
+    group_top_n_appendonly_cache_miss_count: LabelGuardedIntCounterVec<3>,
+    group_top_n_appendonly_total_query_cache_count: LabelGuardedIntCounterVec<3>,
+    group_top_n_appendonly_cached_entry_count: LabelGuardedIntGaugeVec<3>,
 
     // Lookup executor
-    pub lookup_cache_miss_count: GenericCounterVec<AtomicU64>,
-    pub lookup_total_query_cache_count: GenericCounterVec<AtomicU64>,
-    pub lookup_cached_entry_count: GenericGaugeVec<AtomicI64>,
+    lookup_cache_miss_count: LabelGuardedIntCounterVec<3>,
+    lookup_total_query_cache_count: LabelGuardedIntCounterVec<3>,
+    lookup_cached_entry_count: LabelGuardedIntGaugeVec<3>,
 
     // temporal join
-    pub temporal_join_cache_miss_count: GenericCounterVec<AtomicU64>,
-    pub temporal_join_total_query_cache_count: GenericCounterVec<AtomicU64>,
-    pub temporal_join_cached_entry_count: GenericGaugeVec<AtomicI64>,
+    temporal_join_cache_miss_count: LabelGuardedIntCounterVec<3>,
+    temporal_join_total_query_cache_count: LabelGuardedIntCounterVec<3>,
+    temporal_join_cached_entry_count: LabelGuardedIntGaugeVec<3>,
 
     // Backfill
-    pub backfill_snapshot_read_row_count: LabelGuardedIntCounterVec<2>,
-    pub backfill_upstream_output_row_count: LabelGuardedIntCounterVec<2>,
+    backfill_snapshot_read_row_count: LabelGuardedIntCounterVec<2>,
+    backfill_upstream_output_row_count: LabelGuardedIntCounterVec<2>,
 
     // CDC Backfill
-    pub cdc_backfill_snapshot_read_row_count: GenericCounterVec<AtomicU64>,
-    pub cdc_backfill_upstream_output_row_count: GenericCounterVec<AtomicU64>,
+    cdc_backfill_snapshot_read_row_count: LabelGuardedIntCounterVec<2>,
+    cdc_backfill_upstream_output_row_count: LabelGuardedIntCounterVec<2>,
 
     // Over Window
-    pub over_window_cached_entry_count: GenericGaugeVec<AtomicI64>,
-    pub over_window_cache_lookup_count: GenericCounterVec<AtomicU64>,
-    pub over_window_cache_miss_count: GenericCounterVec<AtomicU64>,
-    pub over_window_range_cache_entry_count: GenericGaugeVec<AtomicI64>,
-    pub over_window_range_cache_lookup_count: GenericCounterVec<AtomicU64>,
-    pub over_window_range_cache_left_miss_count: GenericCounterVec<AtomicU64>,
-    pub over_window_range_cache_right_miss_count: GenericCounterVec<AtomicU64>,
+    over_window_cached_entry_count: LabelGuardedIntGaugeVec<3>,
+    over_window_cache_lookup_count: LabelGuardedIntCounterVec<3>,
+    over_window_cache_miss_count: LabelGuardedIntCounterVec<3>,
+    over_window_range_cache_entry_count: LabelGuardedIntGaugeVec<3>,
+    over_window_range_cache_lookup_count: LabelGuardedIntCounterVec<3>,
+    over_window_range_cache_left_miss_count: LabelGuardedIntCounterVec<3>,
+    over_window_range_cache_right_miss_count: LabelGuardedIntCounterVec<3>,
 
     /// The duration from receipt of barrier to all actors collection.
     /// And the max of all node `barrier_inflight_latency` is the latency for a barrier
@@ -145,47 +156,49 @@ pub struct StreamingMetrics {
     pub barrier_manager_progress: IntCounter,
 
     // Sink related metrics
-    pub sink_commit_duration: LabelGuardedHistogramVec<3>,
-    pub connector_sink_rows_received: LabelGuardedIntCounterVec<2>,
-    pub log_store_first_write_epoch: LabelGuardedIntGaugeVec<3>,
-    pub log_store_latest_write_epoch: LabelGuardedIntGaugeVec<3>,
-    pub log_store_write_rows: LabelGuardedIntCounterVec<3>,
-    pub log_store_latest_read_epoch: LabelGuardedIntGaugeVec<3>,
-    pub log_store_read_rows: LabelGuardedIntCounterVec<3>,
+    sink_commit_duration: LabelGuardedHistogramVec<3>,
+    connector_sink_rows_received: LabelGuardedIntCounterVec<2>,
+    log_store_first_write_epoch: LabelGuardedIntGaugeVec<3>,
+    log_store_latest_write_epoch: LabelGuardedIntGaugeVec<3>,
+    log_store_write_rows: LabelGuardedIntCounterVec<3>,
+    log_store_latest_read_epoch: LabelGuardedIntGaugeVec<3>,
+    log_store_read_rows: LabelGuardedIntCounterVec<3>,
+    log_store_reader_wait_new_future_duration_ns: LabelGuardedIntCounterVec<3>,
     pub kv_log_store_storage_write_count: LabelGuardedIntCounterVec<3>,
     pub kv_log_store_storage_write_size: LabelGuardedIntCounterVec<3>,
     pub kv_log_store_rewind_count: LabelGuardedIntCounterVec<3>,
     pub kv_log_store_rewind_delay: LabelGuardedHistogramVec<3>,
     pub kv_log_store_storage_read_count: LabelGuardedIntCounterVec<4>,
     pub kv_log_store_storage_read_size: LabelGuardedIntCounterVec<4>,
+    pub kv_log_store_buffer_unconsumed_item_count: LabelGuardedIntGaugeVec<3>,
+    pub kv_log_store_buffer_unconsumed_row_count: LabelGuardedIntGaugeVec<3>,
+    pub kv_log_store_buffer_unconsumed_epoch_count: LabelGuardedIntGaugeVec<3>,
+    pub kv_log_store_buffer_unconsumed_min_epoch: LabelGuardedIntGaugeVec<3>,
 
     // Sink iceberg metrics
-    pub iceberg_write_qps: LabelGuardedIntCounterVec<2>,
-    pub iceberg_write_latency: LabelGuardedHistogramVec<2>,
-    pub iceberg_rolling_unflushed_data_file: LabelGuardedIntGaugeVec<2>,
-    pub iceberg_position_delete_cache_num: LabelGuardedIntGaugeVec<2>,
-    pub iceberg_partition_num: LabelGuardedIntGaugeVec<2>,
+    iceberg_write_qps: LabelGuardedIntCounterVec<2>,
+    iceberg_write_latency: LabelGuardedHistogramVec<2>,
+    iceberg_rolling_unflushed_data_file: LabelGuardedIntGaugeVec<2>,
+    iceberg_position_delete_cache_num: LabelGuardedIntGaugeVec<2>,
+    iceberg_partition_num: LabelGuardedIntGaugeVec<2>,
 
     // Memory management
-    // FIXME(yuhao): use u64 here
-    pub lru_current_watermark_time_ms: IntGauge,
-    pub lru_physical_now_ms: IntGauge,
     pub lru_runtime_loop_count: IntCounter,
-    pub lru_watermark_step: IntGauge,
-    pub lru_evicted_watermark_time_ms: LabelGuardedIntGaugeVec<3>,
+    pub lru_latest_sequence: IntGauge,
+    pub lru_watermark_sequence: IntGauge,
+    pub lru_eviction_policy: IntGauge,
     pub jemalloc_allocated_bytes: IntGauge,
     pub jemalloc_active_bytes: IntGauge,
     pub jemalloc_resident_bytes: IntGauge,
     pub jemalloc_metadata_bytes: IntGauge,
     pub jvm_allocated_bytes: IntGauge,
     pub jvm_active_bytes: IntGauge,
-
-    // Materialize
-    pub materialize_cache_hit_count: GenericCounterVec<AtomicU64>,
-    pub materialize_cache_total_count: GenericCounterVec<AtomicU64>,
-
-    // Memory
     pub stream_memory_usage: LabelGuardedIntGaugeVec<3>,
+
+    // Materialized view
+    materialize_cache_hit_count: LabelGuardedIntCounterVec<3>,
+    materialize_cache_total_count: LabelGuardedIntCounterVec<3>,
+    materialize_input_row_count: LabelGuardedIntCounterVec<3>,
 }
 
 pub static GLOBAL_STREAMING_METRICS: OnceLock<StreamingMetrics> = OnceLock::new();
@@ -198,7 +211,7 @@ pub fn global_streaming_metrics(metric_level: MetricLevel) -> StreamingMetrics {
 
 impl StreamingMetrics {
     fn new(registry: &Registry, level: MetricLevel) -> Self {
-        let executor_row_count = register_int_counter_vec_with_registry!(
+        let executor_row_count = register_guarded_int_counter_vec_with_registry!(
             "stream_executor_row_count",
             "Total number of rows that have been output from each executor",
             &["actor_id", "fragment_id", "executor_identity"],
@@ -206,7 +219,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let source_output_row_count = register_int_counter_vec_with_registry!(
+        let source_output_row_count = register_guarded_int_counter_vec_with_registry!(
             "stream_source_output_rows_counts",
             "Total number of rows that have been output from source",
             &["source_id", "source_name", "actor_id", "fragment_id"],
@@ -214,7 +227,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let source_split_change_count = register_int_counter_vec_with_registry!(
+        let source_split_change_count = register_guarded_int_counter_vec_with_registry!(
             "stream_source_split_change_event_count",
             "Total number of split change events that have been operated by source",
             &["source_id", "source_name", "actor_id", "fragment_id"],
@@ -238,7 +251,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let mview_input_row_count = register_int_counter_vec_with_registry!(
+        let materialize_input_row_count = register_guarded_int_counter_vec_with_registry!(
             "stream_mview_input_row_count",
             "Total number of rows streamed into materialize executors",
             &["table_id", "actor_id", "fragment_id"],
@@ -246,7 +259,15 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let actor_execution_time = register_gauge_vec_with_registry!(
+        let sink_chunk_buffer_size = register_guarded_int_gauge_vec_with_registry!(
+            "stream_sink_chunk_buffer_size",
+            "Total size of chunks buffered in a barrier",
+            &["sink_id", "actor_id", "fragment_id"],
+            registry
+        )
+        .unwrap();
+
+        let actor_execution_time = register_guarded_gauge_vec_with_registry!(
             "stream_actor_actor_execution_time",
             "Total execution time (s) of an actor",
             &["actor_id"],
@@ -272,7 +293,7 @@ impl StreamingMetrics {
             )
             .unwrap();
 
-        let exchange_frag_recv_size = register_int_counter_vec_with_registry!(
+        let exchange_frag_recv_size = register_guarded_int_counter_vec_with_registry!(
             "stream_exchange_frag_recv_size",
             "Total size of messages that have been received from upstream Fragment",
             &["up_fragment_id", "down_fragment_id"],
@@ -280,7 +301,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let actor_fast_poll_duration = register_gauge_vec_with_registry!(
+        let actor_fast_poll_duration = register_guarded_gauge_vec_with_registry!(
             "stream_actor_fast_poll_duration",
             "tokio's metrics",
             &["actor_id"],
@@ -288,7 +309,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let actor_fast_poll_cnt = register_int_gauge_vec_with_registry!(
+        let actor_fast_poll_cnt = register_guarded_int_gauge_vec_with_registry!(
             "stream_actor_fast_poll_cnt",
             "tokio's metrics",
             &["actor_id"],
@@ -296,7 +317,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let actor_slow_poll_duration = register_gauge_vec_with_registry!(
+        let actor_slow_poll_duration = register_guarded_gauge_vec_with_registry!(
             "stream_actor_slow_poll_duration",
             "tokio's metrics",
             &["actor_id"],
@@ -304,7 +325,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let actor_slow_poll_cnt = register_int_gauge_vec_with_registry!(
+        let actor_slow_poll_cnt = register_guarded_int_gauge_vec_with_registry!(
             "stream_actor_slow_poll_cnt",
             "tokio's metrics",
             &["actor_id"],
@@ -312,7 +333,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let actor_poll_duration = register_gauge_vec_with_registry!(
+        let actor_poll_duration = register_guarded_gauge_vec_with_registry!(
             "stream_actor_poll_duration",
             "tokio's metrics",
             &["actor_id"],
@@ -320,7 +341,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let actor_poll_cnt = register_int_gauge_vec_with_registry!(
+        let actor_poll_cnt = register_guarded_int_gauge_vec_with_registry!(
             "stream_actor_poll_cnt",
             "tokio's metrics",
             &["actor_id"],
@@ -328,7 +349,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let actor_scheduled_duration = register_gauge_vec_with_registry!(
+        let actor_scheduled_duration = register_guarded_gauge_vec_with_registry!(
             "stream_actor_scheduled_duration",
             "tokio's metrics",
             &["actor_id"],
@@ -336,7 +357,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let actor_scheduled_cnt = register_int_gauge_vec_with_registry!(
+        let actor_scheduled_cnt = register_guarded_int_gauge_vec_with_registry!(
             "stream_actor_scheduled_cnt",
             "tokio's metrics",
             &["actor_id"],
@@ -344,7 +365,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let actor_idle_duration = register_gauge_vec_with_registry!(
+        let actor_idle_duration = register_guarded_gauge_vec_with_registry!(
             "stream_actor_idle_duration",
             "tokio's metrics",
             &["actor_id"],
@@ -352,7 +373,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let actor_idle_cnt = register_int_gauge_vec_with_registry!(
+        let actor_idle_cnt = register_guarded_int_gauge_vec_with_registry!(
             "stream_actor_idle_cnt",
             "tokio's metrics",
             &["actor_id"],
@@ -376,7 +397,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let actor_memory_usage = register_int_gauge_vec_with_registry!(
+        let actor_memory_usage = register_guarded_int_gauge_vec_with_registry!(
             "actor_memory_usage",
             "Memory usage (bytes)",
             &["actor_id", "fragment_id"],
@@ -384,16 +405,30 @@ impl StreamingMetrics {
         )
         .unwrap();
 
+        let opts = histogram_opts!(
+            "stream_merge_barrier_align_duration",
+            "Duration of merge align barrier",
+            exponential_buckets(0.0001, 2.0, 21).unwrap() // max 104s
+        );
+        let merge_barrier_align_duration = register_guarded_histogram_vec_with_registry!(
+            opts,
+            &["actor_id", "fragment_id"],
+            registry
+        )
+        .unwrap();
+
+        let merge_barrier_align_duration =
+            RelabeledGuardedHistogramVec::with_metric_level_relabel_n(
+                MetricLevel::Debug,
+                merge_barrier_align_duration,
+                level,
+                1,
+            );
+
         let join_lookup_miss_count = register_guarded_int_counter_vec_with_registry!(
             "stream_join_lookup_miss_count",
             "Join executor lookup miss duration",
-            &[
-                "side",
-                "join_table_id",
-                "degree_table_id",
-                "actor_id",
-                "fragment_id"
-            ],
+            &["side", "join_table_id", "actor_id", "fragment_id"],
             registry
         )
         .unwrap();
@@ -401,13 +436,7 @@ impl StreamingMetrics {
         let join_lookup_total_count = register_guarded_int_counter_vec_with_registry!(
             "stream_join_lookup_total_count",
             "Join executor lookup total operation",
-            &[
-                "side",
-                "join_table_id",
-                "degree_table_id",
-                "actor_id",
-                "fragment_id"
-            ],
+            &["side", "join_table_id", "actor_id", "fragment_id"],
             registry
         )
         .unwrap();
@@ -415,13 +444,7 @@ impl StreamingMetrics {
         let join_insert_cache_miss_count = register_guarded_int_counter_vec_with_registry!(
             "stream_join_insert_cache_miss_count",
             "Join executor cache miss when insert operation",
-            &[
-                "side",
-                "join_table_id",
-                "degree_table_id",
-                "actor_id",
-                "fragment_id"
-            ],
+            &["side", "join_table_id", "actor_id", "fragment_id"],
             registry
         )
         .unwrap();
@@ -443,20 +466,20 @@ impl StreamingMetrics {
         .unwrap();
 
         let opts = histogram_opts!(
-            "stream_join_barrier_align_duration",
+            "stream_barrier_align_duration",
             "Duration of join align barrier",
             exponential_buckets(0.0001, 2.0, 21).unwrap() // max 104s
         );
-        let join_barrier_align_duration = register_guarded_histogram_vec_with_registry!(
+        let barrier_align_duration = register_guarded_histogram_vec_with_registry!(
             opts,
-            &["actor_id", "fragment_id", "wait_side"],
+            &["actor_id", "fragment_id", "wait_side", "executor"],
             registry
         )
         .unwrap();
 
-        let join_barrier_align_duration = RelabeledGuardedHistogramVec::with_metric_level_relabel_n(
+        let barrier_align_duration = RelabeledGuardedHistogramVec::with_metric_level_relabel_n(
             MetricLevel::Debug,
-            join_barrier_align_duration,
+            barrier_align_duration,
             level,
             1,
         );
@@ -489,7 +512,7 @@ impl StreamingMetrics {
             1,
         );
 
-        let agg_lookup_miss_count = register_int_counter_vec_with_registry!(
+        let agg_lookup_miss_count = register_guarded_int_counter_vec_with_registry!(
             "stream_agg_lookup_miss_count",
             "Aggregation executor lookup miss duration",
             &["table_id", "actor_id", "fragment_id"],
@@ -497,7 +520,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let agg_total_lookup_count = register_int_counter_vec_with_registry!(
+        let agg_total_lookup_count = register_guarded_int_counter_vec_with_registry!(
             "stream_agg_lookup_total_count",
             "Aggregation executor lookup total operation",
             &["table_id", "actor_id", "fragment_id"],
@@ -505,7 +528,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let agg_distinct_cache_miss_count = register_int_counter_vec_with_registry!(
+        let agg_distinct_cache_miss_count = register_guarded_int_counter_vec_with_registry!(
             "stream_agg_distinct_cache_miss_count",
             "Aggregation executor dinsinct miss duration",
             &["table_id", "actor_id", "fragment_id"],
@@ -513,7 +536,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let agg_distinct_total_cache_count = register_int_counter_vec_with_registry!(
+        let agg_distinct_total_cache_count = register_guarded_int_counter_vec_with_registry!(
             "stream_agg_distinct_total_cache_count",
             "Aggregation executor distinct total operation",
             &["table_id", "actor_id", "fragment_id"],
@@ -521,7 +544,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let agg_distinct_cached_entry_count = register_int_gauge_vec_with_registry!(
+        let agg_distinct_cached_entry_count = register_guarded_int_gauge_vec_with_registry!(
             "stream_agg_distinct_cached_entry_count",
             "Total entry counts in distinct aggregation executor cache",
             &["table_id", "actor_id", "fragment_id"],
@@ -529,7 +552,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let agg_dirty_groups_count = register_int_gauge_vec_with_registry!(
+        let agg_dirty_groups_count = register_guarded_int_gauge_vec_with_registry!(
             "stream_agg_dirty_groups_count",
             "Total dirty group counts in aggregation executor",
             &["table_id", "actor_id", "fragment_id"],
@@ -537,7 +560,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let agg_dirty_groups_heap_size = register_int_gauge_vec_with_registry!(
+        let agg_dirty_groups_heap_size = register_guarded_int_gauge_vec_with_registry!(
             "stream_agg_dirty_groups_heap_size",
             "Total dirty group heap size in aggregation executor",
             &["table_id", "actor_id", "fragment_id"],
@@ -545,7 +568,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let group_top_n_cache_miss_count = register_int_counter_vec_with_registry!(
+        let group_top_n_cache_miss_count = register_guarded_int_counter_vec_with_registry!(
             "stream_group_top_n_cache_miss_count",
             "Group top n executor cache miss count",
             &["table_id", "actor_id", "fragment_id"],
@@ -553,7 +576,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let group_top_n_total_query_cache_count = register_int_counter_vec_with_registry!(
+        let group_top_n_total_query_cache_count = register_guarded_int_counter_vec_with_registry!(
             "stream_group_top_n_total_query_cache_count",
             "Group top n executor query cache total count",
             &["table_id", "actor_id", "fragment_id"],
@@ -561,7 +584,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let group_top_n_cached_entry_count = register_int_gauge_vec_with_registry!(
+        let group_top_n_cached_entry_count = register_guarded_int_gauge_vec_with_registry!(
             "stream_group_top_n_cached_entry_count",
             "Total entry counts in group top n executor cache",
             &["table_id", "actor_id", "fragment_id"],
@@ -569,16 +592,17 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let group_top_n_appendonly_cache_miss_count = register_int_counter_vec_with_registry!(
-            "stream_group_top_n_appendonly_cache_miss_count",
-            "Group top n appendonly executor cache miss count",
-            &["table_id", "actor_id", "fragment_id"],
-            registry
-        )
-        .unwrap();
+        let group_top_n_appendonly_cache_miss_count =
+            register_guarded_int_counter_vec_with_registry!(
+                "stream_group_top_n_appendonly_cache_miss_count",
+                "Group top n appendonly executor cache miss count",
+                &["table_id", "actor_id", "fragment_id"],
+                registry
+            )
+            .unwrap();
 
         let group_top_n_appendonly_total_query_cache_count =
-            register_int_counter_vec_with_registry!(
+            register_guarded_int_counter_vec_with_registry!(
                 "stream_group_top_n_appendonly_total_query_cache_count",
                 "Group top n appendonly executor total cache count",
                 &["table_id", "actor_id", "fragment_id"],
@@ -586,15 +610,16 @@ impl StreamingMetrics {
             )
             .unwrap();
 
-        let group_top_n_appendonly_cached_entry_count = register_int_gauge_vec_with_registry!(
-            "stream_group_top_n_appendonly_cached_entry_count",
-            "Total entry counts in group top n appendonly executor cache",
-            &["table_id", "actor_id", "fragment_id"],
-            registry
-        )
-        .unwrap();
+        let group_top_n_appendonly_cached_entry_count =
+            register_guarded_int_gauge_vec_with_registry!(
+                "stream_group_top_n_appendonly_cached_entry_count",
+                "Total entry counts in group top n appendonly executor cache",
+                &["table_id", "actor_id", "fragment_id"],
+                registry
+            )
+            .unwrap();
 
-        let lookup_cache_miss_count = register_int_counter_vec_with_registry!(
+        let lookup_cache_miss_count = register_guarded_int_counter_vec_with_registry!(
             "stream_lookup_cache_miss_count",
             "Lookup executor cache miss count",
             &["table_id", "actor_id", "fragment_id"],
@@ -602,7 +627,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let lookup_total_query_cache_count = register_int_counter_vec_with_registry!(
+        let lookup_total_query_cache_count = register_guarded_int_counter_vec_with_registry!(
             "stream_lookup_total_query_cache_count",
             "Lookup executor query cache total count",
             &["table_id", "actor_id", "fragment_id"],
@@ -610,7 +635,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let lookup_cached_entry_count = register_int_gauge_vec_with_registry!(
+        let lookup_cached_entry_count = register_guarded_int_gauge_vec_with_registry!(
             "stream_lookup_cached_entry_count",
             "Total entry counts in lookup executor cache",
             &["table_id", "actor_id", "fragment_id"],
@@ -618,7 +643,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let temporal_join_cache_miss_count = register_int_counter_vec_with_registry!(
+        let temporal_join_cache_miss_count = register_guarded_int_counter_vec_with_registry!(
             "stream_temporal_join_cache_miss_count",
             "Temporal join executor cache miss count",
             &["table_id", "actor_id", "fragment_id"],
@@ -626,15 +651,16 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let temporal_join_total_query_cache_count = register_int_counter_vec_with_registry!(
-            "stream_temporal_join_total_query_cache_count",
-            "Temporal join executor query cache total count",
-            &["table_id", "actor_id", "fragment_id"],
-            registry
-        )
-        .unwrap();
+        let temporal_join_total_query_cache_count =
+            register_guarded_int_counter_vec_with_registry!(
+                "stream_temporal_join_total_query_cache_count",
+                "Temporal join executor query cache total count",
+                &["table_id", "actor_id", "fragment_id"],
+                registry
+            )
+            .unwrap();
 
-        let temporal_join_cached_entry_count = register_int_gauge_vec_with_registry!(
+        let temporal_join_cached_entry_count = register_guarded_int_gauge_vec_with_registry!(
             "stream_temporal_join_cached_entry_count",
             "Total entry count in temporal join executor cache",
             &["table_id", "actor_id", "fragment_id"],
@@ -642,7 +668,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let agg_cached_entry_count = register_int_gauge_vec_with_registry!(
+        let agg_cached_entry_count = register_guarded_int_gauge_vec_with_registry!(
             "stream_agg_cached_entry_count",
             "Number of cached keys in streaming aggregation operators",
             &["table_id", "actor_id", "fragment_id"],
@@ -650,7 +676,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let agg_chunk_lookup_miss_count = register_int_counter_vec_with_registry!(
+        let agg_chunk_lookup_miss_count = register_guarded_int_counter_vec_with_registry!(
             "stream_agg_chunk_lookup_miss_count",
             "Aggregation executor chunk-level lookup miss duration",
             &["table_id", "actor_id", "fragment_id"],
@@ -658,7 +684,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let agg_chunk_total_lookup_count = register_int_counter_vec_with_registry!(
+        let agg_chunk_total_lookup_count = register_guarded_int_counter_vec_with_registry!(
             "stream_agg_chunk_lookup_total_count",
             "Aggregation executor chunk-level lookup total operation",
             &["table_id", "actor_id", "fragment_id"],
@@ -682,7 +708,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let cdc_backfill_snapshot_read_row_count = register_int_counter_vec_with_registry!(
+        let cdc_backfill_snapshot_read_row_count = register_guarded_int_counter_vec_with_registry!(
             "stream_cdc_backfill_snapshot_read_row_count",
             "Total number of rows that have been read from the cdc_backfill snapshot",
             &["table_id", "actor_id"],
@@ -690,15 +716,16 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let cdc_backfill_upstream_output_row_count = register_int_counter_vec_with_registry!(
-            "stream_cdc_backfill_upstream_output_row_count",
-            "Total number of rows that have been output from the cdc_backfill upstream",
-            &["table_id", "actor_id"],
-            registry
-        )
-        .unwrap();
+        let cdc_backfill_upstream_output_row_count =
+            register_guarded_int_counter_vec_with_registry!(
+                "stream_cdc_backfill_upstream_output_row_count",
+                "Total number of rows that have been output from the cdc_backfill upstream",
+                &["table_id", "actor_id"],
+                registry
+            )
+            .unwrap();
 
-        let over_window_cached_entry_count = register_int_gauge_vec_with_registry!(
+        let over_window_cached_entry_count = register_guarded_int_gauge_vec_with_registry!(
             "stream_over_window_cached_entry_count",
             "Total entry (partition) count in over window executor cache",
             &["table_id", "actor_id", "fragment_id"],
@@ -706,7 +733,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let over_window_cache_lookup_count = register_int_counter_vec_with_registry!(
+        let over_window_cache_lookup_count = register_guarded_int_counter_vec_with_registry!(
             "stream_over_window_cache_lookup_count",
             "Over window executor cache lookup count",
             &["table_id", "actor_id", "fragment_id"],
@@ -714,7 +741,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let over_window_cache_miss_count = register_int_counter_vec_with_registry!(
+        let over_window_cache_miss_count = register_guarded_int_counter_vec_with_registry!(
             "stream_over_window_cache_miss_count",
             "Over window executor cache miss count",
             &["table_id", "actor_id", "fragment_id"],
@@ -722,7 +749,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let over_window_range_cache_entry_count = register_int_gauge_vec_with_registry!(
+        let over_window_range_cache_entry_count = register_guarded_int_gauge_vec_with_registry!(
             "stream_over_window_range_cache_entry_count",
             "Over window partition range cache entry count",
             &["table_id", "actor_id", "fragment_id"],
@@ -730,7 +757,7 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let over_window_range_cache_lookup_count = register_int_counter_vec_with_registry!(
+        let over_window_range_cache_lookup_count = register_guarded_int_counter_vec_with_registry!(
             "stream_over_window_range_cache_lookup_count",
             "Over window partition range cache lookup count",
             &["table_id", "actor_id", "fragment_id"],
@@ -738,21 +765,23 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let over_window_range_cache_left_miss_count = register_int_counter_vec_with_registry!(
-            "stream_over_window_range_cache_left_miss_count",
-            "Over window partition range cache left miss count",
-            &["table_id", "actor_id", "fragment_id"],
-            registry
-        )
-        .unwrap();
+        let over_window_range_cache_left_miss_count =
+            register_guarded_int_counter_vec_with_registry!(
+                "stream_over_window_range_cache_left_miss_count",
+                "Over window partition range cache left miss count",
+                &["table_id", "actor_id", "fragment_id"],
+                registry
+            )
+            .unwrap();
 
-        let over_window_range_cache_right_miss_count = register_int_counter_vec_with_registry!(
-            "stream_over_window_range_cache_right_miss_count",
-            "Over window partition range cache right miss count",
-            &["table_id", "actor_id", "fragment_id"],
-            registry
-        )
-        .unwrap();
+        let over_window_range_cache_right_miss_count =
+            register_guarded_int_counter_vec_with_registry!(
+                "stream_over_window_range_cache_right_miss_count",
+                "Over window partition range cache right miss count",
+                &["table_id", "actor_id", "fragment_id"],
+                registry
+            )
+            .unwrap();
 
         let opts = histogram_opts!(
             "stream_barrier_inflight_duration_seconds",
@@ -831,6 +860,15 @@ impl StreamingMetrics {
         )
         .unwrap();
 
+        let log_store_reader_wait_new_future_duration_ns =
+            register_guarded_int_counter_vec_with_registry!(
+                "log_store_reader_wait_new_future_duration_ns",
+                "Accumulated duration of LogReader to wait for next call to create future",
+                &["executor_id", "connector", "sink_id"],
+                registry
+            )
+            .unwrap();
+
         let kv_log_store_storage_write_count = register_guarded_int_counter_vec_with_registry!(
             "kv_log_store_storage_write_count",
             "Write row count throughput of kv log store",
@@ -896,19 +934,41 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let lru_current_watermark_time_ms = register_int_gauge_with_registry!(
-            "lru_current_watermark_time_ms",
-            "Current LRU manager watermark time(ms)",
-            registry
-        )
-        .unwrap();
+        let kv_log_store_buffer_unconsumed_item_count =
+            register_guarded_int_gauge_vec_with_registry!(
+                "kv_log_store_buffer_unconsumed_item_count",
+                "Number of Unconsumed Item in buffer",
+                &["executor_id", "connector", "sink_id"],
+                registry
+            )
+            .unwrap();
 
-        let lru_physical_now_ms = register_int_gauge_with_registry!(
-            "lru_physical_now_ms",
-            "Current physical time in Risingwave(ms)",
-            registry
-        )
-        .unwrap();
+        let kv_log_store_buffer_unconsumed_row_count =
+            register_guarded_int_gauge_vec_with_registry!(
+                "kv_log_store_buffer_unconsumed_row_count",
+                "Number of Unconsumed Row in buffer",
+                &["executor_id", "connector", "sink_id"],
+                registry
+            )
+            .unwrap();
+
+        let kv_log_store_buffer_unconsumed_epoch_count =
+            register_guarded_int_gauge_vec_with_registry!(
+                "kv_log_store_buffer_unconsumed_epoch_count",
+                "Number of Unconsumed Epoch in buffer",
+                &["executor_id", "connector", "sink_id"],
+                registry
+            )
+            .unwrap();
+
+        let kv_log_store_buffer_unconsumed_min_epoch =
+            register_guarded_int_gauge_vec_with_registry!(
+                "kv_log_store_buffer_unconsumed_min_epoch",
+                "Number of Unconsumed Epoch in buffer",
+                &["executor_id", "connector", "sink_id"],
+                registry
+            )
+            .unwrap();
 
         let lru_runtime_loop_count = register_int_counter_with_registry!(
             "lru_runtime_loop_count",
@@ -917,18 +977,24 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let lru_watermark_step = register_int_gauge_with_registry!(
-            "lru_watermark_step",
-            "The steps increase in 1 loop",
-            registry
+        let lru_latest_sequence = register_int_gauge_with_registry!(
+            "lru_latest_sequence",
+            "Current LRU global sequence",
+            registry,
         )
         .unwrap();
 
-        let lru_evicted_watermark_time_ms = register_guarded_int_gauge_vec_with_registry!(
-            "lru_evicted_watermark_time_ms",
-            "The latest evicted watermark time by actors",
-            &["table_id", "actor_id", "desc"],
-            registry
+        let lru_watermark_sequence = register_int_gauge_with_registry!(
+            "lru_watermark_sequence",
+            "Current LRU watermark sequence",
+            registry,
+        )
+        .unwrap();
+
+        let lru_eviction_policy = register_int_gauge_with_registry!(
+            "lru_eviction_policy",
+            "Current LRU eviction policy",
+            registry,
         )
         .unwrap();
 
@@ -974,18 +1040,18 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let materialize_cache_hit_count = register_int_counter_vec_with_registry!(
+        let materialize_cache_hit_count = register_guarded_int_counter_vec_with_registry!(
             "stream_materialize_cache_hit_count",
             "Materialize executor cache hit count",
-            &["table_id", "actor_id"],
+            &["table_id", "actor_id", "fragment_id"],
             registry
         )
         .unwrap();
 
-        let materialize_cache_total_count = register_int_counter_vec_with_registry!(
+        let materialize_cache_total_count = register_guarded_int_counter_vec_with_registry!(
             "stream_materialize_cache_total_count",
             "Materialize executor cache total operation",
-            &["table_id", "actor_id"],
+            &["table_id", "actor_id", "fragment_id"],
             registry
         )
         .unwrap();
@@ -1059,8 +1125,9 @@ impl StreamingMetrics {
             source_split_change_count,
             source_backfill_row_count,
             sink_input_row_count,
-            mview_input_row_count,
+            sink_chunk_buffer_size,
             exchange_frag_recv_size,
+            merge_barrier_align_duration,
             actor_output_buffer_blocking_duration_ns,
             actor_input_buffer_blocking_duration_ns,
             join_lookup_miss_count,
@@ -1068,19 +1135,19 @@ impl StreamingMetrics {
             join_insert_cache_miss_count,
             join_actor_input_waiting_duration_ns,
             join_match_duration_ns,
-            join_barrier_align_duration,
             join_cached_entry_count,
             join_matched_join_keys,
+            barrier_align_duration,
             agg_lookup_miss_count,
             agg_total_lookup_count,
             agg_cached_entry_count,
             agg_chunk_lookup_miss_count,
             agg_chunk_total_lookup_count,
+            agg_dirty_groups_count,
+            agg_dirty_groups_heap_size,
             agg_distinct_cache_miss_count,
             agg_distinct_total_cache_count,
             agg_distinct_cached_entry_count,
-            agg_dirty_groups_count,
-            agg_dirty_groups_heap_size,
             group_top_n_cache_miss_count,
             group_top_n_total_query_cache_count,
             group_top_n_cached_entry_count,
@@ -1114,31 +1181,36 @@ impl StreamingMetrics {
             log_store_write_rows,
             log_store_latest_read_epoch,
             log_store_read_rows,
+            log_store_reader_wait_new_future_duration_ns,
             kv_log_store_storage_write_count,
             kv_log_store_storage_write_size,
             kv_log_store_rewind_count,
             kv_log_store_rewind_delay,
             kv_log_store_storage_read_count,
             kv_log_store_storage_read_size,
+            kv_log_store_buffer_unconsumed_item_count,
+            kv_log_store_buffer_unconsumed_row_count,
+            kv_log_store_buffer_unconsumed_epoch_count,
+            kv_log_store_buffer_unconsumed_min_epoch,
             iceberg_write_qps,
             iceberg_write_latency,
             iceberg_rolling_unflushed_data_file,
             iceberg_position_delete_cache_num,
             iceberg_partition_num,
-            lru_current_watermark_time_ms,
-            lru_physical_now_ms,
             lru_runtime_loop_count,
-            lru_watermark_step,
-            lru_evicted_watermark_time_ms,
+            lru_latest_sequence,
+            lru_watermark_sequence,
+            lru_eviction_policy,
             jemalloc_allocated_bytes,
             jemalloc_active_bytes,
             jemalloc_resident_bytes,
             jemalloc_metadata_bytes,
             jvm_allocated_bytes,
             jvm_active_bytes,
+            stream_memory_usage,
             materialize_cache_hit_count,
             materialize_cache_total_count,
-            stream_memory_usage,
+            materialize_input_row_count,
         }
     }
 
@@ -1179,6 +1251,9 @@ impl StreamingMetrics {
         let log_store_read_rows = self
             .log_store_read_rows
             .with_guarded_label_values(&label_list);
+        let log_store_reader_wait_new_future_duration_ns = self
+            .log_store_reader_wait_new_future_duration_ns
+            .with_guarded_label_values(&label_list);
 
         let label_list = [identity, sink_id_str];
         let iceberg_write_qps = self
@@ -1205,6 +1280,7 @@ impl StreamingMetrics {
             log_store_write_rows,
             log_store_latest_read_epoch,
             log_store_read_rows,
+            log_store_reader_wait_new_future_duration_ns,
             iceberg_write_qps,
             iceberg_write_latency,
             iceberg_rolling_unflushed_data_file,
@@ -1212,4 +1288,428 @@ impl StreamingMetrics {
             iceberg_partition_num,
         }
     }
+
+    pub fn new_actor_metrics(&self, actor_id: ActorId) -> ActorMetrics {
+        let label_list: &[&str; 1] = &[&actor_id.to_string()];
+        let actor_execution_time = self
+            .actor_execution_time
+            .with_guarded_label_values(label_list);
+        let actor_scheduled_duration = self
+            .actor_scheduled_duration
+            .with_guarded_label_values(label_list);
+        let actor_scheduled_cnt = self
+            .actor_scheduled_cnt
+            .with_guarded_label_values(label_list);
+        let actor_fast_poll_duration = self
+            .actor_fast_poll_duration
+            .with_guarded_label_values(label_list);
+        let actor_fast_poll_cnt = self
+            .actor_fast_poll_cnt
+            .with_guarded_label_values(label_list);
+        let actor_slow_poll_duration = self
+            .actor_slow_poll_duration
+            .with_guarded_label_values(label_list);
+        let actor_slow_poll_cnt = self
+            .actor_slow_poll_cnt
+            .with_guarded_label_values(label_list);
+        let actor_poll_duration = self
+            .actor_poll_duration
+            .with_guarded_label_values(label_list);
+        let actor_poll_cnt = self.actor_poll_cnt.with_guarded_label_values(label_list);
+        let actor_idle_duration = self
+            .actor_idle_duration
+            .with_guarded_label_values(label_list);
+        let actor_idle_cnt = self.actor_idle_cnt.with_guarded_label_values(label_list);
+        ActorMetrics {
+            actor_execution_time,
+            actor_scheduled_duration,
+            actor_scheduled_cnt,
+            actor_fast_poll_duration,
+            actor_fast_poll_cnt,
+            actor_slow_poll_duration,
+            actor_slow_poll_cnt,
+            actor_poll_duration,
+            actor_poll_cnt,
+            actor_idle_duration,
+            actor_idle_cnt,
+        }
+    }
+
+    pub(crate) fn new_actor_input_metrics(
+        &self,
+        actor_id: ActorId,
+        fragment_id: FragmentId,
+        upstream_fragment_id: FragmentId,
+    ) -> ActorInputMetrics {
+        let actor_id_str = actor_id.to_string();
+        let fragment_id_str = fragment_id.to_string();
+        let upstream_fragment_id_str = upstream_fragment_id.to_string();
+        ActorInputMetrics {
+            actor_in_record_cnt: self.actor_in_record_cnt.with_guarded_label_values(&[
+                &actor_id_str,
+                &fragment_id_str,
+                &upstream_fragment_id_str,
+            ]),
+            actor_input_buffer_blocking_duration_ns: self
+                .actor_input_buffer_blocking_duration_ns
+                .with_guarded_label_values(&[
+                    &actor_id_str,
+                    &fragment_id_str,
+                    &upstream_fragment_id_str,
+                ]),
+        }
+    }
+
+    pub fn new_sink_exec_metrics(
+        &self,
+        id: SinkId,
+        actor_id: ActorId,
+        fragment_id: FragmentId,
+    ) -> SinkExecutorMetrics {
+        let label_list: &[&str; 3] = &[
+            &id.to_string(),
+            &actor_id.to_string(),
+            &fragment_id.to_string(),
+        ];
+        SinkExecutorMetrics {
+            sink_input_row_count: self
+                .sink_input_row_count
+                .with_guarded_label_values(label_list),
+            sink_chunk_buffer_size: self
+                .sink_chunk_buffer_size
+                .with_guarded_label_values(label_list),
+        }
+    }
+
+    pub fn new_group_top_n_metrics(
+        &self,
+        table_id: u32,
+        actor_id: ActorId,
+        fragment_id: FragmentId,
+    ) -> GroupTopNMetrics {
+        let label_list: &[&str; 3] = &[
+            &table_id.to_string(),
+            &actor_id.to_string(),
+            &fragment_id.to_string(),
+        ];
+
+        GroupTopNMetrics {
+            group_top_n_cache_miss_count: self
+                .group_top_n_cache_miss_count
+                .with_guarded_label_values(label_list),
+            group_top_n_total_query_cache_count: self
+                .group_top_n_total_query_cache_count
+                .with_guarded_label_values(label_list),
+            group_top_n_cached_entry_count: self
+                .group_top_n_cached_entry_count
+                .with_guarded_label_values(label_list),
+        }
+    }
+
+    pub fn new_append_only_group_top_n_metrics(
+        &self,
+        table_id: u32,
+        actor_id: ActorId,
+        fragment_id: FragmentId,
+    ) -> GroupTopNMetrics {
+        let label_list: &[&str; 3] = &[
+            &table_id.to_string(),
+            &actor_id.to_string(),
+            &fragment_id.to_string(),
+        ];
+
+        GroupTopNMetrics {
+            group_top_n_cache_miss_count: self
+                .group_top_n_appendonly_cache_miss_count
+                .with_guarded_label_values(label_list),
+            group_top_n_total_query_cache_count: self
+                .group_top_n_appendonly_total_query_cache_count
+                .with_guarded_label_values(label_list),
+            group_top_n_cached_entry_count: self
+                .group_top_n_appendonly_cached_entry_count
+                .with_guarded_label_values(label_list),
+        }
+    }
+
+    pub fn new_lookup_executor_metrics(
+        &self,
+        table_id: TableId,
+        actor_id: ActorId,
+        fragment_id: FragmentId,
+    ) -> LookupExecutorMetrics {
+        let label_list: &[&str; 3] = &[
+            &table_id.to_string(),
+            &actor_id.to_string(),
+            &fragment_id.to_string(),
+        ];
+
+        LookupExecutorMetrics {
+            lookup_cache_miss_count: self
+                .lookup_cache_miss_count
+                .with_guarded_label_values(label_list),
+            lookup_total_query_cache_count: self
+                .lookup_total_query_cache_count
+                .with_guarded_label_values(label_list),
+            lookup_cached_entry_count: self
+                .lookup_cached_entry_count
+                .with_guarded_label_values(label_list),
+        }
+    }
+
+    pub fn new_hash_agg_metrics(
+        &self,
+        table_id: u32,
+        actor_id: ActorId,
+        fragment_id: FragmentId,
+    ) -> HashAggMetrics {
+        let label_list: &[&str; 3] = &[
+            &table_id.to_string(),
+            &actor_id.to_string(),
+            &fragment_id.to_string(),
+        ];
+        HashAggMetrics {
+            agg_lookup_miss_count: self
+                .agg_lookup_miss_count
+                .with_guarded_label_values(label_list),
+            agg_total_lookup_count: self
+                .agg_total_lookup_count
+                .with_guarded_label_values(label_list),
+            agg_cached_entry_count: self
+                .agg_cached_entry_count
+                .with_guarded_label_values(label_list),
+            agg_chunk_lookup_miss_count: self
+                .agg_chunk_lookup_miss_count
+                .with_guarded_label_values(label_list),
+            agg_chunk_total_lookup_count: self
+                .agg_chunk_total_lookup_count
+                .with_guarded_label_values(label_list),
+            agg_dirty_groups_count: self
+                .agg_dirty_groups_count
+                .with_guarded_label_values(label_list),
+            agg_dirty_groups_heap_size: self
+                .agg_dirty_groups_heap_size
+                .with_guarded_label_values(label_list),
+        }
+    }
+
+    pub fn new_agg_distinct_dedup_metrics(
+        &self,
+        table_id: u32,
+        actor_id: ActorId,
+        fragment_id: FragmentId,
+    ) -> AggDistinctDedupMetrics {
+        let label_list: &[&str; 3] = &[
+            &table_id.to_string(),
+            &actor_id.to_string(),
+            &fragment_id.to_string(),
+        ];
+        AggDistinctDedupMetrics {
+            agg_distinct_cache_miss_count: self
+                .agg_distinct_cache_miss_count
+                .with_guarded_label_values(label_list),
+            agg_distinct_total_cache_count: self
+                .agg_distinct_total_cache_count
+                .with_guarded_label_values(label_list),
+            agg_distinct_cached_entry_count: self
+                .agg_distinct_cached_entry_count
+                .with_guarded_label_values(label_list),
+        }
+    }
+
+    pub fn new_temporal_join_metrics(
+        &self,
+        table_id: TableId,
+        actor_id: ActorId,
+        fragment_id: FragmentId,
+    ) -> TemporalJoinMetrics {
+        let label_list: &[&str; 3] = &[
+            &table_id.to_string(),
+            &actor_id.to_string(),
+            &fragment_id.to_string(),
+        ];
+        TemporalJoinMetrics {
+            temporal_join_cache_miss_count: self
+                .temporal_join_cache_miss_count
+                .with_guarded_label_values(label_list),
+            temporal_join_total_query_cache_count: self
+                .temporal_join_total_query_cache_count
+                .with_guarded_label_values(label_list),
+            temporal_join_cached_entry_count: self
+                .temporal_join_cached_entry_count
+                .with_guarded_label_values(label_list),
+        }
+    }
+
+    pub fn new_backfill_metrics(&self, table_id: u32, actor_id: ActorId) -> BackfillMetrics {
+        let label_list: &[&str; 2] = &[&table_id.to_string(), &actor_id.to_string()];
+        BackfillMetrics {
+            backfill_snapshot_read_row_count: self
+                .backfill_snapshot_read_row_count
+                .with_guarded_label_values(label_list),
+            backfill_upstream_output_row_count: self
+                .backfill_upstream_output_row_count
+                .with_guarded_label_values(label_list),
+        }
+    }
+
+    pub fn new_cdc_backfill_metrics(
+        &self,
+        table_id: TableId,
+        actor_id: ActorId,
+    ) -> CdcBackfillMetrics {
+        let label_list: &[&str; 2] = &[&table_id.to_string(), &actor_id.to_string()];
+        CdcBackfillMetrics {
+            cdc_backfill_snapshot_read_row_count: self
+                .cdc_backfill_snapshot_read_row_count
+                .with_guarded_label_values(label_list),
+            cdc_backfill_upstream_output_row_count: self
+                .cdc_backfill_upstream_output_row_count
+                .with_guarded_label_values(label_list),
+        }
+    }
+
+    pub fn new_over_window_metrics(
+        &self,
+        table_id: u32,
+        actor_id: ActorId,
+        fragment_id: FragmentId,
+    ) -> OverWindowMetrics {
+        let label_list: &[&str; 3] = &[
+            &table_id.to_string(),
+            &actor_id.to_string(),
+            &fragment_id.to_string(),
+        ];
+        OverWindowMetrics {
+            over_window_cached_entry_count: self
+                .over_window_cached_entry_count
+                .with_guarded_label_values(label_list),
+            over_window_cache_lookup_count: self
+                .over_window_cache_lookup_count
+                .with_guarded_label_values(label_list),
+            over_window_cache_miss_count: self
+                .over_window_cache_miss_count
+                .with_guarded_label_values(label_list),
+            over_window_range_cache_entry_count: self
+                .over_window_range_cache_entry_count
+                .with_guarded_label_values(label_list),
+            over_window_range_cache_lookup_count: self
+                .over_window_range_cache_lookup_count
+                .with_guarded_label_values(label_list),
+            over_window_range_cache_left_miss_count: self
+                .over_window_range_cache_left_miss_count
+                .with_guarded_label_values(label_list),
+            over_window_range_cache_right_miss_count: self
+                .over_window_range_cache_right_miss_count
+                .with_guarded_label_values(label_list),
+        }
+    }
+
+    pub fn new_materialize_metrics(
+        &self,
+        table_id: TableId,
+        actor_id: ActorId,
+        fragment_id: FragmentId,
+    ) -> MaterializeMetrics {
+        let label_list: &[&str; 3] = &[
+            &table_id.to_string(),
+            &actor_id.to_string(),
+            &fragment_id.to_string(),
+        ];
+        MaterializeMetrics {
+            materialize_cache_hit_count: self
+                .materialize_cache_hit_count
+                .with_guarded_label_values(label_list),
+            materialize_cache_total_count: self
+                .materialize_cache_total_count
+                .with_guarded_label_values(label_list),
+            materialize_input_row_count: self
+                .materialize_input_row_count
+                .with_guarded_label_values(label_list),
+        }
+    }
+}
+
+pub(crate) struct ActorInputMetrics {
+    pub(crate) actor_in_record_cnt: LabelGuardedIntCounter<3>,
+    pub(crate) actor_input_buffer_blocking_duration_ns: LabelGuardedIntCounter<3>,
+}
+
+/// Tokio metrics for actors
+pub struct ActorMetrics {
+    pub actor_execution_time: LabelGuardedGauge<1>,
+    pub actor_scheduled_duration: LabelGuardedGauge<1>,
+    pub actor_scheduled_cnt: LabelGuardedIntGauge<1>,
+    pub actor_fast_poll_duration: LabelGuardedGauge<1>,
+    pub actor_fast_poll_cnt: LabelGuardedIntGauge<1>,
+    pub actor_slow_poll_duration: LabelGuardedGauge<1>,
+    pub actor_slow_poll_cnt: LabelGuardedIntGauge<1>,
+    pub actor_poll_duration: LabelGuardedGauge<1>,
+    pub actor_poll_cnt: LabelGuardedIntGauge<1>,
+    pub actor_idle_duration: LabelGuardedGauge<1>,
+    pub actor_idle_cnt: LabelGuardedIntGauge<1>,
+}
+
+pub struct SinkExecutorMetrics {
+    pub sink_input_row_count: LabelGuardedIntCounter<3>,
+    pub sink_chunk_buffer_size: LabelGuardedIntGauge<3>,
+}
+
+pub struct MaterializeMetrics {
+    pub materialize_cache_hit_count: LabelGuardedIntCounter<3>,
+    pub materialize_cache_total_count: LabelGuardedIntCounter<3>,
+    pub materialize_input_row_count: LabelGuardedIntCounter<3>,
+}
+
+pub struct GroupTopNMetrics {
+    pub group_top_n_cache_miss_count: LabelGuardedIntCounter<3>,
+    pub group_top_n_total_query_cache_count: LabelGuardedIntCounter<3>,
+    pub group_top_n_cached_entry_count: LabelGuardedIntGauge<3>,
+}
+
+pub struct LookupExecutorMetrics {
+    pub lookup_cache_miss_count: LabelGuardedIntCounter<3>,
+    pub lookup_total_query_cache_count: LabelGuardedIntCounter<3>,
+    pub lookup_cached_entry_count: LabelGuardedIntGauge<3>,
+}
+
+pub struct HashAggMetrics {
+    pub agg_lookup_miss_count: LabelGuardedIntCounter<3>,
+    pub agg_total_lookup_count: LabelGuardedIntCounter<3>,
+    pub agg_cached_entry_count: LabelGuardedIntGauge<3>,
+    pub agg_chunk_lookup_miss_count: LabelGuardedIntCounter<3>,
+    pub agg_chunk_total_lookup_count: LabelGuardedIntCounter<3>,
+    pub agg_dirty_groups_count: LabelGuardedIntGauge<3>,
+    pub agg_dirty_groups_heap_size: LabelGuardedIntGauge<3>,
+}
+
+pub struct AggDistinctDedupMetrics {
+    pub agg_distinct_cache_miss_count: LabelGuardedIntCounter<3>,
+    pub agg_distinct_total_cache_count: LabelGuardedIntCounter<3>,
+    pub agg_distinct_cached_entry_count: LabelGuardedIntGauge<3>,
+}
+
+pub struct TemporalJoinMetrics {
+    pub temporal_join_cache_miss_count: LabelGuardedIntCounter<3>,
+    pub temporal_join_total_query_cache_count: LabelGuardedIntCounter<3>,
+    pub temporal_join_cached_entry_count: LabelGuardedIntGauge<3>,
+}
+
+pub struct BackfillMetrics {
+    pub backfill_snapshot_read_row_count: LabelGuardedIntCounter<2>,
+    pub backfill_upstream_output_row_count: LabelGuardedIntCounter<2>,
+}
+
+pub struct CdcBackfillMetrics {
+    pub cdc_backfill_snapshot_read_row_count: LabelGuardedIntCounter<2>,
+    pub cdc_backfill_upstream_output_row_count: LabelGuardedIntCounter<2>,
+}
+
+pub struct OverWindowMetrics {
+    pub over_window_cached_entry_count: LabelGuardedIntGauge<3>,
+    pub over_window_cache_lookup_count: LabelGuardedIntCounter<3>,
+    pub over_window_cache_miss_count: LabelGuardedIntCounter<3>,
+    pub over_window_range_cache_entry_count: LabelGuardedIntGauge<3>,
+    pub over_window_range_cache_lookup_count: LabelGuardedIntCounter<3>,
+    pub over_window_range_cache_left_miss_count: LabelGuardedIntCounter<3>,
+    pub over_window_range_cache_right_miss_count: LabelGuardedIntCounter<3>,
 }
diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs
index 29a3e131fd2f9..1861313baddc6 100644
--- a/src/stream/src/executor/mview/materialize.rs
+++ b/src/stream/src/executor/mview/materialize.rs
@@ -24,7 +24,7 @@ use itertools::Itertools;
 use risingwave_common::array::Op;
 use risingwave_common::buffer::Bitmap;
 use risingwave_common::catalog::{ColumnDesc, ColumnId, ConflictBehavior, TableId};
-use risingwave_common::row::{CompactedRow, OwnedRow, RowDeserializer};
+use risingwave_common::row::{CompactedRow, RowDeserializer};
 use risingwave_common::types::DefaultOrd;
 use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
 use risingwave_common::util::iter_util::{ZipEqDebug, ZipEqFast};
@@ -34,11 +34,11 @@ use risingwave_pb::catalog::Table;
 use risingwave_storage::mem_table::KeyOp;
 use risingwave_storage::row_serde::value_serde::{ValueRowSerde, ValueRowSerdeNew};
 
-use crate::cache::{new_unbounded, ManagedLruCache};
+use crate::cache::ManagedLruCache;
 use crate::common::metrics::MetricsInfo;
-use crate::common::table::state_table::StateTableInner;
+use crate::common::table::state_table::{StateTableInner, StateTableOpConsistencyLevel};
+use crate::executor::monitor::MaterializeMetrics;
 use crate::executor::prelude::*;
-use crate::executor::{AddMutation, UpdateMutation};
 
 /// `MaterializeExecutor` materializes changes in stream into a materialized view on storage.
 pub struct MaterializeExecutor<S: StateStore, SD: ValueRowSerde> {
@@ -58,6 +58,28 @@ pub struct MaterializeExecutor<S: StateStore, SD: ValueRowSerde> {
     conflict_behavior: ConflictBehavior,
 
     version_column_index: Option<u32>,
+
+    may_have_downstream: bool,
+
+    depended_subscription_ids: HashSet<u32>,
+
+    metrics: MaterializeMetrics,
+}
+
+fn get_op_consistency_level(
+    conflict_behavior: ConflictBehavior,
+    may_have_downstream: bool,
+    depended_subscriptions: &HashSet<u32>,
+) -> StateTableOpConsistencyLevel {
+    if !depended_subscriptions.is_empty() {
+        StateTableOpConsistencyLevel::LogStoreEnabled
+    } else if !may_have_downstream && matches!(conflict_behavior, ConflictBehavior::Overwrite) {
+        // Table with overwrite conflict behavior could disable conflict check
+        // if no downstream mv depends on it, so we use a inconsistent_op to skip sanity check as well.
+        StateTableOpConsistencyLevel::Inconsistent
+    } else {
+        StateTableOpConsistencyLevel::ConsistentOldValue
+    }
 }
 
 impl<S: StateStore, SD: ValueRowSerde> MaterializeExecutor<S, SD> {
@@ -90,19 +112,35 @@ impl<S: StateStore, SD: ValueRowSerde> MaterializeExecutor<S, SD> {
         );
 
         let arrange_key_indices: Vec<usize> = arrange_key.iter().map(|k| k.column_index).collect();
-
+        let may_have_downstream = actor_context.initial_dispatch_num != 0;
+        let depended_subscription_ids = actor_context
+            .related_subscriptions
+            .get(&TableId::new(table_catalog.id))
+            .cloned()
+            .unwrap_or_default();
+        let op_consistency_level = get_op_consistency_level(
+            conflict_behavior,
+            may_have_downstream,
+            &depended_subscription_ids,
+        );
         // Note: The current implementation could potentially trigger a switch on the inconsistent_op flag. If the storage relies on this flag to perform optimizations, it would be advisable to maintain consistency with it throughout the lifecycle.
-        let state_table = if matches!(conflict_behavior, ConflictBehavior::Overwrite)
-            && actor_context.dispatch_num == 0
-        {
-            // Table with overwrite conflict behavior could disable conflict check if no downstream mv depends on it, so we use a inconsistent_op to skip sanity check as well.
-            StateTableInner::from_table_catalog_inconsistent_op(table_catalog, store, vnodes).await
-        } else {
-            StateTableInner::from_table_catalog(table_catalog, store, vnodes).await
-        };
+        let state_table = StateTableInner::from_table_catalog_with_consistency_level(
+            table_catalog,
+            store,
+            vnodes,
+            op_consistency_level,
+        )
+        .await;
+
+        let mv_metrics = metrics.new_materialize_metrics(
+            TableId::new(table_catalog.id),
+            actor_context.id,
+            actor_context.fragment_id,
+        );
 
         let metrics_info =
             MetricsInfo::new(metrics, table_catalog.id, actor_context.id, "Materialize");
+
         Self {
             input,
             schema,
@@ -117,15 +155,15 @@ impl<S: StateStore, SD: ValueRowSerde> MaterializeExecutor<S, SD> {
             ),
             conflict_behavior,
             version_column_index,
+            may_have_downstream,
+            depended_subscription_ids,
+            metrics: mv_metrics,
         }
     }
 
     #[try_stream(ok = Message, error = StreamExecutorError)]
     async fn execute_inner(mut self) {
-        // for metrics
-        let table_id_str = self.state_table.table_id().to_string();
-        let actor_id_str = self.actor_context.id.to_string();
-        let fragment_id_str = self.actor_context.fragment_id.to_string();
+        let mv_table_id = TableId::new(self.state_table.table_id());
 
         let data_types = self.schema.data_types();
         let mut input = self.input.execute();
@@ -144,10 +182,8 @@ impl<S: StateStore, SD: ValueRowSerde> MaterializeExecutor<S, SD> {
             yield match msg {
                 Message::Watermark(w) => Message::Watermark(w),
                 Message::Chunk(chunk) => {
-                    self.actor_context
-                        .streaming_metrics
-                        .mview_input_row_count
-                        .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
+                    self.metrics
+                        .materialize_input_row_count
                         .inc_by(chunk.cardinality() as u64);
 
                     // This is an optimization that handles conflicts only when a particular materialized view downstream has no MV dependencies.
@@ -200,7 +236,12 @@ impl<S: StateStore, SD: ValueRowSerde> MaterializeExecutor<S, SD> {
 
                             let fixed_changes = self
                                 .materialize_cache
-                                .handle(row_ops, &self.state_table, &self.conflict_behavior)
+                                .handle(
+                                    row_ops,
+                                    &self.state_table,
+                                    &self.conflict_behavior,
+                                    &self.metrics,
+                                )
                                 .await?;
 
                             match generate_output(fixed_changes, data_types.clone())? {
@@ -223,17 +264,27 @@ impl<S: StateStore, SD: ValueRowSerde> MaterializeExecutor<S, SD> {
                     }
                 }
                 Message::Barrier(b) => {
-                    let mutation = b.mutation.clone();
                     // If a downstream mv depends on the current table, we need to do conflict check again.
-                    if !self.state_table.is_consistent_op()
-                        && Self::new_downstream_created(mutation, self.actor_context.id)
+                    if !self.may_have_downstream
+                        && b.has_more_downstream_fragments(self.actor_context.id)
                     {
+                        self.may_have_downstream = true;
+                    }
+                    Self::may_update_depended_subscriptions(
+                        &mut self.depended_subscription_ids,
+                        b.mutation.as_deref(),
+                        mv_table_id,
+                    );
+                    let op_consistency_level = get_op_consistency_level(
+                        self.conflict_behavior,
+                        self.may_have_downstream,
+                        &self.depended_subscription_ids,
+                    );
+                    self.state_table
+                        .commit_may_switch_consistent_op(b.epoch, op_consistency_level)
+                        .await?;
+                    if !self.state_table.is_consistent_op() {
                         assert_eq!(self.conflict_behavior, ConflictBehavior::Overwrite);
-                        self.state_table
-                            .commit_with_switch_consistent_op(b.epoch, Some(true))
-                            .await?;
-                    } else {
-                        self.state_table.commit(b.epoch).await?;
                     }
 
                     // Update the vnode bitmap for the state table if asked.
@@ -245,39 +296,60 @@ impl<S: StateStore, SD: ValueRowSerde> MaterializeExecutor<S, SD> {
                             self.materialize_cache.data.clear();
                         }
                     }
-                    self.materialize_cache.data.update_epoch(b.epoch.curr);
                     Message::Barrier(b)
                 }
             }
         }
     }
 
-    fn new_downstream_created(mutation: Option<Arc<Mutation>>, actor_id: ActorId) -> bool {
-        let Some(mutation) = mutation.as_deref() else {
-            return false;
+    /// return true when changed
+    fn may_update_depended_subscriptions(
+        depended_subscriptions: &mut HashSet<u32>,
+        mutation: Option<&Mutation>,
+        mv_table_id: TableId,
+    ) {
+        let Some(mutation) = mutation else {
+            return;
         };
         match mutation {
-            // Add is for mv, index and sink creation.
-            Mutation::Add(AddMutation { adds, .. }) => adds.get(&actor_id).is_some(),
-            // AddAndUpdate is for sink-into-table.
-            Mutation::AddAndUpdate(
-                AddMutation { adds, .. },
-                UpdateMutation {
-                    dispatchers,
-                    actor_new_dispatchers: actor_dispatchers,
-                    ..
-                },
-            ) => {
-                adds.get(&actor_id).is_some()
-                    || actor_dispatchers.get(&actor_id).is_some()
-                    || dispatchers.get(&actor_id).is_some()
+            Mutation::CreateSubscription {
+                subscription_id,
+                upstream_mv_table_id,
+            } => {
+                if *upstream_mv_table_id == mv_table_id
+                    && !depended_subscriptions.insert(*subscription_id)
+                {
+                    warn!(
+                        ?depended_subscriptions,
+                        ?mv_table_id,
+                        subscription_id,
+                        "subscription id already exists"
+                    );
+                }
+            }
+            Mutation::DropSubscription {
+                subscription_id,
+                upstream_mv_table_id,
+            } => {
+                if *upstream_mv_table_id == mv_table_id
+                    && !depended_subscriptions.remove(subscription_id)
+                {
+                    warn!(
+                        ?depended_subscriptions,
+                        ?mv_table_id,
+                        subscription_id,
+                        "drop non existing subscription id"
+                    );
+                }
             }
-            Mutation::Update(_)
-            | Mutation::Stop(_)
+            Mutation::Stop(_)
+            | Mutation::Update(_)
+            | Mutation::Add(_)
+            | Mutation::SourceChangeSplit(_)
             | Mutation::Pause
             | Mutation::Resume
-            | Mutation::SourceChangeSplit(_)
-            | Mutation::Throttle(_) => false,
+            | Mutation::Throttle(_)
+            | Mutation::AddAndUpdate(_, _) => {}
         }
     }
 }
@@ -316,6 +388,8 @@ impl<S: StateStore> MaterializeExecutor<S, BasicSerde> {
         )
         .await;
 
+        let metrics = StreamingMetrics::unused().new_materialize_metrics(table_id, 1, 2);
+
         Self {
             input,
             schema,
@@ -330,6 +404,9 @@ impl<S: StateStore> MaterializeExecutor<S, BasicSerde> {
             ),
             conflict_behavior,
             version_column_index: None,
+            may_have_downstream: true,
+            depended_subscription_ids: HashSet::new(),
+            metrics,
         }
     }
 }
@@ -473,7 +550,6 @@ impl<S: StateStore, SD: ValueRowSerde> std::fmt::Debug for MaterializeExecutor<S
 /// A cache for materialize executors.
 pub struct MaterializeCache<SD> {
     data: ManagedLruCache<Vec<u8>, CacheValue>,
-    metrics_info: MetricsInfo,
     row_serde: BasicSerde,
     version_column_index: Option<u32>,
     _serde: PhantomData<SD>,
@@ -483,16 +559,15 @@ type CacheValue = Option<CompactedRow>;
 
 impl<SD: ValueRowSerde> MaterializeCache<SD> {
     pub fn new(
-        watermark_epoch: AtomicU64Ref,
+        watermark_sequence: AtomicU64Ref,
         metrics_info: MetricsInfo,
         row_serde: BasicSerde,
         version_column_index: Option<u32>,
     ) -> Self {
         let cache: ManagedLruCache<Vec<u8>, CacheValue> =
-            new_unbounded(watermark_epoch, metrics_info.clone());
+            ManagedLruCache::unbounded(watermark_sequence, metrics_info.clone());
         Self {
             data: cache,
-            metrics_info,
             row_serde,
             version_column_index,
             _serde: PhantomData,
@@ -504,13 +579,19 @@ impl<SD: ValueRowSerde> MaterializeCache<SD> {
         row_ops: Vec<(Op, Vec<u8>, Bytes)>,
         table: &StateTableInner<S, SD>,
         conflict_behavior: &ConflictBehavior,
+        metrics: &MaterializeMetrics,
     ) -> StreamExecutorResult<MaterializeBuffer> {
         let key_set: HashSet<Box<[u8]>> = row_ops
             .iter()
             .map(|(_, k, _)| k.as_slice().into())
             .collect();
-        self.fetch_keys(key_set.iter().map(|v| v.deref()), table, conflict_behavior)
-            .await?;
+        self.fetch_keys(
+            key_set.iter().map(|v| v.deref()),
+            table,
+            conflict_behavior,
+            metrics,
+        )
+        .await?;
         let mut fixed_changes = MaterializeBuffer::new();
         let row_serde = self.row_serde.clone();
         let version_column_index = self.version_column_index;
@@ -655,20 +736,14 @@ impl<SD: ValueRowSerde> MaterializeCache<SD> {
         keys: impl Iterator<Item = &'a [u8]>,
         table: &StateTableInner<S, SD>,
         conflict_behavior: &ConflictBehavior,
+        metrics: &MaterializeMetrics,
     ) -> StreamExecutorResult<()> {
         let mut futures = vec![];
         for key in keys {
-            self.metrics_info
-                .metrics
-                .materialize_cache_total_count
-                .with_label_values(&[&self.metrics_info.table_id, &self.metrics_info.actor_id])
-                .inc();
+            metrics.materialize_cache_total_count.inc();
+
             if self.data.contains(key) {
-                self.metrics_info
-                    .metrics
-                    .materialize_cache_hit_count
-                    .with_label_values(&[&self.metrics_info.table_id, &self.metrics_info.actor_id])
-                    .inc();
+                metrics.materialize_cache_hit_count.inc();
                 continue;
             }
             futures.push(async {
@@ -767,23 +842,17 @@ mod tests {
     use std::iter;
     use std::sync::atomic::AtomicU64;
 
-    use futures::stream::StreamExt;
     use rand::rngs::SmallRng;
     use rand::{Rng, RngCore, SeedableRng};
     use risingwave_common::array::stream_chunk::{StreamChunkMut, StreamChunkTestExt};
-    use risingwave_common::array::stream_chunk_builder::StreamChunkBuilder;
-    use risingwave_common::array::Op;
-    use risingwave_common::catalog::{ColumnDesc, ConflictBehavior, Field, Schema, TableId};
-    use risingwave_common::row::OwnedRow;
-    use risingwave_common::types::DataType;
+    use risingwave_common::catalog::Field;
     use risingwave_common::util::epoch::test_epoch;
-    use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
+    use risingwave_common::util::sort_util::OrderType;
     use risingwave_hummock_sdk::HummockReadEpoch;
     use risingwave_storage::memory::MemoryStateStore;
     use risingwave_storage::table::batch_table::storage_table::StorageTable;
 
     use super::*;
-    use crate::executor::test_utils::prelude::StateTable;
     use crate::executor::test_utils::*;
 
     #[tokio::test]
diff --git a/src/stream/src/executor/over_window/eowc.rs b/src/stream/src/executor/over_window/eowc.rs
index 1691f0f0c1b60..263b578d5d832 100644
--- a/src/stream/src/executor/over_window/eowc.rs
+++ b/src/stream/src/executor/over_window/eowc.rs
@@ -31,7 +31,7 @@ use risingwave_expr::window_function::{
 };
 use risingwave_storage::store::PrefetchOptions;
 
-use crate::cache::{new_unbounded, ManagedLruCache};
+use crate::cache::ManagedLruCache;
 use crate::common::metrics::MetricsInfo;
 use crate::executor::prelude::*;
 
@@ -100,7 +100,7 @@ struct ExecutorInner<S: StateStore> {
     order_key_index: usize, // no `OrderType` here, cuz we expect the input is ascending
     state_table: StateTable<S>,
     state_table_schema_len: usize,
-    watermark_epoch: AtomicU64Ref,
+    watermark_sequence: AtomicU64Ref,
 }
 
 struct ExecutionVars<S: StateStore> {
@@ -142,7 +142,7 @@ impl<S: StateStore> EowcOverWindowExecutor<S> {
                 order_key_index: args.order_key_index,
                 state_table: args.state_table,
                 state_table_schema_len: input_info.schema.len(),
-                watermark_epoch: args.watermark_epoch,
+                watermark_sequence: args.watermark_epoch,
             },
         }
     }
@@ -341,14 +341,13 @@ impl<S: StateStore> EowcOverWindowExecutor<S> {
         );
 
         let mut vars = ExecutionVars {
-            partitions: new_unbounded(this.watermark_epoch.clone(), metrics_info),
+            partitions: ManagedLruCache::unbounded(this.watermark_sequence.clone(), metrics_info),
             _phantom: PhantomData::<S>,
         };
 
         let mut input = input.execute();
         let barrier = expect_first_barrier(&mut input).await?;
         this.state_table.init_epoch(barrier.epoch);
-        vars.partitions.update_epoch(barrier.epoch.curr);
 
         yield Message::Barrier(barrier);
 
@@ -378,8 +377,6 @@ impl<S: StateStore> EowcOverWindowExecutor<S> {
                         }
                     }
 
-                    vars.partitions.update_epoch(barrier.epoch.curr);
-
                     yield Message::Barrier(barrier);
                 }
             }
diff --git a/src/stream/src/executor/over_window/frame_finder.rs b/src/stream/src/executor/over_window/frame_finder.rs
index 3154284653f11..12e9f21842887 100644
--- a/src/stream/src/executor/over_window/frame_finder.rs
+++ b/src/stream/src/executor/over_window/frame_finder.rs
@@ -103,7 +103,7 @@ pub(super) fn merge_rows_frames(rows_frames: &[&RowsFrameBounds]) -> RowsFrameBo
 ///
 /// More examples can be found in the comment inside [`find_curr_for_rows_frame`].
 pub(super) fn find_first_curr_for_rows_frame<'cache>(
-    frame_bounds: &'_ RowsFrameBounds,
+    frame_bounds: &RowsFrameBounds,
     part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
     delta_key: &'cache CacheKey,
 ) -> &'cache CacheKey {
@@ -116,7 +116,7 @@ pub(super) fn find_first_curr_for_rows_frame<'cache>(
 ///
 /// This is the symmetric function of [`find_first_curr_for_rows_frame`].
 pub(super) fn find_last_curr_for_rows_frame<'cache>(
-    frame_bounds: &'_ RowsFrameBounds,
+    frame_bounds: &RowsFrameBounds,
     part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
     delta_key: &'cache CacheKey,
 ) -> &'cache CacheKey {
@@ -127,7 +127,7 @@ pub(super) fn find_last_curr_for_rows_frame<'cache>(
 /// to some CURRENT ROW, find the cache key corresponding to the start row in
 /// that frame.
 pub(super) fn find_frame_start_for_rows_frame<'cache>(
-    frame_bounds: &'_ RowsFrameBounds,
+    frame_bounds: &RowsFrameBounds,
     part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
     curr_key: &'cache CacheKey,
 ) -> &'cache CacheKey {
@@ -140,7 +140,7 @@ pub(super) fn find_frame_start_for_rows_frame<'cache>(
 ///
 /// This is the symmetric function of [`find_frame_start_for_rows_frame`].
 pub(super) fn find_frame_end_for_rows_frame<'cache>(
-    frame_bounds: &'_ RowsFrameBounds,
+    frame_bounds: &RowsFrameBounds,
     part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
     curr_key: &'cache CacheKey,
 ) -> &'cache CacheKey {
@@ -189,7 +189,7 @@ pub(super) fn find_left_for_range_frames<'cache>(
     logical_order_value: impl ToDatumRef,
     cache_key_pk_len: usize, // this is dirty but we have no better choice
 ) -> &'cache CacheKey {
-    find_for_range_frames::<true>(
+    find_for_range_frames::<true /* LEFT */>(
         range_frames,
         part_with_delta,
         logical_order_value,
@@ -206,7 +206,7 @@ pub(super) fn find_right_for_range_frames<'cache>(
     logical_order_value: impl ToDatumRef,
     cache_key_pk_len: usize, // this is dirty but we have no better choice
 ) -> &'cache CacheKey {
-    find_for_range_frames::<false>(
+    find_for_range_frames::<false /* RIGHT */>(
         range_frames,
         part_with_delta,
         logical_order_value,
@@ -217,7 +217,7 @@ pub(super) fn find_right_for_range_frames<'cache>(
 // -------------------------- ↑ PUBLIC INTERFACE ↑ --------------------------
 
 fn find_curr_for_rows_frame<'cache, const LEFT: bool>(
-    frame_bounds: &'_ RowsFrameBounds,
+    frame_bounds: &RowsFrameBounds,
     part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
     delta_key: &'cache CacheKey,
 ) -> &'cache CacheKey {
@@ -329,7 +329,7 @@ fn find_curr_for_rows_frame<'cache, const LEFT: bool>(
 }
 
 fn find_boundary_for_rows_frame<'cache, const LEFT: bool>(
-    frame_bounds: &'_ RowsFrameBounds,
+    frame_bounds: &RowsFrameBounds,
     part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
     curr_key: &'cache CacheKey,
 ) -> &'cache CacheKey {
@@ -1219,12 +1219,12 @@ mod tests {
             ];
 
             let ord_key_1 = StateKey {
-                order_key: memcmp_encoding::encode_value(&Some(ScalarImpl::Int64(1)), order_type)
+                order_key: memcmp_encoding::encode_value(Some(ScalarImpl::Int64(1)), order_type)
                     .unwrap(),
                 pk: OwnedRow::empty().into(),
             };
             let ord_key_2 = StateKey {
-                order_key: memcmp_encoding::encode_value(&Some(ScalarImpl::Int64(3)), order_type)
+                order_key: memcmp_encoding::encode_value(Some(ScalarImpl::Int64(3)), order_type)
                     .unwrap(),
                 pk: OwnedRow::empty().into(),
             };
@@ -1261,7 +1261,7 @@ mod tests {
 
             let ord_key_1 = StateKey {
                 order_key: memcmp_encoding::encode_value(
-                    &Some(ScalarImpl::Timestamp(
+                    Some(ScalarImpl::Timestamp(
                         "2024-01-28 00:30:00".parse().unwrap(),
                     )),
                     order_type,
@@ -1271,7 +1271,7 @@ mod tests {
             };
             let ord_key_2 = StateKey {
                 order_key: memcmp_encoding::encode_value(
-                    &Some(ScalarImpl::Timestamp(
+                    Some(ScalarImpl::Timestamp(
                         "2024-01-26 15:47:00".parse().unwrap(),
                     )),
                     order_type,
diff --git a/src/stream/src/executor/over_window/general.rs b/src/stream/src/executor/over_window/general.rs
index 7cb29c39f7e37..23623be6e0f2c 100644
--- a/src/stream/src/executor/over_window/general.rs
+++ b/src/stream/src/executor/over_window/general.rs
@@ -29,17 +29,15 @@ use risingwave_common::util::sort_util::OrderType;
 use risingwave_expr::window_function::{
     create_window_state, StateKey, WindowFuncCall, WindowStates,
 };
-use risingwave_storage::row_serde::row_serde_util::serialize_pk_with_vnode;
-use risingwave_storage::StateStore;
 
 use super::over_partition::{
     new_empty_partition_cache, shrink_partition_cache, CacheKey, OverPartition, PartitionCache,
     PartitionDelta,
 };
-use crate::cache::{new_unbounded, ManagedLruCache};
+use crate::cache::ManagedLruCache;
 use crate::common::metrics::MetricsInfo;
-use crate::common::table::state_table::StateTable;
-use crate::executor::monitor::StreamingMetrics;
+use crate::consistency::consistency_panic;
+use crate::executor::monitor::OverWindowMetrics;
 use crate::executor::over_window::over_partition::AffectedRange;
 use crate::executor::prelude::*;
 
@@ -67,8 +65,7 @@ struct ExecutorInner<S: StateStore> {
     state_key_to_table_sub_pk_proj: Vec<usize>,
 
     state_table: StateTable<S>,
-    watermark_epoch: AtomicU64Ref,
-    metrics: Arc<StreamingMetrics>,
+    watermark_sequence: AtomicU64Ref,
 
     /// The maximum size of the chunk produced by executor at a time.
     chunk_size: usize,
@@ -185,8 +182,7 @@ impl<S: StateStore> OverWindowExecutor<S> {
                 input_schema_len: input_schema.len(),
                 state_key_to_table_sub_pk_proj,
                 state_table: args.state_table,
-                watermark_epoch: args.watermark_epoch,
-                metrics: args.metrics,
+                watermark_sequence: args.watermark_epoch,
                 chunk_size: args.chunk_size,
                 cache_policy,
             },
@@ -216,7 +212,20 @@ impl<S: StateStore> OverWindowExecutor<S> {
                                     new_row: row,
                                 };
                             }
-                            _ => panic!("inconsistent changes in input chunk"),
+                            _ => {
+                                consistency_panic!(
+                                    ?pk,
+                                    "inconsistent changes in input chunk, double-inserting"
+                                );
+                                if let Record::Update { old_row, .. } = prev_change {
+                                    *prev_change = Record::Update {
+                                        old_row: *old_row,
+                                        new_row: row,
+                                    };
+                                } else {
+                                    *prev_change = Record::Insert { new_row: row };
+                                }
+                            }
                         }
                     } else {
                         changes_merged.insert(pk, Record::Insert { new_row: row });
@@ -236,7 +245,13 @@ impl<S: StateStore> OverWindowExecutor<S> {
                                     old_row: *real_old_row,
                                 };
                             }
-                            _ => panic!("inconsistent changes in input chunk"),
+                            _ => {
+                                consistency_panic!(
+                                    ?pk,
+                                    "inconsistent changes in input chunk, double-deleting"
+                                );
+                                *prev_change = Record::Delete { old_row: row };
+                            }
                         }
                     } else {
                         changes_merged.insert(pk, Record::Delete { old_row: row });
@@ -252,6 +267,7 @@ impl<S: StateStore> OverWindowExecutor<S> {
         this: &'a mut ExecutorInner<S>,
         vars: &'a mut ExecutionVars<S>,
         chunk: StreamChunk,
+        metrics: &'a OverWindowMetrics,
     ) {
         // partition key => changes happened in the partition.
         let mut deltas: BTreeMap<DefaultOrdered<OwnedRow>, PartitionDelta> = BTreeMap::new();
@@ -310,11 +326,6 @@ impl<S: StateStore> OverWindowExecutor<S> {
             BTreeMap::new();
         let mut chunk_builder = StreamChunkBuilder::new(this.chunk_size, this.schema.data_types());
 
-        // Prepare things needed by metrics.
-        let actor_id = this.actor_ctx.id.to_string();
-        let fragment_id = this.actor_ctx.fragment_id.to_string();
-        let table_id = this.state_table.table_id().to_string();
-
         // Build final changes partition by partition.
         for (part_key, delta) in deltas {
             vars.stats.cache_lookup += 1;
@@ -365,13 +376,17 @@ impl<S: StateStore> OverWindowExecutor<S> {
                                 }
                             }
                             (existed, record) => {
-                                let vnode = this.state_table.compute_vnode_by_pk(&key.pk);
-                                let raw_key = serialize_pk_with_vnode(
-                                    &key.pk,
-                                    this.state_table.pk_serde(),
-                                    vnode,
+                                // when stream is inconsistent, there may be an `Update` of which the old pk does not actually exist
+                                consistency_panic!(
+                                    ?existed,
+                                    ?record,
+                                    "other cases should not exist",
                                 );
-                                panic!("other cases should not exist. raw_key: {:?}, existed: {:?}, new: {:?}", raw_key, existed, record);
+
+                                key_change_update_buffer.insert(pk, record);
+                                if let Some(chunk) = chunk_builder.append_record(existed) {
+                                    yield chunk;
+                                }
                             }
                         }
                     } else {
@@ -383,24 +398,28 @@ impl<S: StateStore> OverWindowExecutor<S> {
                 partition.write_record(&mut this.state_table, key, record);
             }
 
+            if !key_change_update_buffer.is_empty() {
+                consistency_panic!(
+                    ?key_change_update_buffer,
+                    "key-change update buffer should be empty after processing"
+                );
+                // if in non-strict mode, we can reach here, but we don't know the `StateKey`,
+                // so just ignore the buffer.
+            }
+
             let cache_len = partition.cache_real_len();
             let stats = partition.summarize();
-            let metrics = this.actor_ctx.streaming_metrics.clone();
             metrics
                 .over_window_range_cache_entry_count
-                .with_label_values(&[&table_id, &actor_id, &fragment_id])
                 .set(cache_len as i64);
             metrics
                 .over_window_range_cache_lookup_count
-                .with_label_values(&[&table_id, &actor_id, &fragment_id])
                 .inc_by(stats.lookup_count);
             metrics
                 .over_window_range_cache_left_miss_count
-                .with_label_values(&[&table_id, &actor_id, &fragment_id])
                 .inc_by(stats.left_miss_count);
             metrics
                 .over_window_range_cache_right_miss_count
-                .with_label_values(&[&table_id, &actor_id, &fragment_id])
                 .inc_by(stats.right_miss_count);
 
             // Update recently accessed range for later shrinking cache.
@@ -436,21 +455,17 @@ impl<S: StateStore> OverWindowExecutor<S> {
     async fn build_changes_for_partition(
         this: &ExecutorInner<S>,
         partition: &mut OverPartition<'_, S>,
-        delta: PartitionDelta,
+        mut delta: PartitionDelta,
     ) -> StreamExecutorResult<(
         BTreeMap<StateKey, Record<OwnedRow>>,
         Option<RangeInclusive<StateKey>>,
     )> {
-        assert!(!delta.is_empty(), "if there's no delta, we won't be here");
-
         let mut part_changes = BTreeMap::new();
 
         // Find affected ranges, this also ensures that all rows in the affected ranges are loaded
         // into the cache.
-        // TODO(rc): maybe we can find affected ranges for each window function call (each frame) to simplify
-        // the implementation of `find_affected_ranges`
         let (part_with_delta, affected_ranges) = partition
-            .find_affected_ranges(&this.state_table, &delta)
+            .find_affected_ranges(&this.state_table, &mut delta)
             .await?;
 
         let snapshot = part_with_delta.snapshot();
@@ -600,8 +615,17 @@ impl<S: StateStore> OverWindowExecutor<S> {
             "OverWindow",
         );
 
+        let metrics = metrics_info.metrics.new_over_window_metrics(
+            this.state_table.table_id(),
+            this.actor_ctx.id,
+            this.actor_ctx.fragment_id,
+        );
+
         let mut vars = ExecutionVars {
-            cached_partitions: new_unbounded(this.watermark_epoch.clone(), metrics_info),
+            cached_partitions: ManagedLruCache::unbounded(
+                this.watermark_sequence.clone(),
+                metrics_info,
+            ),
             recently_accessed_ranges: Default::default(),
             stats: Default::default(),
             _phantom: PhantomData::<S>,
@@ -610,7 +634,6 @@ impl<S: StateStore> OverWindowExecutor<S> {
         let mut input = input.execute();
         let barrier = expect_first_barrier(&mut input).await?;
         this.state_table.init_epoch(barrier.epoch);
-        vars.cached_partitions.update_epoch(barrier.epoch.curr);
 
         yield Message::Barrier(barrier);
 
@@ -625,7 +648,7 @@ impl<S: StateStore> OverWindowExecutor<S> {
                 }
                 Message::Chunk(chunk) => {
                     #[for_await]
-                    for chunk in Self::apply_chunk(&mut this, &mut vars, chunk) {
+                    for chunk in Self::apply_chunk(&mut this, &mut vars, chunk, &metrics) {
                         yield Message::Chunk(chunk?);
                     }
                     this.state_table.try_flush().await?;
@@ -634,24 +657,15 @@ impl<S: StateStore> OverWindowExecutor<S> {
                     this.state_table.commit(barrier.epoch).await?;
                     vars.cached_partitions.evict();
 
-                    {
-                        // update metrics
-                        let actor_id_str = this.actor_ctx.id.to_string();
-                        let fragment_id_str = this.actor_ctx.fragment_id.to_string();
-                        let table_id_str = this.state_table.table_id().to_string();
-                        this.metrics
-                            .over_window_cached_entry_count
-                            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-                            .set(vars.cached_partitions.len() as _);
-                        this.metrics
-                            .over_window_cache_lookup_count
-                            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-                            .inc_by(std::mem::take(&mut vars.stats.cache_lookup));
-                        this.metrics
-                            .over_window_cache_miss_count
-                            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-                            .inc_by(std::mem::take(&mut vars.stats.cache_miss));
-                    }
+                    metrics
+                        .over_window_cached_entry_count
+                        .set(vars.cached_partitions.len() as _);
+                    metrics
+                        .over_window_cache_lookup_count
+                        .inc_by(std::mem::take(&mut vars.stats.cache_lookup));
+                    metrics
+                        .over_window_cache_miss_count
+                        .inc_by(std::mem::take(&mut vars.stats.cache_miss));
 
                     if let Some(vnode_bitmap) = barrier.as_update_vnode_bitmap(this.actor_ctx.id) {
                         let (_, cache_may_stale) =
@@ -679,8 +693,6 @@ impl<S: StateStore> OverWindowExecutor<S> {
                         }
                     }
 
-                    vars.cached_partitions.update_epoch(barrier.epoch.curr);
-
                     yield Message::Barrier(barrier);
                 }
             }
diff --git a/src/stream/src/executor/over_window/over_partition.rs b/src/stream/src/executor/over_window/over_partition.rs
index db431a17dafe4..a00f86aff9082 100644
--- a/src/stream/src/executor/over_window/over_partition.rs
+++ b/src/stream/src/executor/over_window/over_partition.rs
@@ -32,9 +32,11 @@ use risingwave_expr::window_function::{
 };
 use risingwave_storage::store::PrefetchOptions;
 use risingwave_storage::StateStore;
+use static_assertions::const_assert;
 
 use super::general::RowConverter;
 use crate::common::table::state_table::StateTable;
+use crate::consistency::{consistency_error, enable_strict_consistency};
 use crate::executor::over_window::frame_finder::*;
 use crate::executor::StreamExecutorResult;
 
@@ -88,27 +90,31 @@ pub(super) fn shrink_partition_cache(
             let (sk_start, sk_end) = recently_accessed_range.into_inner();
             let (ck_start, ck_end) = (CacheKey::from(sk_start), CacheKey::from(sk_end));
 
+            // find the cursor just before `ck_start`
             let mut cursor = range_cache.inner().upper_bound(Bound::Excluded(&ck_start));
             for _ in 0..MAGIC_JITTER_PREVENTION {
-                if cursor.key().is_none() {
+                if cursor.prev().is_none() {
+                    // already at the beginning
                     break;
                 }
-                cursor.move_prev();
             }
             let start = cursor
-                .key()
+                .peek_prev()
+                .map(|(k, _)| k)
                 .unwrap_or_else(|| range_cache.first_key_value().unwrap().0)
                 .clone();
 
+            // find the cursor just after `ck_end`
             let mut cursor = range_cache.inner().lower_bound(Bound::Excluded(&ck_end));
             for _ in 0..MAGIC_JITTER_PREVENTION {
-                if cursor.key().is_none() {
+                if cursor.next().is_none() {
+                    // already at the end
                     break;
                 }
-                cursor.move_next();
             }
             let end = cursor
-                .key()
+                .peek_next()
+                .map(|(k, _)| k)
                 .unwrap_or_else(|| range_cache.last_key_value().unwrap().0)
                 .clone();
 
@@ -122,32 +128,39 @@ pub(super) fn shrink_partition_cache(
                 let (sk_start, _sk_end) = recently_accessed_range.into_inner();
                 let ck_start = CacheKey::from(sk_start);
 
-                let mut capacity_remain = MAGIC_CACHE_SIZE; // precision is not important here, code simplicity is first
+                let mut capacity_remain = MAGIC_CACHE_SIZE; // precision is not important here, code simplicity is the first
+                const_assert!(MAGIC_JITTER_PREVENTION < MAGIC_CACHE_SIZE);
 
-                let mut cursor = range_cache.inner().upper_bound(Bound::Excluded(&ck_start));
+                // find the cursor just before `ck_start`
+                let cursor_just_before_ck_start =
+                    range_cache.inner().upper_bound(Bound::Excluded(&ck_start));
+
+                let mut cursor = cursor_just_before_ck_start.clone();
                 // go back for at most `MAGIC_JITTER_PREVENTION` entries
                 for _ in 0..MAGIC_JITTER_PREVENTION {
-                    if cursor.key().is_none() {
+                    if cursor.prev().is_none() {
+                        // already at the beginning
                         break;
                     }
-                    cursor.move_prev();
                     capacity_remain -= 1;
                 }
                 let start = cursor
-                    .key()
+                    .peek_prev()
+                    .map(|(k, _)| k)
                     .unwrap_or_else(|| range_cache.first_key_value().unwrap().0)
                     .clone();
 
-                let mut cursor = range_cache.inner().lower_bound(Bound::Included(&ck_start));
+                let mut cursor = cursor_just_before_ck_start;
                 // go forward for at most `capacity_remain` entries
                 for _ in 0..capacity_remain {
-                    if cursor.key().is_none() {
+                    if cursor.next().is_none() {
+                        // already at the end
                         break;
                     }
-                    cursor.move_next();
                 }
                 let end = cursor
-                    .key()
+                    .peek_next()
+                    .map(|(k, _)| k)
                     .unwrap_or_else(|| range_cache.last_key_value().unwrap().0)
                     .clone();
 
@@ -162,32 +175,39 @@ pub(super) fn shrink_partition_cache(
                 let (_sk_start, sk_end) = recently_accessed_range.into_inner();
                 let ck_end = CacheKey::from(sk_end);
 
-                let mut capacity_remain = MAGIC_CACHE_SIZE; // precision is not important here, code simplicity is first
+                let mut capacity_remain = MAGIC_CACHE_SIZE; // precision is not important here, code simplicity is the first
+                const_assert!(MAGIC_JITTER_PREVENTION < MAGIC_CACHE_SIZE);
+
+                // find the cursor just after `ck_end`
+                let cursor_just_after_ck_end =
+                    range_cache.inner().lower_bound(Bound::Excluded(&ck_end));
 
-                let mut cursor = range_cache.inner().lower_bound(Bound::Excluded(&ck_end));
+                let mut cursor = cursor_just_after_ck_end.clone();
                 // go forward for at most `MAGIC_JITTER_PREVENTION` entries
                 for _ in 0..MAGIC_JITTER_PREVENTION {
-                    if cursor.key().is_none() {
+                    if cursor.next().is_none() {
+                        // already at the end
                         break;
                     }
-                    cursor.move_next();
                     capacity_remain -= 1;
                 }
                 let end = cursor
-                    .key()
+                    .peek_next()
+                    .map(|(k, _)| k)
                     .unwrap_or_else(|| range_cache.last_key_value().unwrap().0)
                     .clone();
 
-                let mut cursor = range_cache.inner().upper_bound(Bound::Included(&ck_end));
+                let mut cursor = cursor_just_after_ck_end;
                 // go back for at most `capacity_remain` entries
                 for _ in 0..capacity_remain {
-                    if cursor.key().is_none() {
+                    if cursor.prev().is_none() {
+                        // already at the beginning
                         break;
                     }
-                    cursor.move_prev();
                 }
                 let start = cursor
-                    .key()
+                    .peek_prev()
+                    .map(|(k, _)| k)
                     .unwrap_or_else(|| range_cache.first_key_value().unwrap().0)
                     .clone();
 
@@ -230,25 +250,25 @@ pub(super) struct OverPartitionStats {
 /// represented by [`DeltaBTreeMap`].
 ///
 /// - `first_curr_key` and `last_curr_key` are the current keys of the first and the last
-///  windows affected. They are used to pinpoint the bounds where state needs to be updated.
+///   windows affected. They are used to pinpoint the bounds where state needs to be updated.
 /// - `first_frame_start` and `last_frame_end` are the frame start and end of the first and
-///  the last windows affected. They are used to pinpoint the bounds where state needs to be
-///  included for computing the new state.
+///   the last windows affected. They are used to pinpoint the bounds where state needs to be
+///   included for computing the new state.
 #[derive(Debug, Educe)]
 #[educe(Clone, Copy)]
-pub(super) struct AffectedRange<'cache> {
-    pub first_frame_start: &'cache CacheKey,
-    pub first_curr_key: &'cache CacheKey,
-    pub last_curr_key: &'cache CacheKey,
-    pub last_frame_end: &'cache CacheKey,
+pub(super) struct AffectedRange<'a> {
+    pub first_frame_start: &'a CacheKey,
+    pub first_curr_key: &'a CacheKey,
+    pub last_curr_key: &'a CacheKey,
+    pub last_frame_end: &'a CacheKey,
 }
 
-impl<'cache> AffectedRange<'cache> {
+impl<'a> AffectedRange<'a> {
     fn new(
-        first_frame_start: &'cache CacheKey,
-        first_curr_key: &'cache CacheKey,
-        last_curr_key: &'cache CacheKey,
-        last_frame_end: &'cache CacheKey,
+        first_frame_start: &'a CacheKey,
+        first_curr_key: &'a CacheKey,
+        last_curr_key: &'a CacheKey,
+        last_frame_end: &'a CacheKey,
     ) -> Self {
         Self {
             first_frame_start,
@@ -417,35 +437,31 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
     /// Find all ranges in the partition that are affected by the given delta.
     /// The returned ranges are guaranteed to be sorted and non-overlapping. All keys in the ranges
     /// are guaranteed to be cached, which means they should be [`Sentinelled::Normal`]s.
-    pub async fn find_affected_ranges<'s, 'cache>(
+    pub async fn find_affected_ranges<'s, 'delta>(
         &'s mut self,
-        table: &'_ StateTable<S>,
-        delta: &'cache PartitionDelta,
+        table: &StateTable<S>,
+        delta: &'delta mut PartitionDelta,
     ) -> StreamExecutorResult<(
-        DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
-        Vec<AffectedRange<'cache>>,
+        DeltaBTreeMap<'delta, CacheKey, OwnedRow>,
+        Vec<AffectedRange<'delta>>,
     )>
     where
-        's: 'cache,
+        'a: 'delta,
+        's: 'delta,
     {
+        self.ensure_delta_in_cache(table, delta).await?;
+        let delta = &*delta; // let's make it immutable
+
+        if delta.is_empty() {
+            return Ok((DeltaBTreeMap::new(self.range_cache.inner(), delta), vec![]));
+        }
+
         let delta_first = delta.first_key_value().unwrap().0.as_normal_expect();
         let delta_last = delta.last_key_value().unwrap().0.as_normal_expect();
 
         let range_frame_logical_curr =
             calc_logical_curr_for_range_frames(&self.range_frames, delta_first, delta_last);
 
-        if self.cache_policy.is_full() {
-            // ensure everything is in the cache
-            self.extend_cache_to_boundary(table).await?;
-        } else {
-            // TODO(rc): later we should extend cache using `self.super_rows_frame_bounds` and
-            // `range_frame_logical_curr` as hints.
-
-            // ensure the cache covers all delta (if possible)
-            self.extend_cache_by_range(table, delta_first..=delta_last)
-                .await?;
-        }
-
         loop {
             // TERMINATEABILITY: `extend_cache_leftward_by_n` and `extend_cache_rightward_by_n` keep
             // pushing the cache to the boundary of current partition. In these two methods, when
@@ -453,7 +469,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             // `Self::find_affected_ranges_readonly` will return `Ok`.
 
             // SAFETY: Here we shortly borrow the range cache and turn the reference into a
-            // `'cache` one to bypass the borrow checker. This is safe because we only return
+            // `'delta` one to bypass the borrow checker. This is safe because we only return
             // the reference once we don't need to do any further mutation.
             let cache_inner = unsafe { &*(self.range_cache.inner() as *const _) };
             let part_with_delta = DeltaBTreeMap::new(cache_inner, delta);
@@ -483,6 +499,53 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
         }
     }
 
+    async fn ensure_delta_in_cache(
+        &mut self,
+        table: &StateTable<S>,
+        delta: &mut PartitionDelta,
+    ) -> StreamExecutorResult<()> {
+        if delta.is_empty() {
+            return Ok(());
+        }
+
+        let delta_first = delta.first_key_value().unwrap().0.as_normal_expect();
+        let delta_last = delta.last_key_value().unwrap().0.as_normal_expect();
+
+        if self.cache_policy.is_full() {
+            // ensure everything is in the cache
+            self.extend_cache_to_boundary(table).await?;
+        } else {
+            // TODO(rc): later we should extend cache using `self.super_rows_frame_bounds` and
+            // `range_frame_logical_curr` as hints.
+
+            // ensure the cache covers all delta (if possible)
+            self.extend_cache_by_range(table, delta_first..=delta_last)
+                .await?;
+        }
+
+        if !enable_strict_consistency() {
+            // in non-strict mode, we should ensure the delta is consistent with the cache
+            let cache = self.range_cache.inner();
+            delta.retain(|key, change| match &*change {
+                Change::Insert(_) => {
+                    // this also includes the case of double-insert and ghost-update,
+                    // but since we already lost the information, let's just ignore it
+                    true
+                }
+                Change::Delete => {
+                    // if the key is not in the cache, it's a ghost-delete
+                    let consistent = cache.contains_key(key);
+                    if !consistent {
+                        consistency_error!(?key, "removing a row with non-existing key");
+                    }
+                    consistent
+                }
+            });
+        }
+
+        Ok(())
+    }
+
     /// Try to find affected ranges on immutable range cache + delta. If the algorithm reaches
     /// any sentinel node in the cache, which means some entries in the affected range may be
     /// in the state table, it returns an `Err((bool, bool))` to notify the caller that the
@@ -491,11 +554,11 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
     /// TODO(rc): Currently at most one range will be in the result vector. Ideally we should
     /// recognize uncontinuous changes in the delta and find multiple ranges, but that will be
     /// too complex for now.
-    fn find_affected_ranges_readonly<'cache>(
-        &'_ self,
-        part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
+    fn find_affected_ranges_readonly<'delta>(
+        &self,
+        part_with_delta: DeltaBTreeMap<'delta, CacheKey, OwnedRow>,
         range_frame_logical_curr: Option<&(Sentinelled<Datum>, Sentinelled<Datum>)>,
-    ) -> std::result::Result<Vec<AffectedRange<'cache>>, (bool, bool)> {
+    ) -> std::result::Result<Vec<AffectedRange<'delta>>, (bool, bool)> {
         if part_with_delta.first_key().is_none() {
             // nothing is left after applying the delta, meaning all entries are deleted
             return Ok(vec![]);
diff --git a/src/stream/src/executor/project.rs b/src/stream/src/executor/project.rs
index e78238585c9fa..6ea579afea524 100644
--- a/src/stream/src/executor/project.rs
+++ b/src/stream/src/executor/project.rs
@@ -44,6 +44,10 @@ struct Inner {
     /// the selectivity threshold which should be in `[0,1]`. for the chunk with selectivity less
     /// than the threshold, the Project executor will construct a new chunk before expr evaluation,
     materialize_selectivity_threshold: f64,
+
+    /// Whether there are likely no-op updates in the output chunks, so that eliminating them with
+    /// `StreamChunk::eliminate_adjacent_noop_update` could be beneficial.
+    noop_update_hint: bool,
 }
 
 impl ProjectExecutor {
@@ -55,6 +59,7 @@ impl ProjectExecutor {
         watermark_derivations: MultiMap<usize, usize>,
         nondecreasing_expr_indices: Vec<usize>,
         materialize_selectivity_threshold: f64,
+        noop_update_hint: bool,
     ) -> Self {
         let n_nondecreasing_exprs = nondecreasing_expr_indices.len();
         Self {
@@ -66,6 +71,7 @@ impl ProjectExecutor {
                 nondecreasing_expr_indices,
                 last_nondec_expr_values: vec![None; n_nondecreasing_exprs],
                 materialize_selectivity_threshold,
+                noop_update_hint,
             },
         }
     }
@@ -103,7 +109,11 @@ impl Inner {
             projected_columns.push(evaluated_expr);
         }
         let (_, vis) = data_chunk.into_parts();
-        let new_chunk = StreamChunk::with_visibility(ops, projected_columns, vis);
+
+        let mut new_chunk = StreamChunk::with_visibility(ops, projected_columns, vis);
+        if self.noop_update_hint {
+            new_chunk = new_chunk.eliminate_adjacent_noop_update();
+        }
         Ok(Some(new_chunk))
     }
 
@@ -189,13 +199,11 @@ impl Inner {
 mod tests {
     use std::sync::atomic::{self, AtomicI64};
 
-    use futures::StreamExt;
     use risingwave_common::array::stream_chunk::StreamChunkTestExt;
-    use risingwave_common::array::{DataChunk, StreamChunk};
-    use risingwave_common::catalog::{Field, Schema};
-    use risingwave_common::types::{DataType, Datum};
+    use risingwave_common::array::DataChunk;
+    use risingwave_common::catalog::Field;
     use risingwave_common::util::epoch::test_epoch;
-    use risingwave_expr::expr::{self, Expression, ValueImpl};
+    use risingwave_expr::expr::{self, ValueImpl};
 
     use super::super::test_utils::MockSource;
     use super::super::*;
@@ -235,6 +243,7 @@ mod tests {
             MultiMap::new(),
             vec![],
             0.0,
+            false,
         );
         let mut project = project.boxed().execute();
 
@@ -317,6 +326,7 @@ mod tests {
             MultiMap::from_iter(vec![(0, 0), (0, 1)].into_iter()),
             vec![2],
             0.0,
+            false,
         );
         let mut project = project.boxed().execute();
 
diff --git a/src/stream/src/executor/project_set.rs b/src/stream/src/executor/project_set.rs
index 43a2d65cbfb11..8f5c0e533bfbd 100644
--- a/src/stream/src/executor/project_set.rs
+++ b/src/stream/src/executor/project_set.rs
@@ -14,15 +14,19 @@
 
 use either::Either;
 use multimap::MultiMap;
-use risingwave_common::array::Op;
+use risingwave_common::array::{ArrayRef, DataChunk, Op};
 use risingwave_common::bail;
 use risingwave_common::row::RowExt;
 use risingwave_common::types::ToOwnedDatum;
 use risingwave_common::util::iter_util::ZipEqFast;
-use risingwave_expr::expr::{LogReport, NonStrictExpression};
-use risingwave_expr::table_function::ProjectSetSelectItem;
+use risingwave_expr::expr::{self, EvalErrorReport, NonStrictExpression};
+use risingwave_expr::table_function::{self, BoxedTableFunction, TableFunctionOutputIter};
+use risingwave_expr::ExprError;
+use risingwave_pb::expr::project_set_select_item::PbSelectItem;
+use risingwave_pb::expr::PbProjectSetSelectItem;
 
 use crate::executor::prelude::*;
+use crate::task::ActorEvalErrorReport;
 
 const PROJ_ROW_ID_OFFSET: usize = 1;
 
@@ -45,6 +49,7 @@ struct Inner {
     watermark_derivations: MultiMap<usize, usize>,
     /// Indices of nondecreasing expressions in the expression list.
     nondecreasing_expr_indices: Vec<usize>,
+    error_report: ActorEvalErrorReport,
 }
 
 impl ProjectSetExecutor {
@@ -56,6 +61,7 @@ impl ProjectSetExecutor {
         chunk_size: usize,
         watermark_derivations: MultiMap<usize, usize>,
         nondecreasing_expr_indices: Vec<usize>,
+        error_report: ActorEvalErrorReport,
     ) -> Self {
         let inner = Inner {
             _ctx: ctx,
@@ -63,6 +69,7 @@ impl ProjectSetExecutor {
             chunk_size,
             watermark_derivations,
             nondecreasing_expr_indices,
+            error_report,
         };
 
         Self { input, inner }
@@ -152,11 +159,19 @@ impl Inner {
                             for (item, value) in results.iter_mut().zip_eq_fast(&mut row[1..]) {
                                 *value = match item {
                                     Either::Left(state) => {
-                                        if let Some((i, value)) = state.peek()
+                                        if let Some((i, result)) = state.peek()
                                             && i == row_idx
                                         {
-                                            valid = true;
-                                            value
+                                            match result {
+                                                Ok(value) => {
+                                                    valid = true;
+                                                    value
+                                                }
+                                                Err(err) => {
+                                                    self.error_report.report(err);
+                                                    None
+                                                }
+                                            }
                                         } else {
                                             None
                                         }
@@ -226,17 +241,13 @@ impl Inner {
         for expr_idx in expr_indices {
             let expr_idx = *expr_idx;
             let derived_watermark = match &self.select_list[expr_idx] {
-                ProjectSetSelectItem::Expr(expr) => {
+                ProjectSetSelectItem::Scalar(expr) => {
                     watermark
                         .clone()
-                        .transform_with_expr(
-                            // TODO: should we build `expr` in non-strict mode?
-                            &NonStrictExpression::new_topmost(expr, LogReport),
-                            expr_idx + PROJ_ROW_ID_OFFSET,
-                        )
+                        .transform_with_expr(expr, expr_idx + PROJ_ROW_ID_OFFSET)
                         .await
                 }
-                ProjectSetSelectItem::TableFunction(_) => {
+                ProjectSetSelectItem::Set(_) => {
                     bail!("Watermark should not be produced by a table function");
                 }
             };
@@ -253,3 +264,63 @@ impl Inner {
         Ok(ret)
     }
 }
+
+/// Either a scalar expression or a set-returning function.
+///
+/// See also [`PbProjectSetSelectItem`].
+///
+/// A similar enum is defined in the `batch` module. The difference is that
+/// we use `NonStrictExpression` instead of `BoxedExpression` here.
+#[derive(Debug)]
+pub enum ProjectSetSelectItem {
+    Scalar(NonStrictExpression),
+    Set(BoxedTableFunction),
+}
+
+impl From<BoxedTableFunction> for ProjectSetSelectItem {
+    fn from(table_function: BoxedTableFunction) -> Self {
+        ProjectSetSelectItem::Set(table_function)
+    }
+}
+
+impl From<NonStrictExpression> for ProjectSetSelectItem {
+    fn from(expr: NonStrictExpression) -> Self {
+        ProjectSetSelectItem::Scalar(expr)
+    }
+}
+
+impl ProjectSetSelectItem {
+    pub fn from_prost(
+        prost: &PbProjectSetSelectItem,
+        error_report: impl EvalErrorReport + 'static,
+        chunk_size: usize,
+    ) -> Result<Self, ExprError> {
+        match prost.select_item.as_ref().unwrap() {
+            PbSelectItem::Expr(expr) => {
+                expr::build_non_strict_from_prost(expr, error_report).map(Self::Scalar)
+            }
+            PbSelectItem::TableFunction(tf) => {
+                table_function::build_from_prost(tf, chunk_size).map(Self::Set)
+            }
+        }
+    }
+
+    pub fn return_type(&self) -> DataType {
+        match self {
+            ProjectSetSelectItem::Scalar(expr) => expr.return_type(),
+            ProjectSetSelectItem::Set(tf) => tf.return_type(),
+        }
+    }
+
+    pub async fn eval<'a>(
+        &'a self,
+        input: &'a DataChunk,
+    ) -> Result<Either<TableFunctionOutputIter<'a>, ArrayRef>, ExprError> {
+        match self {
+            Self::Scalar(expr) => Ok(Either::Right(expr.eval_infallible(input).await)),
+            Self::Set(tf) => Ok(Either::Left(
+                TableFunctionOutputIter::new(tf.eval(input).await).await?,
+            )),
+        }
+    }
+}
diff --git a/src/stream/src/executor/receiver.rs b/src/stream/src/executor/receiver.rs
index 3966b25febf5d..effe773d54594 100644
--- a/src/stream/src/executor/receiver.rs
+++ b/src/stream/src/executor/receiver.rs
@@ -19,7 +19,6 @@ use tokio::time::Instant;
 use super::exchange::input::BoxedInput;
 use crate::executor::exchange::input::new_input;
 use crate::executor::prelude::*;
-use crate::executor::utils::ActorInputMetrics;
 use crate::task::{FragmentId, SharedContext};
 
 /// `ReceiverExecutor` is used along with a channel. After creating a mpsc channel,
@@ -93,8 +92,7 @@ impl Execute for ReceiverExecutor {
     fn execute(mut self: Box<Self>) -> BoxedMessageStream {
         let actor_id = self.actor_context.id;
 
-        let mut metrics = ActorInputMetrics::new(
-            &self.metrics,
+        let mut metrics = self.metrics.new_actor_input_metrics(
             actor_id,
             self.fragment_id,
             self.upstream_fragment_id,
@@ -169,8 +167,7 @@ impl Execute for ReceiverExecutor {
                             self.input = new_upstream;
 
                             self.upstream_fragment_id = new_upstream_fragment_id;
-                            metrics = ActorInputMetrics::new(
-                                &self.metrics,
+                            metrics = self.metrics.new_actor_input_metrics(
                                 actor_id,
                                 self.fragment_id,
                                 self.upstream_fragment_id,
@@ -191,15 +188,13 @@ impl Execute for ReceiverExecutor {
 #[cfg(test)]
 mod tests {
     use std::collections::HashMap;
-    use std::sync::Arc;
 
     use futures::{pin_mut, FutureExt};
-    use risingwave_common::array::StreamChunk;
     use risingwave_common::util::epoch::test_epoch;
     use risingwave_pb::stream_plan::update_mutation::MergeUpdate;
 
     use super::*;
-    use crate::executor::{ActorContext, Barrier, Execute, Mutation, UpdateMutation};
+    use crate::executor::UpdateMutation;
     use crate::task::test_utils::helper_make_local_actor;
 
     #[tokio::test]
diff --git a/src/stream/src/executor/row_id_gen.rs b/src/stream/src/executor/row_id_gen.rs
index c779092a307fb..2da75389fd9f7 100644
--- a/src/stream/src/executor/row_id_gen.rs
+++ b/src/stream/src/executor/row_id_gen.rs
@@ -123,16 +123,14 @@ impl Execute for RowIdGenExecutor {
 
 #[cfg(test)]
 mod tests {
-    use risingwave_common::array::{Array, PrimitiveArray};
-    use risingwave_common::catalog::{Field, Schema};
+    use risingwave_common::array::PrimitiveArray;
+    use risingwave_common::catalog::Field;
     use risingwave_common::hash::VirtualNode;
     use risingwave_common::test_prelude::StreamChunkTestExt;
-    use risingwave_common::types::DataType;
     use risingwave_common::util::epoch::test_epoch;
 
     use super::*;
     use crate::executor::test_utils::MockSource;
-    use crate::executor::{ActorContext, Execute};
 
     #[tokio::test]
     async fn test_row_id_gen_executor() {
diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs
index 9c35627857828..3ed0c2533e45d 100644
--- a/src/stream/src/executor/simple_agg.rs
+++ b/src/stream/src/executor/simple_agg.rs
@@ -232,15 +232,12 @@ impl<S: StateStore> SimpleAggExecutor<S> {
             table.init_epoch(barrier.epoch);
         });
 
-        let mut distinct_dedup = DistinctDeduplicater::new(
+        let distinct_dedup = DistinctDeduplicater::new(
             &this.agg_calls,
             this.watermark_epoch.clone(),
             &this.distinct_dedup_tables,
-            this.actor_ctx.clone(),
+            &this.actor_ctx,
         );
-        distinct_dedup.dedup_caches_mut().for_each(|cache| {
-            cache.update_epoch(barrier.epoch.curr);
-        });
 
         yield Message::Barrier(barrier);
 
@@ -278,9 +275,6 @@ impl<S: StateStore> SimpleAggExecutor<S> {
                     {
                         yield Message::Chunk(chunk);
                     }
-                    vars.distinct_dedup.dedup_caches_mut().for_each(|cache| {
-                        cache.update_epoch(barrier.epoch.curr);
-                    });
                     yield Message::Barrier(barrier);
                 }
             }
@@ -295,9 +289,7 @@ mod tests {
     use risingwave_common::catalog::Field;
     use risingwave_common::types::*;
     use risingwave_common::util::epoch::test_epoch;
-    use risingwave_expr::aggregate::AggCall;
     use risingwave_storage::memory::MemoryStateStore;
-    use risingwave_storage::StateStore;
 
     use super::*;
     use crate::executor::test_utils::agg_executor::new_boxed_simple_agg_executor;
diff --git a/src/stream/src/executor/sink.rs b/src/stream/src/executor/sink.rs
index 858fe39bd1b8e..e8dfc09e9130c 100644
--- a/src/stream/src/executor/sink.rs
+++ b/src/stream/src/executor/sink.rs
@@ -21,7 +21,9 @@ use itertools::Itertools;
 use risingwave_common::array::stream_chunk::StreamChunkMut;
 use risingwave_common::array::Op;
 use risingwave_common::catalog::{ColumnCatalog, Field};
-use risingwave_common::metrics::GLOBAL_ERROR_METRICS;
+use risingwave_common::metrics::{LabelGuardedIntGauge, GLOBAL_ERROR_METRICS};
+use risingwave_common_estimate_size::collections::EstimatedVec;
+use risingwave_common_estimate_size::EstimateSize;
 use risingwave_connector::dispatch_sink;
 use risingwave_connector::sink::catalog::SinkType;
 use risingwave_connector::sink::log_store::{
@@ -122,8 +124,14 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
     fn execute_inner(self) -> BoxedMessageStream {
         let sink_id = self.sink_param.sink_id;
         let actor_id = self.actor_context.id;
+        let fragment_id = self.actor_context.fragment_id;
 
         let stream_key = self.info.pk_indices.clone();
+        let metrics = self.actor_context.streaming_metrics.new_sink_exec_metrics(
+            sink_id,
+            actor_id,
+            fragment_id,
+        );
 
         let stream_key_sink_pk_mismatch = {
             stream_key
@@ -133,19 +141,9 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
 
         let input = self.input.execute();
 
-        let input_row_count = self
-            .actor_context
-            .streaming_metrics
-            .sink_input_row_count
-            .with_guarded_label_values(&[
-                &sink_id.to_string(),
-                &actor_id.to_string(),
-                &self.actor_context.fragment_id.to_string(),
-            ]);
-
         let input = input.inspect_ok(move |msg| {
             if let Message::Chunk(c) = msg {
-                input_row_count.inc_by(c.capacity() as u64);
+                metrics.sink_input_row_count.inc_by(c.capacity() as u64);
             }
         });
 
@@ -194,6 +192,7 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
             self.chunk_size,
             self.input_data_types,
             self.sink_param.downstream_pk.clone(),
+            metrics.sink_chunk_buffer_size,
         );
 
         if self.sink.is_sink_into_table() {
@@ -216,7 +215,10 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
                             self.sink_param,
                             self.sink_writer_param,
                             self.actor_context,
-                        );
+                        )
+                        .instrument_await(format!("consume_log (sink_id {sink_id})"))
+                        .map_ok(|never| match never {}); // unify return type to `Message`
+
                         // TODO: may try to remove the boxed
                         select(consume_log_stream.into_stream(), write_log_stream).boxed()
                     })
@@ -295,10 +297,11 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
         chunk_size: usize,
         input_data_types: Vec<DataType>,
         down_stream_pk: Vec<usize>,
+        sink_chunk_buffer_size_metrics: LabelGuardedIntGauge<3>,
     ) {
         // need to buffer chunks during one barrier
         if need_advance_delete || re_construct_with_sink_pk {
-            let mut chunk_buffer = vec![];
+            let mut chunk_buffer = EstimatedVec::new();
             let mut watermark: Option<super::Watermark> = None;
             #[for_await]
             for msg in input {
@@ -306,9 +309,11 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
                     Message::Watermark(w) => watermark = Some(w),
                     Message::Chunk(c) => {
                         chunk_buffer.push(c);
+
+                        sink_chunk_buffer_size_metrics.set(chunk_buffer.estimated_size() as i64);
                     }
                     Message::Barrier(barrier) => {
-                        let chunks = mem::take(&mut chunk_buffer);
+                        let chunks = mem::take(&mut chunk_buffer).into_inner();
                         let chunks = if need_advance_delete {
                             let mut delete_chunks = vec![];
                             let mut insert_chunks = vec![];
@@ -361,16 +366,25 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
                         // Compact the chunk to eliminate any useless intermediate result (e.g. UPDATE
                         // V->V).
                         let chunk = merge_chunk_row(chunk, &stream_key);
-                        let chunk = if sink_type == SinkType::ForceAppendOnly {
-                            // Force append-only by dropping UPDATE/DELETE messages. We do this when the
-                            // user forces the sink to be append-only while it is actually not based on
-                            // the frontend derivation result.
-                            force_append_only(chunk)
-                        } else {
-                            chunk
-                        };
-
-                        yield Message::Chunk(chunk);
+                        match sink_type {
+                            SinkType::AppendOnly => yield Message::Chunk(chunk),
+                            SinkType::ForceAppendOnly => {
+                                // Force append-only by dropping UPDATE/DELETE messages. We do this when the
+                                // user forces the sink to be append-only while it is actually not based on
+                                // the frontend derivation result.
+                                yield Message::Chunk(force_append_only(chunk))
+                            }
+                            SinkType::Upsert => {
+                                // Making sure the optimization in https://github.com/risingwavelabs/risingwave/pull/12250 is correct,
+                                // it is needed to do the compaction here.
+                                for chunk in
+                                    StreamChunkCompactor::new(stream_key.clone(), vec![chunk])
+                                        .into_compacted_chunks()
+                                {
+                                    yield Message::Chunk(chunk)
+                                }
+                            }
+                        }
                     }
                     Message::Barrier(barrier) => {
                         yield Message::Barrier(barrier);
@@ -387,7 +401,7 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
         sink_param: SinkParam,
         mut sink_writer_param: SinkWriterParam,
         actor_context: ActorContextRef,
-    ) -> StreamExecutorResult<Message> {
+    ) -> StreamExecutorResult<!> {
         let metrics = sink_writer_param.sink_metrics.clone();
 
         let visible_columns = columns
@@ -460,7 +474,6 @@ mod test {
     use super::*;
     use crate::common::log_store_impl::in_mem::BoundedInMemLogStoreFactory;
     use crate::executor::test_utils::*;
-    use crate::executor::ActorContext;
 
     #[tokio::test]
     async fn test_force_append_only_sink() {
@@ -470,7 +483,7 @@ mod test {
 
         use crate::executor::Barrier;
 
-        let properties = maplit::hashmap! {
+        let properties = maplit::btreemap! {
             "connector".into() => "blackhole".into(),
             "type".into() => "append-only".into(),
             "force_append_only".into() => "true".into()
@@ -596,7 +609,7 @@ mod test {
 
         use crate::executor::Barrier;
 
-        let properties = maplit::hashmap! {
+        let properties = maplit::btreemap! {
             "connector".into() => "blackhole".into(),
         };
 
@@ -719,7 +732,7 @@ mod test {
 
         use crate::executor::Barrier;
 
-        let properties = maplit::hashmap! {
+        let properties = maplit::btreemap! {
             "connector".into() => "blackhole".into(),
             "type".into() => "append-only".into(),
             "force_append_only".into() => "true".into()
diff --git a/src/stream/src/executor/sort.rs b/src/stream/src/executor/sort.rs
index 37d08c746bba8..c2a01c8915a96 100644
--- a/src/stream/src/executor/sort.rs
+++ b/src/stream/src/executor/sort.rs
@@ -143,16 +143,13 @@ impl<S: StateStore> SortExecutor<S> {
 #[cfg(test)]
 mod tests {
     use risingwave_common::array::stream_chunk::StreamChunkTestExt;
-    use risingwave_common::array::StreamChunk;
-    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, Schema, TableId};
-    use risingwave_common::types::DataType;
+    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, TableId};
     use risingwave_common::util::epoch::test_epoch;
     use risingwave_common::util::sort_util::OrderType;
     use risingwave_storage::memory::MemoryStateStore;
 
     use super::*;
     use crate::executor::test_utils::{MessageSender, MockSource, StreamExecutorTestExt};
-    use crate::executor::{ActorContext, BoxedMessageStream, Execute};
 
     async fn create_executor<S: StateStore>(
         sort_column_index: usize,
diff --git a/src/stream/src/executor/source/fetch_executor.rs b/src/stream/src/executor/source/fetch_executor.rs
index 6399b2bc930ce..b4c006469e650 100644
--- a/src/stream/src/executor/source/fetch_executor.rs
+++ b/src/stream/src/executor/source/fetch_executor.rs
@@ -31,11 +31,12 @@ use risingwave_connector::source::{
 use risingwave_storage::store::PrefetchOptions;
 use thiserror_ext::AsReport;
 
-use super::{get_split_offset_col_idx, SourceStateTableHandler, StreamSourceCore};
-use crate::executor::prelude::*;
-use crate::executor::source::{
-    apply_rate_limit, get_split_offset_mapping_from_chunk, prune_additional_cols,
+use super::{
+    apply_rate_limit, get_split_offset_col_idx, get_split_offset_mapping_from_chunk,
+    prune_additional_cols, SourceStateTableHandler, StreamSourceCore,
 };
+use crate::common::rate_limit::limited_chunk_size;
+use crate::executor::prelude::*;
 use crate::executor::stream_reader::StreamReaderWithPause;
 
 const SPLIT_BATCH_SIZE: usize = 1000;
@@ -51,8 +52,8 @@ pub struct FsFetchExecutor<S: StateStore, Src: OpendalSource> {
     /// Upstream list executor.
     upstream: Option<Executor>,
 
-    // control options for connector level
-    source_ctrl_opts: SourceCtrlOpts,
+    /// Rate limit in rows/s.
+    rate_limit_rps: Option<u32>,
 
     _marker: PhantomData<Src>,
 }
@@ -62,13 +63,13 @@ impl<S: StateStore, Src: OpendalSource> FsFetchExecutor<S, Src> {
         actor_ctx: ActorContextRef,
         stream_source_core: StreamSourceCore<S>,
         upstream: Executor,
-        source_ctrl_opts: SourceCtrlOpts,
+        rate_limit_rps: Option<u32>,
     ) -> Self {
         Self {
             actor_ctx,
             stream_source_core: Some(stream_source_core),
             upstream: Some(upstream),
-            source_ctrl_opts,
+            rate_limit_rps,
             _marker: PhantomData,
         }
     }
@@ -80,6 +81,7 @@ impl<S: StateStore, Src: OpendalSource> FsFetchExecutor<S, Src> {
         source_ctx: SourceContext,
         source_desc: &SourceDesc,
         stream: &mut StreamReaderWithPause<BIASED, StreamChunk>,
+        rate_limit_rps: Option<u32>,
     ) -> StreamExecutorResult<()> {
         let mut batch = Vec::with_capacity(SPLIT_BATCH_SIZE);
         'vnodes: for vnode in state_store_handler.state_table.vnodes().iter_vnodes() {
@@ -128,10 +130,15 @@ impl<S: StateStore, Src: OpendalSource> FsFetchExecutor<S, Src> {
             stream.replace_data_stream(stream::pending().boxed());
         } else {
             *splits_on_fetch += batch.len();
-            let batch_reader =
-                Self::build_batched_stream_reader(column_ids, source_ctx, source_desc, Some(batch))
-                    .await?
-                    .map_err(StreamExecutorError::connector_error);
+            let batch_reader = Self::build_batched_stream_reader(
+                column_ids,
+                source_ctx,
+                source_desc,
+                Some(batch),
+                rate_limit_rps,
+            )
+            .await?
+            .map_err(StreamExecutorError::connector_error);
             stream.replace_data_stream(batch_reader);
         }
 
@@ -143,14 +150,14 @@ impl<S: StateStore, Src: OpendalSource> FsFetchExecutor<S, Src> {
         source_ctx: SourceContext,
         source_desc: &SourceDesc,
         batch: SplitBatch,
+        rate_limit_rps: Option<u32>,
     ) -> StreamExecutorResult<BoxChunkSourceStream> {
-        let rate_limit = source_ctx.source_ctrl_opts.rate_limit;
         let stream = source_desc
             .source
-            .to_stream(batch, column_ids, Arc::new(source_ctx))
+            .build_stream(batch, column_ids, Arc::new(source_ctx))
             .await
             .map_err(StreamExecutorError::connector_error)?;
-        Ok(apply_rate_limit(stream, rate_limit).boxed())
+        Ok(apply_rate_limit(stream, rate_limit_rps).boxed())
     }
 
     fn build_source_ctx(
@@ -165,7 +172,10 @@ impl<S: StateStore, Src: OpendalSource> FsFetchExecutor<S, Src> {
             self.actor_ctx.fragment_id,
             source_name.to_owned(),
             source_desc.metrics.clone(),
-            self.source_ctrl_opts.clone(),
+            SourceCtrlOpts {
+                chunk_size: limited_chunk_size(self.rate_limit_rps),
+                rate_limit: self.rate_limit_rps,
+            },
             source_desc.source.config.clone(),
         )
     }
@@ -211,6 +221,7 @@ impl<S: StateStore, Src: OpendalSource> FsFetchExecutor<S, Src> {
             self.build_source_ctx(&source_desc, core.source_id, &core.source_name),
             &source_desc,
             &mut stream,
+            self.rate_limit_rps,
         )
         .await?;
 
@@ -235,10 +246,11 @@ impl<S: StateStore, Src: OpendalSource> FsFetchExecutor<S, Src> {
                                             Mutation::Pause => stream.pause_stream(),
                                             Mutation::Resume => stream.resume_stream(),
                                             Mutation::Throttle(actor_to_apply) => {
-                                                if let Some(throttle) =
+                                                if let Some(new_rate_limit) =
                                                     actor_to_apply.get(&self.actor_ctx.id)
+                                                    && *new_rate_limit != self.rate_limit_rps
                                                 {
-                                                    self.source_ctrl_opts.rate_limit = *throttle;
+                                                    self.rate_limit_rps = *new_rate_limit;
                                                     need_rebuild_reader = true;
                                                 }
                                             }
@@ -277,6 +289,7 @@ impl<S: StateStore, Src: OpendalSource> FsFetchExecutor<S, Src> {
                                             ),
                                             &source_desc,
                                             &mut stream,
+                                            self.rate_limit_rps,
                                         )
                                         .await?;
                                     }
diff --git a/src/stream/src/executor/source/fs_source_executor.rs b/src/stream/src/executor/source/fs_source_executor.rs
index 77e93e086c8fd..32d5d533d904a 100644
--- a/src/stream/src/executor/source/fs_source_executor.rs
+++ b/src/stream/src/executor/source/fs_source_executor.rs
@@ -34,11 +34,12 @@ use tokio::sync::mpsc::UnboundedReceiver;
 use tokio::time::Instant;
 
 use super::executor_core::StreamSourceCore;
-use crate::executor::prelude::*;
-use crate::executor::source::{
+use super::{
     apply_rate_limit, barrier_to_message_stream, get_split_offset_col_idx,
     get_split_offset_mapping_from_chunk, prune_additional_cols,
 };
+use crate::common::rate_limit::limited_chunk_size;
+use crate::executor::prelude::*;
 use crate::executor::stream_reader::StreamReaderWithPause;
 use crate::executor::{AddMutation, UpdateMutation};
 
@@ -63,18 +64,18 @@ pub struct FsSourceExecutor<S: StateStore> {
     /// System parameter reader to read barrier interval
     system_params: SystemParamsReaderRef,
 
-    source_ctrl_opts: SourceCtrlOpts,
+    /// Rate limit in rows/s.
+    rate_limit_rps: Option<u32>,
 }
 
 impl<S: StateStore> FsSourceExecutor<S> {
-    #[allow(clippy::too_many_arguments)]
     pub fn new(
         actor_ctx: ActorContextRef,
         stream_source_core: StreamSourceCore<S>,
         metrics: Arc<StreamingMetrics>,
         barrier_receiver: UnboundedReceiver<Barrier>,
         system_params: SystemParamsReaderRef,
-        source_ctrl_opts: SourceCtrlOpts,
+        rate_limit_rps: Option<u32>,
     ) -> StreamResult<Self> {
         Ok(Self {
             actor_ctx,
@@ -82,7 +83,7 @@ impl<S: StateStore> FsSourceExecutor<S> {
             metrics,
             barrier_receiver: Some(barrier_receiver),
             system_params,
-            source_ctrl_opts,
+            rate_limit_rps,
         })
     }
 
@@ -102,7 +103,10 @@ impl<S: StateStore> FsSourceExecutor<S> {
             self.actor_ctx.fragment_id,
             self.stream_source_core.source_name.clone(),
             source_desc.metrics.clone(),
-            self.source_ctrl_opts.clone(),
+            SourceCtrlOpts {
+                chunk_size: limited_chunk_size(self.rate_limit_rps),
+                rate_limit: self.rate_limit_rps,
+            },
             source_desc.source.config.clone(),
         );
         let stream = source_desc
@@ -111,7 +115,7 @@ impl<S: StateStore> FsSourceExecutor<S> {
             .await
             .map_err(StreamExecutorError::connector_error)?;
 
-        Ok(apply_rate_limit(stream, self.source_ctrl_opts.rate_limit).boxed())
+        Ok(apply_rate_limit(stream, self.rate_limit_rps).boxed())
     }
 
     async fn rebuild_stream_reader<const BIASED: bool>(
@@ -311,18 +315,16 @@ impl<S: StateStore> FsSourceExecutor<S> {
         let start_with_paused = barrier.is_pause_on_startup();
 
         let mut boot_state = Vec::default();
-        if let Some(mutation) = barrier.mutation.as_deref() {
-            match mutation {
-                Mutation::Add(AddMutation { splits, .. })
-                | Mutation::Update(UpdateMutation {
-                    actor_splits: splits,
-                    ..
-                }) => {
-                    if let Some(splits) = splits.get(&self.actor_ctx.id) {
-                        boot_state.clone_from(splits);
-                    }
-                }
-                _ => {}
+        if let Some(
+            Mutation::Add(AddMutation { splits, .. })
+            | Mutation::Update(UpdateMutation {
+                actor_splits: splits,
+                ..
+            }),
+        ) = barrier.mutation.as_deref()
+        {
+            if let Some(splits) = splits.get(&self.actor_ctx.id) {
+                boot_state.clone_from(splits);
             }
         }
 
@@ -382,6 +384,17 @@ impl<S: StateStore> FsSourceExecutor<S> {
             self.system_params.load().barrier_interval_ms() as u128 * WAIT_BARRIER_MULTIPLE_TIMES;
         let mut last_barrier_time = Instant::now();
         let mut self_paused = false;
+
+        let source_output_row_count = self
+            .metrics
+            .source_output_row_count
+            .with_guarded_label_values(&[
+                self.stream_source_core.source_id.to_string().as_ref(),
+                self.stream_source_core.source_name.as_ref(),
+                self.actor_ctx.id.to_string().as_str(),
+                self.actor_ctx.fragment_id.to_string().as_str(),
+            ]);
+
         while let Some(msg) = stream.next().await {
             match msg? {
                 // This branch will be preferred.
@@ -411,8 +424,11 @@ impl<S: StateStore> FsSourceExecutor<S> {
                                     .await?;
                                 }
                                 Mutation::Throttle(actor_to_apply) => {
-                                    if let Some(throttle) = actor_to_apply.get(&self.actor_ctx.id) {
-                                        self.source_ctrl_opts.rate_limit = *throttle;
+                                    if let Some(new_rate_limit) =
+                                        actor_to_apply.get(&self.actor_ctx.id)
+                                        && *new_rate_limit != self.rate_limit_rps
+                                    {
+                                        self.rate_limit_rps = *new_rate_limit;
                                         self.rebuild_stream_reader(&source_desc, &mut stream)
                                             .await?;
                                     }
@@ -467,15 +483,7 @@ impl<S: StateStore> FsSourceExecutor<S> {
                             .extend(state);
                     }
 
-                    self.metrics
-                        .source_output_row_count
-                        .with_label_values(&[
-                            self.stream_source_core.source_id.to_string().as_ref(),
-                            self.stream_source_core.source_name.as_ref(),
-                            self.actor_ctx.id.to_string().as_str(),
-                            self.actor_ctx.fragment_id.to_string().as_str(),
-                        ])
-                        .inc_by(chunk.cardinality() as u64);
+                    source_output_row_count.inc_by(chunk.cardinality() as u64);
 
                     let chunk =
                         prune_additional_cols(&chunk, split_idx, offset_idx, &source_desc.columns);
diff --git a/src/stream/src/executor/source/list_executor.rs b/src/stream/src/executor/source/list_executor.rs
index a5c2be3cb10b6..9317cfbc9aa4a 100644
--- a/src/stream/src/executor/source/list_executor.rs
+++ b/src/stream/src/executor/source/list_executor.rs
@@ -19,7 +19,6 @@ use futures_async_stream::try_stream;
 use risingwave_common::array::Op;
 use risingwave_common::system_param::local_manager::SystemParamsReaderRef;
 use risingwave_connector::source::reader::desc::{SourceDesc, SourceDescBuilder};
-use risingwave_connector::source::SourceCtrlOpts;
 use thiserror_ext::AsReport;
 use tokio::sync::mpsc::UnboundedReceiver;
 
@@ -45,19 +44,18 @@ pub struct FsListExecutor<S: StateStore> {
     /// System parameter reader to read barrier interval
     system_params: SystemParamsReaderRef,
 
-    // control options for connector level
-    source_ctrl_opts: SourceCtrlOpts,
+    /// Rate limit in rows/s.
+    rate_limit_rps: Option<u32>,
 }
 
 impl<S: StateStore> FsListExecutor<S> {
-    #[allow(clippy::too_many_arguments)]
     pub fn new(
         actor_ctx: ActorContextRef,
         stream_source_core: Option<StreamSourceCore<S>>,
         metrics: Arc<StreamingMetrics>,
         barrier_receiver: UnboundedReceiver<Barrier>,
         system_params: SystemParamsReaderRef,
-        source_ctrl_opts: SourceCtrlOpts,
+        rate_limit_rps: Option<u32>,
     ) -> Self {
         Self {
             actor_ctx,
@@ -65,7 +63,7 @@ impl<S: StateStore> FsListExecutor<S> {
             metrics,
             barrier_receiver: Some(barrier_receiver),
             system_params,
-            source_ctrl_opts,
+            rate_limit_rps,
         }
     }
 
@@ -84,21 +82,25 @@ impl<S: StateStore> FsListExecutor<S> {
         let chunked_stream = stream.chunks(CHUNK_SIZE).map(|chunk| {
             let rows = chunk
                 .into_iter()
-                .map(|item| {
-                    let page_item = item.expect("list file failed, please check whether the source connector is configured correctly.");
-                    (
+                .map(|item| match item {
+                    Ok(page_item) => Ok((
                         Op::Insert,
                         OwnedRow::new(vec![
                             Some(ScalarImpl::Utf8(page_item.name.into_boxed_str())),
                             Some(ScalarImpl::Timestamptz(page_item.timestamp)),
                             Some(ScalarImpl::Int64(page_item.size)),
                         ]),
-                    )
+                    )),
+                    Err(e) => {
+                        tracing::error!(error = %e.as_report(), "Connector fail to list item");
+                        Err(e)
+                    }
                 })
                 .collect::<Vec<_>>();
 
+            let res: Vec<(Op, OwnedRow)> = rows.into_iter().flatten().collect();
             Ok(StreamChunk::from_rows(
-                &rows,
+                &res,
                 &[DataType::Varchar, DataType::Timestamptz, DataType::Int64],
             ))
         });
diff --git a/src/stream/src/executor/source/mod.rs b/src/stream/src/executor/source/mod.rs
index 5777926a6a34f..cab3527a32f25 100644
--- a/src/stream/src/executor/source/mod.rs
+++ b/src/stream/src/executor/source/mod.rs
@@ -13,13 +13,17 @@
 // limitations under the License.
 
 use std::collections::HashMap;
+use std::num::NonZeroU32;
 
 use await_tree::InstrumentAwait;
+use governor::clock::MonotonicClock;
+use governor::{Quota, RateLimiter};
 use itertools::Itertools;
 use risingwave_common::array::StreamChunk;
 use risingwave_common::bail;
 use risingwave_common::row::Row;
-use risingwave_connector::source::{SourceColumnDesc, SplitId};
+use risingwave_connector::error::ConnectorError;
+use risingwave_connector::source::{BoxChunkSourceStream, SourceColumnDesc, SplitId};
 use risingwave_pb::plan_common::additional_column::ColumnType;
 use risingwave_pb::plan_common::AdditionalColumn;
 pub use state_table_handler::*;
@@ -111,3 +115,70 @@ pub fn prune_additional_cols(
             .collect_vec(),
     )
 }
+
+#[try_stream(ok = StreamChunk, error = ConnectorError)]
+pub async fn apply_rate_limit(stream: BoxChunkSourceStream, rate_limit_rps: Option<u32>) {
+    if rate_limit_rps == Some(0) {
+        // block the stream until the rate limit is reset
+        let future = futures::future::pending::<()>();
+        future.await;
+        unreachable!();
+    }
+
+    let limiter = rate_limit_rps.map(|limit| {
+        tracing::info!(rate_limit = limit, "rate limit applied");
+        RateLimiter::direct_with_clock(
+            Quota::per_second(NonZeroU32::new(limit).unwrap()),
+            &MonotonicClock,
+        )
+    });
+
+    fn compute_chunk_permits(chunk: &StreamChunk, limit: usize) -> usize {
+        let chunk_size = chunk.capacity();
+        let ends_with_update = if chunk_size >= 2 {
+            // Note we have to check if the 2nd last is `U-` to be consistenct with `StreamChunkBuilder`.
+            // If something inconsistent happens in the stream, we may not have `U+` after this `U-`.
+            chunk.ops()[chunk_size - 2].is_update_delete()
+        } else {
+            false
+        };
+        if chunk_size == limit + 1 && ends_with_update {
+            // If the chunk size exceed limit because of the last `Update` operation,
+            // we should minus 1 to make sure the permits consumed is within the limit (max burst).
+            chunk_size - 1
+        } else {
+            chunk_size
+        }
+    }
+
+    #[for_await]
+    for chunk in stream {
+        let chunk = chunk?;
+        let chunk_size = chunk.capacity();
+
+        if rate_limit_rps.is_none() || chunk_size == 0 {
+            // no limit, or empty chunk
+            yield chunk;
+            continue;
+        }
+
+        let limiter = limiter.as_ref().unwrap();
+        let limit = rate_limit_rps.unwrap() as usize;
+
+        let required_permits = compute_chunk_permits(&chunk, limit);
+        if required_permits <= limit {
+            let n = NonZeroU32::new(required_permits as u32).unwrap();
+            // `InsufficientCapacity` should never happen because we have check the cardinality
+            limiter.until_n_ready(n).await.unwrap();
+            yield chunk;
+        } else {
+            // Cut the chunk into smaller chunks
+            for chunk in chunk.split(limit) {
+                let n = NonZeroU32::new(compute_chunk_permits(&chunk, limit) as u32).unwrap();
+                // chunks split should have effective chunk size <= limit
+                limiter.until_n_ready(n).await.unwrap();
+                yield chunk;
+            }
+        }
+    }
+}
diff --git a/src/stream/src/executor/source/source_backfill_executor.rs b/src/stream/src/executor/source/source_backfill_executor.rs
index f399fdb8d5a55..7135a3fc3bf8e 100644
--- a/src/stream/src/executor/source/source_backfill_executor.rs
+++ b/src/stream/src/executor/source/source_backfill_executor.rs
@@ -21,9 +21,8 @@ use anyhow::anyhow;
 use either::Either;
 use futures::stream::{select_with_strategy, PollNext};
 use itertools::Itertools;
-use prometheus::IntCounter;
 use risingwave_common::buffer::BitmapBuilder;
-use risingwave_common::metrics::GLOBAL_ERROR_METRICS;
+use risingwave_common::metrics::{LabelGuardedIntCounter, GLOBAL_ERROR_METRICS};
 use risingwave_common::system_param::local_manager::SystemParamsReaderRef;
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::types::JsonbVal;
@@ -36,9 +35,10 @@ use thiserror_ext::AsReport;
 
 use super::executor_core::StreamSourceCore;
 use super::source_backfill_state_table::BackfillStateTableHandler;
+use super::{apply_rate_limit, get_split_offset_col_idx};
+use crate::common::rate_limit::limited_chunk_size;
 use crate::executor::prelude::*;
 use crate::executor::source::source_executor::WAIT_BARRIER_MULTIPLE_TIMES;
-use crate::executor::source::{apply_rate_limit, get_split_offset_col_idx};
 use crate::executor::{AddMutation, UpdateMutation};
 
 #[derive(Clone, Debug, Deserialize, Serialize)]
@@ -126,15 +126,15 @@ pub struct SourceBackfillExecutorInner<S: StateStore> {
 
     /// Metrics for monitor.
     metrics: Arc<StreamingMetrics>,
-    source_split_change_count: IntCounter,
+    source_split_change_count: LabelGuardedIntCounter<4>,
 
     // /// Receiver of barrier channel.
     // barrier_receiver: Option<UnboundedReceiver<Barrier>>,
     /// System parameter reader to read barrier interval
     system_params: SystemParamsReaderRef,
 
-    // control options for connector level
-    source_ctrl_opts: SourceCtrlOpts,
+    /// Rate limit in rows/s.
+    rate_limit_rps: Option<u32>,
 }
 
 /// Local variables used in the backfill stage.
@@ -172,15 +172,17 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
         stream_source_core: StreamSourceCore<S>,
         metrics: Arc<StreamingMetrics>,
         system_params: SystemParamsReaderRef,
-        source_ctrl_opts: SourceCtrlOpts,
         backfill_state_store: BackfillStateTableHandler<S>,
+        rate_limit_rps: Option<u32>,
     ) -> Self {
-        let source_split_change_count = metrics.source_split_change_count.with_label_values(&[
-            &stream_source_core.source_id.to_string(),
-            &stream_source_core.source_name,
-            &actor_ctx.id.to_string(),
-            &actor_ctx.fragment_id.to_string(),
-        ]);
+        let source_split_change_count = metrics
+            .source_split_change_count
+            .with_guarded_label_values(&[
+                &stream_source_core.source_id.to_string(),
+                &stream_source_core.source_name,
+                &actor_ctx.id.to_string(),
+                &actor_ctx.fragment_id.to_string(),
+            ]);
         Self {
             actor_ctx,
             info,
@@ -189,7 +191,7 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
             metrics,
             source_split_change_count,
             system_params,
-            source_ctrl_opts,
+            rate_limit_rps,
         }
     }
 
@@ -209,15 +211,18 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
             self.actor_ctx.fragment_id,
             self.stream_source_core.source_name.clone(),
             source_desc.metrics.clone(),
-            self.source_ctrl_opts.clone(),
+            SourceCtrlOpts {
+                chunk_size: limited_chunk_size(self.rate_limit_rps),
+                rate_limit: self.rate_limit_rps,
+            },
             source_desc.source.config.clone(),
         );
         let stream = source_desc
             .source
-            .to_stream(Some(splits), column_ids, Arc::new(source_ctx))
+            .build_stream(Some(splits), column_ids, Arc::new(source_ctx))
             .await
             .map_err(StreamExecutorError::connector_error)?;
-        Ok(apply_rate_limit(stream, self.source_ctrl_opts.rate_limit).boxed())
+        Ok(apply_rate_limit(stream, self.rate_limit_rps).boxed())
     }
 
     #[try_stream(ok = Message, error = StreamExecutorError)]
@@ -729,11 +734,11 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
         if split_changed {
             stage
                 .unfinished_splits
-                .retain(|split| target_state.get(split.id().as_ref()).is_some());
+                .retain(|split| target_state.contains_key(split.id().as_ref()));
 
             let dropped_splits = stage
                 .states
-                .extract_if(|split_id, _| target_state.get(split_id).is_none())
+                .extract_if(|split_id, _| !target_state.contains_key(split_id))
                 .map(|(split_id, _)| split_id);
 
             if should_trim_state {
@@ -822,7 +827,7 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
             );
 
             let dropped_splits =
-                current_splits.extract_if(|split_id| target_splits.get(split_id).is_none());
+                current_splits.extract_if(|split_id| !target_splits.contains(split_id));
 
             if should_trim_state {
                 // trim dropped splits' state
diff --git a/src/stream/src/executor/source/source_backfill_state_table.rs b/src/stream/src/executor/source/source_backfill_state_table.rs
index 678a76f39f927..be9abe8490e63 100644
--- a/src/stream/src/executor/source/source_backfill_state_table.rs
+++ b/src/stream/src/executor/source/source_backfill_state_table.rs
@@ -33,6 +33,7 @@ pub struct BackfillStateTableHandler<S: StateStore> {
 }
 
 impl<S: StateStore> BackfillStateTableHandler<S> {
+    /// See also [`super::SourceStateTableHandler::from_table_catalog`] for how the state table looks like.
     pub async fn from_table_catalog(table_catalog: &PbTable, store: S) -> Self {
         Self {
             state_store: StateTable::from_table_catalog(table_catalog, store, None).await,
diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs
index a752fa234cad5..8e5c3f9726c28 100644
--- a/src/stream/src/executor/source/source_executor.rs
+++ b/src/stream/src/executor/source/source_executor.rs
@@ -13,26 +13,22 @@
 // limitations under the License.
 
 use std::collections::HashMap;
-use std::num::NonZeroU32;
-use std::str::FromStr;
 use std::time::Duration;
 
 use anyhow::anyhow;
 use either::Either;
 use futures::TryStreamExt;
-use governor::clock::MonotonicClock;
-use governor::{Quota, RateLimiter};
 use itertools::Itertools;
-use risingwave_common::metrics::GLOBAL_ERROR_METRICS;
+use risingwave_common::array::ArrayRef;
+use risingwave_common::metrics::{LabelGuardedIntCounter, GLOBAL_ERROR_METRICS};
 use risingwave_common::system_param::local_manager::SystemParamsReaderRef;
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::util::epoch::{Epoch, EpochPair};
-use risingwave_connector::error::ConnectorError;
-use risingwave_connector::source::cdc::jni_source;
 use risingwave_connector::source::reader::desc::{SourceDesc, SourceDescBuilder};
+use risingwave_connector::source::reader::reader::SourceReader;
 use risingwave_connector::source::{
     BoxChunkSourceStream, ConnectorState, SourceContext, SourceCtrlOpts, SplitId, SplitImpl,
-    SplitMetaData,
+    SplitMetaData, WaitCheckpointTask,
 };
 use risingwave_hummock_sdk::HummockReadEpoch;
 use thiserror_ext::AsReport;
@@ -41,11 +37,12 @@ use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 use tokio::time::Instant;
 
 use super::executor_core::StreamSourceCore;
-use crate::executor::prelude::*;
-use crate::executor::source::{
-    barrier_to_message_stream, get_split_offset_col_idx, get_split_offset_mapping_from_chunk,
-    prune_additional_cols,
+use super::{
+    apply_rate_limit, barrier_to_message_stream, get_split_offset_col_idx,
+    get_split_offset_mapping_from_chunk, prune_additional_cols,
 };
+use crate::common::rate_limit::limited_chunk_size;
+use crate::executor::prelude::*;
 use crate::executor::stream_reader::StreamReaderWithPause;
 use crate::executor::{AddMutation, UpdateMutation};
 
@@ -68,56 +65,10 @@ pub struct SourceExecutor<S: StateStore> {
     /// System parameter reader to read barrier interval
     system_params: SystemParamsReaderRef,
 
-    // control options for connector level
-    source_ctrl_opts: SourceCtrlOpts,
-}
+    /// Rate limit in rows/s.
+    rate_limit_rps: Option<u32>,
 
-#[try_stream(ok = StreamChunk, error = ConnectorError)]
-pub async fn apply_rate_limit(stream: BoxChunkSourceStream, rate_limit: Option<u32>) {
-    if let Some(limit) = rate_limit
-        && limit == 0
-    {
-        // block the stream until the rate limit is reset
-        let future = futures::future::pending::<()>();
-        future.await;
-    }
-    let get_rate_limiter = |rate_limit: u32| {
-        let quota = Quota::per_second(NonZeroU32::new(rate_limit).unwrap());
-        let clock = MonotonicClock;
-        RateLimiter::direct_with_clock(quota, &clock)
-    };
-    let limiter = rate_limit.map(get_rate_limiter);
-    if rate_limit.is_some() {
-        tracing::info!(rate_limit = ?rate_limit, "applied rate limit");
-    }
-    #[for_await]
-    for batch in stream {
-        let chunk: StreamChunk = batch?;
-        let chunk_cardinality = chunk.cardinality();
-        let Some(n) = NonZeroU32::new(chunk_cardinality as u32) else {
-            // pass empty chunk
-            yield chunk;
-            continue;
-        };
-        if let Some(limiter) = &limiter {
-            let limit = NonZeroU32::new(rate_limit.unwrap()).unwrap();
-            if n <= limit {
-                // `InsufficientCapacity` should never happen because we have done the check
-                limiter.until_n_ready(n).await.unwrap();
-                yield chunk;
-            } else {
-                // Cut the chunk into smaller chunks
-                for chunk in chunk.split(limit.get() as usize) {
-                    let n = NonZeroU32::new(chunk.cardinality() as u32).unwrap();
-                    // Ditto.
-                    limiter.until_n_ready(n).await.unwrap();
-                    yield chunk;
-                }
-            }
-        } else {
-            yield chunk;
-        }
-    }
+    is_shared: bool,
 }
 
 impl<S: StateStore> SourceExecutor<S> {
@@ -127,7 +78,8 @@ impl<S: StateStore> SourceExecutor<S> {
         metrics: Arc<StreamingMetrics>,
         barrier_receiver: UnboundedReceiver<Barrier>,
         system_params: SystemParamsReaderRef,
-        source_ctrl_opts: SourceCtrlOpts,
+        rate_limit_rps: Option<u32>,
+        is_shared: bool,
     ) -> Self {
         Self {
             actor_ctx,
@@ -135,20 +87,29 @@ impl<S: StateStore> SourceExecutor<S> {
             metrics,
             barrier_receiver: Some(barrier_receiver),
             system_params,
-            source_ctrl_opts,
+            rate_limit_rps,
+            is_shared,
         }
     }
 
-    pub fn spawn_wait_epoch_worker(
+    async fn spawn_wait_checkpoint_worker(
         core: &StreamSourceCore<S>,
-    ) -> UnboundedSender<(Epoch, HashMap<SplitId, String>)> {
-        let (wait_epoch_tx, wait_epoch_rx) = mpsc::unbounded_channel();
-        let wait_epoch_worker = WaitEpochWorker {
-            wait_epoch_rx,
+        source_reader: SourceReader,
+    ) -> StreamExecutorResult<Option<WaitCheckpointTaskBuilder>> {
+        let Some(initial_task) = source_reader.create_wait_checkpoint_task().await? else {
+            return Ok(None);
+        };
+        let (wait_checkpoint_tx, wait_checkpoint_rx) = mpsc::unbounded_channel();
+        let wait_checkpoint_worker = WaitCheckpointWorker {
+            wait_checkpoint_rx,
             state_store: core.split_state_store.state_table.state_store().clone(),
         };
-        tokio::spawn(wait_epoch_worker.run());
-        wait_epoch_tx
+        tokio::spawn(wait_checkpoint_worker.run());
+        Ok(Some(WaitCheckpointTaskBuilder {
+            wait_checkpoint_tx,
+            source_reader,
+            building_task: initial_task,
+        }))
     }
 
     pub async fn build_stream_source_reader(
@@ -171,16 +132,19 @@ impl<S: StateStore> SourceExecutor<S> {
                 .source_name
                 .clone(),
             source_desc.metrics.clone(),
-            self.source_ctrl_opts.clone(),
+            SourceCtrlOpts {
+                chunk_size: limited_chunk_size(self.rate_limit_rps),
+                rate_limit: self.rate_limit_rps,
+            },
             source_desc.source.config.clone(),
         );
         let stream = source_desc
             .source
-            .to_stream(state, column_ids, Arc::new(source_ctx))
+            .build_stream(state, column_ids, Arc::new(source_ctx))
             .await
             .map_err(StreamExecutorError::connector_error);
 
-        Ok(apply_rate_limit(stream?, self.source_ctrl_opts.rate_limit).boxed())
+        Ok(apply_rate_limit(stream?, self.rate_limit_rps).boxed())
     }
 
     /// `source_id | source_name | actor_id | fragment_id`
@@ -216,17 +180,9 @@ impl<S: StateStore> SourceExecutor<S> {
         stream: &mut StreamReaderWithPause<BIASED, StreamChunk>,
         split_assignment: &HashMap<ActorId, Vec<SplitImpl>>,
         should_trim_state: bool,
+        source_split_change_count_metrics: &LabelGuardedIntCounter<4>,
     ) -> StreamExecutorResult<()> {
-        self.metrics
-            .source_split_change_count
-            .with_label_values(
-                &self
-                    .get_metric_labels()
-                    .iter()
-                    .map(AsRef::as_ref)
-                    .collect::<Vec<&str>>(),
-            )
-            .inc();
+        source_split_change_count_metrics.inc();
         if let Some(target_splits) = split_assignment.get(&self.actor_ctx.id).cloned() {
             if self
                 .update_state_if_changed(target_splits, should_trim_state)
@@ -301,11 +257,11 @@ impl<S: StateStore> SourceExecutor<S> {
             );
 
             core.updated_splits_in_epoch
-                .retain(|split_id, _| target_state.get(split_id).is_some());
+                .retain(|split_id, _| target_state.contains_key(split_id));
 
             let dropped_splits = core
                 .latest_split_info
-                .extract_if(|split_id, _| target_state.get(split_id).is_none())
+                .extract_if(|split_id, _| !target_state.contains_key(split_id))
                 .map(|(_, split)| split)
                 .collect_vec();
 
@@ -431,11 +387,8 @@ impl<S: StateStore> SourceExecutor<S> {
             .build()
             .map_err(StreamExecutorError::connector_error)?;
 
-        let wait_epoch_tx = if source_desc.source.need_commit_offset_to_upstream() {
-            Some(Self::spawn_wait_epoch_worker(&core))
-        } else {
-            None
-        };
+        let mut wait_checkpoint_task_builder =
+            Self::spawn_wait_checkpoint_worker(&core, source_desc.source.clone()).await?;
 
         let (Some(split_idx), Some(offset_idx)) = get_split_offset_col_idx(&source_desc.columns)
         else {
@@ -443,23 +396,21 @@ impl<S: StateStore> SourceExecutor<S> {
         };
 
         let mut boot_state = Vec::default();
-        if let Some(mutation) = barrier.mutation.as_deref() {
-            match mutation {
-                Mutation::Add(AddMutation { splits, .. })
-                | Mutation::Update(UpdateMutation {
-                    actor_splits: splits,
-                    ..
-                }) => {
-                    if let Some(splits) = splits.get(&self.actor_ctx.id) {
-                        tracing::debug!(
-                            "source exector: actor {:?} boot with splits: {:?}",
-                            self.actor_ctx.id,
-                            splits
-                        );
-                        boot_state.clone_from(splits);
-                    }
-                }
-                _ => {}
+        if let Some(
+            Mutation::Add(AddMutation { splits, .. })
+            | Mutation::Update(UpdateMutation {
+                actor_splits: splits,
+                ..
+            }),
+        ) = barrier.mutation.as_deref()
+        {
+            if let Some(splits) = splits.get(&self.actor_ctx.id) {
+                tracing::debug!(
+                    "source exector: actor {:?} boot with splits: {:?}",
+                    self.actor_ctx.id,
+                    splits
+                );
+                boot_state.clone_from(splits);
             }
         }
 
@@ -472,11 +423,30 @@ impl<S: StateStore> SourceExecutor<S> {
                 .await?
             {
                 *ele = recover_state;
+            } else {
+                // This is a new split, not in state table.
+                if self.is_shared {
+                    // For shared source, we start from latest and let the downstream SourceBackfillExecutors to read historical data.
+                    // It's highly probable that the work of scanning historical data cannot be shared,
+                    // so don't waste work on it.
+                    // For more details, see https://github.com/risingwavelabs/risingwave/issues/16576#issuecomment-2095413297
+                    if ele.is_cdc_split() {
+                        // shared CDC source already starts from latest.
+                        continue;
+                    }
+                    match ele {
+                        SplitImpl::Kafka(split) => {
+                            split.seek_to_latest_offset();
+                        }
+                        _ => unreachable!("only kafka source can be shared, got {:?}", ele),
+                    }
+                }
             }
         }
 
         // init in-memory split states with persisted state if any
         core.init_split_state(boot_state.clone());
+        let mut is_uninitialized = self.actor_ctx.initial_dispatch_num == 0;
 
         // Return the ownership of `stream_source_core` to the source executor.
         self.stream_source_core = Some(core);
@@ -495,11 +465,16 @@ impl<S: StateStore> SourceExecutor<S> {
         let mut stream =
             StreamReaderWithPause::<true, StreamChunk>::new(barrier_stream, source_chunk_reader);
 
-        // If the first barrier requires us to pause on startup, pause the stream.
-        if barrier.is_pause_on_startup() {
+        // - For shared source, pause until there's a MV.
+        // - If the first barrier requires us to pause on startup, pause the stream.
+        if (self.is_shared && is_uninitialized) || barrier.is_pause_on_startup() {
+            tracing::info!(
+                is_shared = self.is_shared,
+                is_uninitialized = is_uninitialized,
+                "source paused on startup"
+            );
             stream.pause_stream();
         }
-        // TODO: for shared source, pause until there's a MV.
 
         yield Message::Barrier(barrier);
 
@@ -510,6 +485,16 @@ impl<S: StateStore> SourceExecutor<S> {
         let mut last_barrier_time = Instant::now();
         let mut self_paused = false;
 
+        let source_output_row_count = self
+            .metrics
+            .source_output_row_count
+            .with_guarded_label_values(&self.get_metric_labels().each_ref().map(AsRef::as_ref));
+
+        let source_split_change_count = self
+            .metrics
+            .source_split_change_count
+            .with_guarded_label_values(&self.get_metric_labels().each_ref().map(AsRef::as_ref));
+
         while let Some(msg) = stream.next().await {
             let Ok(msg) = msg else {
                 tokio::time::sleep(Duration::from_millis(1000)).await;
@@ -530,8 +515,17 @@ impl<S: StateStore> SourceExecutor<S> {
 
                     let epoch = barrier.epoch;
 
+                    if self.is_shared
+                        && is_uninitialized
+                        && barrier.has_more_downstream_fragments(self.actor_ctx.id)
+                    {
+                        stream.resume_stream();
+                        is_uninitialized = false;
+                    }
+
                     if let Some(mutation) = barrier.mutation.as_deref() {
                         match mutation {
+                            // XXX: Is it possible that the stream is self_paused, and we have pause mutation now? In this case, it will panic.
                             Mutation::Pause => stream.pause_stream(),
                             Mutation::Resume => stream.resume_stream(),
                             Mutation::SourceChangeSplit(actor_splits) => {
@@ -546,6 +540,7 @@ impl<S: StateStore> SourceExecutor<S> {
                                     &mut stream,
                                     actor_splits,
                                     true,
+                                    &source_split_change_count,
                                 )
                                 .await?;
                             }
@@ -556,12 +551,15 @@ impl<S: StateStore> SourceExecutor<S> {
                                     &mut stream,
                                     actor_splits,
                                     false,
+                                    &source_split_change_count,
                                 )
                                 .await?;
                             }
                             Mutation::Throttle(actor_to_apply) => {
-                                if let Some(throttle) = actor_to_apply.get(&self.actor_ctx.id) {
-                                    self.source_ctrl_opts.rate_limit = *throttle;
+                                if let Some(new_rate_limit) = actor_to_apply.get(&self.actor_ctx.id)
+                                    && *new_rate_limit != self.rate_limit_rps
+                                {
+                                    self.rate_limit_rps = *new_rate_limit;
                                     // recreate from latest_split_info
                                     self.rebuild_stream_reader(&source_desc, &mut stream)
                                         .await?;
@@ -575,19 +573,12 @@ impl<S: StateStore> SourceExecutor<S> {
 
                     // when handle a checkpoint barrier, spawn a task to wait for epoch commit notification
                     if barrier.kind.is_checkpoint()
-                        && !updated_splits.is_empty()
-                        && let Some(ref tx) = wait_epoch_tx
+                        && let Some(task_builder) = &mut wait_checkpoint_task_builder
                     {
-                        let mut updated_offsets = HashMap::new();
-                        for (split_id, split_impl) in updated_splits {
-                            if split_impl.is_cdc_split() {
-                                updated_offsets.insert(split_id, split_impl.get_cdc_split_offset());
-                            }
-                        }
+                        task_builder.update_task_on_checkpoint(updated_splits);
 
                         tracing::debug!("epoch to wait {:?}", epoch);
-                        tx.send((Epoch(epoch.prev), updated_offsets))
-                            .expect("wait_epoch_tx send success");
+                        task_builder.send(Epoch(epoch.prev)).await?
                     }
 
                     yield Message::Barrier(barrier);
@@ -599,6 +590,10 @@ impl<S: StateStore> SourceExecutor<S> {
                 }
 
                 Either::Right(chunk) => {
+                    if let Some(task_builder) = &mut wait_checkpoint_task_builder {
+                        let offset_col = chunk.column_at(offset_idx);
+                        task_builder.update_task_on_chunk(offset_col.clone());
+                    }
                     // TODO: confirm when split_offset_mapping is None
                     let split_offset_mapping =
                         get_split_offset_mapping_from_chunk(&chunk, split_idx, offset_idx);
@@ -648,16 +643,7 @@ impl<S: StateStore> SourceExecutor<S> {
                             .extend(state);
                     }
 
-                    self.metrics
-                        .source_output_row_count
-                        .with_label_values(
-                            &self
-                                .get_metric_labels()
-                                .iter()
-                                .map(AsRef::as_ref)
-                                .collect::<Vec<&str>>(),
-                        )
-                        .inc_by(chunk.cardinality() as u64);
+                    source_output_row_count.inc_by(chunk.cardinality() as u64);
                     let chunk =
                         prune_additional_cols(&chunk, split_idx, offset_idx, &source_desc.columns);
                     yield Message::Chunk(chunk);
@@ -719,18 +705,94 @@ impl<S: StateStore> Debug for SourceExecutor<S> {
     }
 }
 
-struct WaitEpochWorker<S: StateStore> {
-    wait_epoch_rx: UnboundedReceiver<(Epoch, HashMap<SplitId, String>)>,
+struct WaitCheckpointTaskBuilder {
+    wait_checkpoint_tx: UnboundedSender<(Epoch, WaitCheckpointTask)>,
+    source_reader: SourceReader,
+    building_task: WaitCheckpointTask,
+}
+
+impl WaitCheckpointTaskBuilder {
+    fn update_task_on_chunk(&mut self, offset_col: ArrayRef) {
+        #[expect(clippy::single_match)]
+        match &mut self.building_task {
+            WaitCheckpointTask::AckPubsubMessage(_, arrays) => {
+                arrays.push(offset_col);
+            }
+            _ => {}
+        }
+    }
+
+    fn update_task_on_checkpoint(&mut self, updated_splits: HashMap<SplitId, SplitImpl>) {
+        #[expect(clippy::single_match)]
+        match &mut self.building_task {
+            WaitCheckpointTask::CommitCdcOffset(offsets) => {
+                if !updated_splits.is_empty() {
+                    // cdc source only has one split
+                    assert_eq!(1, updated_splits.len());
+                    for (split_id, split_impl) in updated_splits {
+                        if split_impl.is_cdc_split() {
+                            *offsets = Some((split_id, split_impl.get_cdc_split_offset()));
+                        } else {
+                            unreachable!()
+                        }
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+
+    /// Send and reset the building task to a new one.
+    async fn send(&mut self, epoch: Epoch) -> Result<(), anyhow::Error> {
+        let new_task = self
+            .source_reader
+            .create_wait_checkpoint_task()
+            .await?
+            .expect("wait checkpoint task should be created");
+        self.wait_checkpoint_tx
+            .send((epoch, std::mem::replace(&mut self.building_task, new_task)))
+            .expect("wait_checkpoint_tx send should succeed");
+        Ok(())
+    }
+}
+
+/// A worker used to do some work after each checkpoint epoch is committed.
+///
+/// # Usage Cases
+///
+/// Typically there are 2 issues related with ack on checkpoint:
+///
+/// 1. Correctness (at-least-once), or don't let upstream clean uncommitted data.
+///    For message queueing semantics (delete after ack), we should ack to avoid redelivery,
+///    and only ack after checkpoint to avoid data loss.
+///
+/// 2. Allow upstream to clean data after commit.
+///
+/// See also <https://github.com/risingwavelabs/risingwave/issues/16736#issuecomment-2109379790>
+///
+/// ## CDC
+///
+/// Commit last consumed offset to upstream DB, so that old data can be discarded.
+///
+/// ## Google Pub/Sub
+///
+/// Due to queueing semantics.
+/// Although Pub/Sub supports `retain_acked_messages` and `seek` functionality,
+/// it's quite limited unlike Kafka.
+///
+/// See also <https://cloud.google.com/pubsub/docs/subscribe-best-practices#process-messages>
+struct WaitCheckpointWorker<S: StateStore> {
+    wait_checkpoint_rx: UnboundedReceiver<(Epoch, WaitCheckpointTask)>,
     state_store: S,
 }
 
-impl<S: StateStore> WaitEpochWorker<S> {
+impl<S: StateStore> WaitCheckpointWorker<S> {
     pub async fn run(mut self) {
         tracing::debug!("wait epoch worker start success");
         loop {
             // poll the rx and wait for the epoch commit
-            match self.wait_epoch_rx.recv().await {
-                Some((epoch, updated_offsets)) => {
+            match self.wait_checkpoint_rx.recv().await {
+                Some((epoch, task)) => {
                     tracing::debug!("start to wait epoch {}", epoch.0);
                     let ret = self
                         .state_store
@@ -740,20 +802,7 @@ impl<S: StateStore> WaitEpochWorker<S> {
                     match ret {
                         Ok(()) => {
                             tracing::debug!(epoch = epoch.0, "wait epoch success");
-                            // cdc source only has one split
-                            assert_eq!(1, updated_offsets.len());
-                            let (split_id, offset) = updated_offsets.into_iter().next().unwrap();
-                            let source_id: u64 = u64::from_str(split_id.as_ref()).unwrap();
-                            // notify cdc connector to commit offset
-                            match jni_source::commit_cdc_offset(source_id, offset.clone()) {
-                                Ok(_) => {}
-                                Err(e) => {
-                                    tracing::error!(
-                                        error = %e.as_report(),
-                                        "source#{source_id}: failed to commit cdc offset: {offset}.",
-                                    )
-                                }
-                            }
+                            task.run().await;
                         }
                         Err(e) => {
                             tracing::error!(
@@ -775,15 +824,11 @@ impl<S: StateStore> WaitEpochWorker<S> {
 #[cfg(test)]
 mod tests {
     use std::collections::HashSet;
-    use std::time::Duration;
 
-    use futures::StreamExt;
-    use maplit::{convert_args, hashmap};
-    use risingwave_common::array::StreamChunk;
-    use risingwave_common::catalog::{ColumnId, Field, Schema, TableId};
+    use maplit::{btreemap, convert_args, hashmap};
+    use risingwave_common::catalog::{ColumnId, Field, TableId};
     use risingwave_common::system_param::local_manager::LocalSystemParamsManager;
     use risingwave_common::test_prelude::StreamChunkTestExt;
-    use risingwave_common::types::DataType;
     use risingwave_common::util::epoch::test_epoch;
     use risingwave_connector::source::datagen::DatagenSplit;
     use risingwave_connector::source::reader::desc::test_utils::create_source_desc_builder;
@@ -795,7 +840,6 @@ mod tests {
 
     use super::*;
     use crate::executor::source::{default_source_internal_table, SourceStateTableHandler};
-    use crate::executor::ActorContext;
 
     const MOCK_SOURCE_NAME: &str = "mock_source";
 
@@ -814,7 +858,7 @@ mod tests {
         let column_ids = vec![0].into_iter().map(ColumnId::from).collect();
 
         // This datagen will generate 3 rows at one time.
-        let properties: HashMap<String, String> = convert_args!(hashmap!(
+        let properties = convert_args!(btreemap!(
             "connector" => "datagen",
             "datagen.rows.per.second" => "3",
             "fields.sequence_int.kind" => "sequence",
@@ -846,10 +890,8 @@ mod tests {
             Arc::new(StreamingMetrics::unused()),
             barrier_rx,
             system_params_manager.get_params(),
-            SourceCtrlOpts {
-                chunk_size: 1024,
-                rate_limit: None,
-            },
+            None,
+            false,
         );
         let mut executor = executor.boxed().execute();
 
@@ -900,7 +942,7 @@ mod tests {
             row_format: PbRowFormatType::Native as i32,
             ..Default::default()
         };
-        let properties = convert_args!(hashmap!(
+        let properties = convert_args!(btreemap!(
             "connector" => "datagen",
             "fields.v1.kind" => "sequence",
             "fields.v1.start" => "11",
@@ -937,10 +979,8 @@ mod tests {
             Arc::new(StreamingMetrics::unused()),
             barrier_rx,
             system_params_manager.get_params(),
-            SourceCtrlOpts {
-                chunk_size: 1024,
-                rate_limit: None,
-            },
+            None,
+            false,
         );
         let mut handler = executor.boxed().execute();
 
diff --git a/src/stream/src/executor/source/state_table_handler.rs b/src/stream/src/executor/source/state_table_handler.rs
index f85f12c793a2b..1d14de9a493c9 100644
--- a/src/stream/src/executor/source/state_table_handler.rs
+++ b/src/stream/src/executor/source/state_table_handler.rs
@@ -51,12 +51,16 @@ pub struct SourceStateTableHandler<S: StateStore> {
 }
 
 impl<S: StateStore> SourceStateTableHandler<S> {
+    /// Creates a state table with singleton distribution (only one vnode 0).
+    ///
+    /// Refer to `infer_internal_table_catalog` in `src/frontend/src/optimizer/plan_node/generic/source.rs` for more details.
     pub async fn from_table_catalog(table_catalog: &PbTable, store: S) -> Self {
         Self {
             state_table: StateTable::from_table_catalog(table_catalog, store, None).await,
         }
     }
 
+    /// For [`super::FsFetchExecutor`], each actor accesses splits according to the `vnode` computed from `partition_id`.
     pub async fn from_table_catalog_with_vnodes(
         table_catalog: &PbTable,
         store: S,
@@ -253,11 +257,9 @@ pub fn default_source_internal_table(id: u32) -> PbTable {
 
 #[cfg(test)]
 pub(crate) mod tests {
-    use std::sync::Arc;
 
-    use risingwave_common::row::OwnedRow;
-    use risingwave_common::types::{Datum, ScalarImpl};
-    use risingwave_common::util::epoch::{test_epoch, EpochPair};
+    use risingwave_common::types::Datum;
+    use risingwave_common::util::epoch::test_epoch;
     use risingwave_connector::source::kafka::KafkaSplit;
     use risingwave_storage::memory::MemoryStateStore;
     use serde_json::Value;
diff --git a/src/stream/src/executor/stateless_simple_agg.rs b/src/stream/src/executor/stateless_simple_agg.rs
index 1360f4166008e..540569f8bdff7 100644
--- a/src/stream/src/executor/stateless_simple_agg.rs
+++ b/src/stream/src/executor/stateless_simple_agg.rs
@@ -62,7 +62,7 @@ impl StatelessSimpleAggExecutor {
         } = self;
         let input = input.execute();
         let mut is_dirty = false;
-        let mut states = aggs.iter().map(|agg| agg.create_state()).collect_vec();
+        let mut states: Vec<_> = aggs.iter().map(|agg| agg.create_state()).try_collect()?;
 
         #[for_await]
         for msg in input {
@@ -84,7 +84,7 @@ impl StatelessSimpleAggExecutor {
                             .zip_eq_fast(builders.iter_mut())
                         {
                             let data = agg.get_result(state).await?;
-                            *state = agg.create_state();
+                            *state = agg.create_state()?;
                             trace!("append: {:?}", data);
                             builder.append(data);
                         }
@@ -125,16 +125,13 @@ impl StatelessSimpleAggExecutor {
 #[cfg(test)]
 mod tests {
     use assert_matches::assert_matches;
-    use futures::StreamExt;
     use risingwave_common::array::stream_chunk::StreamChunkTestExt;
-    use risingwave_common::array::StreamChunk;
     use risingwave_common::catalog::schema_test_utils;
     use risingwave_common::util::epoch::test_epoch;
 
     use super::*;
     use crate::executor::test_utils::agg_executor::generate_agg_schema;
     use crate::executor::test_utils::MockSource;
-    use crate::executor::{Execute, StatelessSimpleAggExecutor};
 
     #[tokio::test]
     async fn test_no_chunk() {
diff --git a/src/stream/src/executor/stream_reader.rs b/src/stream/src/executor/stream_reader.rs
index 7fbf43a5794b0..92a31979c1c56 100644
--- a/src/stream/src/executor/stream_reader.rs
+++ b/src/stream/src/executor/stream_reader.rs
@@ -96,12 +96,14 @@ impl<const BIASED: bool, M: Send + 'static> StreamReaderWithPause<BIASED, M> {
     /// Pause the data stream.
     pub fn pause_stream(&mut self) {
         assert!(!self.paused, "already paused");
+        tracing::info!("data stream paused");
         self.paused = true;
     }
 
     /// Resume the data stream. Panic if the data stream is not paused.
     pub fn resume_stream(&mut self) {
         assert!(self.paused, "not paused");
+        tracing::info!("data stream resumed");
         self.paused = false;
     }
 }
diff --git a/src/stream/src/executor/subscription.rs b/src/stream/src/executor/subscription.rs
deleted file mode 100644
index 4bacb2dc75f05..0000000000000
--- a/src/stream/src/executor/subscription.rs
+++ /dev/null
@@ -1,130 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use core::str::FromStr;
-use core::time::Duration;
-use std::collections::HashMap;
-
-use risingwave_common::types::{Interval, Timestamptz};
-use risingwave_common::util::epoch::Epoch;
-use risingwave_storage::store::LocalStateStore;
-use tokio::time::Instant;
-
-use crate::common::log_store_impl::kv_log_store::ReaderTruncationOffsetType;
-use crate::common::log_store_impl::subscription_log_store::SubscriptionLogStoreWriter;
-use crate::executor::prelude::*;
-
-const EXECUTE_GC_INTERVAL: u64 = 3600;
-const MAX_RETENTION_DAYS: i32 = 365;
-const RETENTION_SECONDS_KEY: &str = "retention";
-
-pub struct SubscriptionExecutor<LS: LocalStateStore> {
-    actor_context: ActorContextRef,
-    input: Executor,
-    log_store: SubscriptionLogStoreWriter<LS>,
-    retention_seconds: i64,
-}
-
-impl<LS: LocalStateStore> SubscriptionExecutor<LS> {
-    #[allow(clippy::too_many_arguments)]
-    #[expect(clippy::unused_async)]
-    pub async fn new(
-        actor_context: ActorContextRef,
-        input: Executor,
-        log_store: SubscriptionLogStoreWriter<LS>,
-        properties: HashMap<String, String>,
-    ) -> StreamExecutorResult<Self> {
-        let retention_seconds_str = properties.get(RETENTION_SECONDS_KEY).ok_or_else(|| {
-            StreamExecutorError::serde_error("Subscription retention time not set.".to_string())
-        })?;
-        let retention_seconds_interval =
-            Interval::from_str(retention_seconds_str).map_err(|_| {
-                StreamExecutorError::serde_error(
-                    "Retention needs to be set in Interval format".to_string(),
-                )
-            })?;
-        if retention_seconds_interval.days() > MAX_RETENTION_DAYS {
-            return Err(StreamExecutorError::serde_error(format!(
-                "Retention time cannot exceed {} days",
-                MAX_RETENTION_DAYS
-            )));
-        }
-        let retention_seconds = (retention_seconds_interval.epoch_in_micros() / 1000000) as i64;
-
-        Ok(Self {
-            actor_context,
-            input,
-            log_store,
-            retention_seconds,
-        })
-    }
-
-    #[try_stream(ok = Message, error = StreamExecutorError)]
-    async fn execute_inner(mut self) {
-        let mut input = self.input.execute();
-
-        let barrier = expect_first_barrier(&mut input).await?;
-        self.log_store.init(barrier.epoch, false).await?;
-
-        // The first barrier message should be propagated.
-        yield Message::Barrier(barrier);
-
-        let mut next_truncate_time = Instant::now() + Duration::from_secs(EXECUTE_GC_INTERVAL);
-
-        #[for_await]
-        for msg in input {
-            let msg = msg?;
-            yield match msg {
-                Message::Watermark(w) => Message::Watermark(w),
-                Message::Chunk(chunk) => {
-                    if chunk.cardinality() == 0 {
-                        // empty chunk
-                        continue;
-                    }
-                    self.log_store.write_chunk(chunk.clone())?;
-                    Message::Chunk(chunk)
-                }
-                Message::Barrier(barrier) => {
-                    let truncate_offset: Option<ReaderTruncationOffsetType> = if next_truncate_time
-                        < Instant::now()
-                    {
-                        let truncate_timestamptz = Timestamptz::from_secs(barrier.get_curr_epoch().as_timestamptz().timestamp() - self.retention_seconds).ok_or_else(||{StreamExecutorError::from("Subscription retention time calculation error: timestamp is out of range.".to_string())})?;
-                        let epoch =
-                            Epoch::from_unix_millis(truncate_timestamptz.timestamp_millis() as u64);
-                        next_truncate_time =
-                            Instant::now() + Duration::from_secs(EXECUTE_GC_INTERVAL);
-                        Some((epoch.0, None))
-                    } else {
-                        None
-                    };
-                    self.log_store
-                        .flush_current_epoch(barrier.epoch.curr, truncate_offset)
-                        .await?;
-
-                    if let Some(vnode_bitmap) =
-                        barrier.as_update_vnode_bitmap(self.actor_context.id)
-                    {
-                        self.log_store.update_vnode_bitmap(vnode_bitmap)?;
-                    }
-                    Message::Barrier(barrier)
-                }
-            }
-        }
-    }
-}
-impl<LS: LocalStateStore> Execute for SubscriptionExecutor<LS> {
-    fn execute(self: Box<Self>) -> BoxedMessageStream {
-        self.execute_inner().boxed()
-    }
-}
diff --git a/src/stream/src/executor/temporal_join.rs b/src/stream/src/executor/temporal_join.rs
index b744fb245a3b4..ce0c4d29621d6 100644
--- a/src/stream/src/executor/temporal_join.rs
+++ b/src/stream/src/executor/temporal_join.rs
@@ -35,7 +35,8 @@ use risingwave_storage::table::batch_table::storage_table::StorageTable;
 use risingwave_storage::table::TableIter;
 
 use super::join::{JoinType, JoinTypePrimitive};
-use crate::cache::{cache_may_stale, new_with_hasher_in, ManagedLruCache};
+use super::monitor::TemporalJoinMetrics;
+use crate::cache::{cache_may_stale, ManagedLruCache};
 use crate::common::metrics::MetricsInfo;
 use crate::executor::join::builder::JoinStreamChunkBuilder;
 use crate::executor::prelude::*;
@@ -59,6 +60,7 @@ pub struct TemporalJoinExecutor<
     output_indices: Vec<usize>,
     chunk_size: usize,
     memo_table: Option<StateTable<S>>,
+    metrics: TemporalJoinMetrics,
 }
 
 #[derive(Default)]
@@ -106,7 +108,6 @@ struct TemporalSide<K: HashKey, S: StateStore> {
     table_stream_key_indices: Vec<usize>,
     table_output_indices: Vec<usize>,
     cache: ManagedLruCache<K, JoinEntry, DefaultHasher, SharedStatsAlloc<Global>>,
-    ctx: ActorContextRef,
     join_key_data_types: Vec<DataType>,
 }
 
@@ -117,25 +118,14 @@ impl<K: HashKey, S: StateStore> TemporalSide<K, S> {
         &mut self,
         keys: impl Iterator<Item = &K>,
         epoch: HummockEpoch,
+        metrics: &TemporalJoinMetrics,
     ) -> StreamExecutorResult<()> {
-        let table_id_str = self.source.table_id().to_string();
-        let actor_id_str = self.ctx.id.to_string();
-        let fragment_id_str = self.ctx.id.to_string();
-
         let mut futs = Vec::with_capacity(keys.size_hint().1.unwrap_or(0));
         for key in keys {
-            self.ctx
-                .streaming_metrics
-                .temporal_join_total_query_cache_count
-                .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-                .inc();
+            metrics.temporal_join_total_query_cache_count.inc();
 
             if self.cache.get(key).is_none() {
-                self.ctx
-                    .streaming_metrics
-                    .temporal_join_cache_miss_count
-                    .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-                    .inc();
+                metrics.temporal_join_cache_miss_count.inc();
 
                 futs.push(async {
                     let pk_prefix = key.deserialize(&self.join_key_data_types)?;
@@ -318,6 +308,7 @@ mod phase1 {
 
     use super::{StreamExecutorError, TemporalSide};
     use crate::common::table::state_table::StateTable;
+    use crate::executor::monitor::TemporalJoinMetrics;
 
     pub(super) trait Phase1Evaluation {
         /// Called when a matched row is found.
@@ -439,6 +430,7 @@ mod phase1 {
         memo_table: &'a mut Option<StateTable<S>>,
         null_matched: &'a K::Bitmap,
         chunk: StreamChunk,
+        metrics: &'a TemporalJoinMetrics,
     ) {
         let mut builder = StreamChunkBuilder::new(chunk_size, full_schema);
         let keys = K::build_many(left_join_keys, chunk.data_chunk());
@@ -463,7 +455,7 @@ mod phase1 {
                 }
             });
         right_table
-            .fetch_or_promote_keys(to_fetch_keys, epoch)
+            .fetch_or_promote_keys(to_fetch_keys, epoch, metrics)
             .await?;
 
         for (r, key) in chunk.rows_with_holes().zip_eq_debug(keys.into_iter()) {
@@ -579,8 +571,8 @@ mod phase1 {
     }
 }
 
-impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
-    TemporalJoinExecutor<K, S, T, A>
+impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const APPEND_ONLY: bool>
+    TemporalJoinExecutor<K, S, T, APPEND_ONLY>
 {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
@@ -596,7 +588,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
         output_indices: Vec<usize>,
         table_output_indices: Vec<usize>,
         table_stream_key_indices: Vec<usize>,
-        watermark_epoch: AtomicU64Ref,
+        watermark_sequence: AtomicU64Ref,
         metrics: Arc<StreamingMetrics>,
         chunk_size: usize,
         join_key_data_types: Vec<DataType>,
@@ -611,13 +603,15 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
             "temporal join",
         );
 
-        let cache = new_with_hasher_in(
-            watermark_epoch,
+        let cache = ManagedLruCache::unbounded_with_hasher_in(
+            watermark_sequence,
             metrics_info,
             DefaultHasher::default(),
             alloc,
         );
 
+        let metrics = metrics.new_temporal_join_metrics(table.table_id(), ctx.id, ctx.fragment_id);
+
         Self {
             ctx: ctx.clone(),
             info,
@@ -628,7 +622,6 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
                 table_stream_key_indices,
                 table_output_indices,
                 cache,
-                ctx,
                 join_key_data_types,
             },
             left_join_keys,
@@ -638,6 +631,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
             output_indices,
             chunk_size,
             memo_table,
+            metrics,
         }
     }
 
@@ -677,9 +671,6 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
 
         let mut prev_epoch = None;
 
-        let table_id_str = self.right_table.source.table_id().to_string();
-        let actor_id_str = self.ctx.id.to_string();
-        let fragment_id_str = self.ctx.fragment_id.to_string();
         let full_schema: Vec<_> = self
             .left
             .schema()
@@ -693,10 +684,8 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
         #[for_await]
         for msg in align_input(self.left, self.right) {
             self.right_table.cache.evict();
-            self.ctx
-                .streaming_metrics
+            self.metrics
                 .temporal_join_cached_entry_count
-                .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
                 .set(self.right_table.cache.len() as i64);
             match msg? {
                 InternalMessage::WaterMark(watermark) => {
@@ -709,7 +698,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
                     let full_schema = full_schema.clone();
 
                     if T == JoinType::Inner {
-                        let st1 = phase1::handle_chunk::<K, S, phase1::Inner, A>(
+                        let st1 = phase1::handle_chunk::<K, S, phase1::Inner, APPEND_ONLY>(
                             self.chunk_size,
                             right_size,
                             full_schema,
@@ -720,6 +709,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
                             &mut self.memo_table,
                             &null_matched,
                             chunk,
+                            &self.metrics,
                         );
                         #[for_await]
                         for chunk in st1 {
@@ -741,18 +731,20 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
                         }
                     } else if let Some(ref cond) = self.condition {
                         // Joined result without evaluating non-lookup conditions.
-                        let st1 = phase1::handle_chunk::<K, S, phase1::LeftOuterWithCond, A>(
-                            self.chunk_size,
-                            right_size,
-                            full_schema,
-                            epoch,
-                            &self.left_join_keys,
-                            &mut self.right_table,
-                            &memo_table_lookup_prefix,
-                            &mut self.memo_table,
-                            &null_matched,
-                            chunk,
-                        );
+                        let st1 =
+                            phase1::handle_chunk::<K, S, phase1::LeftOuterWithCond, APPEND_ONLY>(
+                                self.chunk_size,
+                                right_size,
+                                full_schema,
+                                epoch,
+                                &self.left_join_keys,
+                                &mut self.right_table,
+                                &memo_table_lookup_prefix,
+                                &mut self.memo_table,
+                                &null_matched,
+                                chunk,
+                                &self.metrics,
+                            );
                         let mut matched_count = 0usize;
                         #[for_await]
                         for chunk in st1 {
@@ -792,7 +784,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
                         // The last row should always be marker row,
                         assert_eq!(matched_count, 0);
                     } else {
-                        let st1 = phase1::handle_chunk::<K, S, phase1::LeftOuter, A>(
+                        let st1 = phase1::handle_chunk::<K, S, phase1::LeftOuter, APPEND_ONLY>(
                             self.chunk_size,
                             right_size,
                             full_schema,
@@ -803,6 +795,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
                             &mut self.memo_table,
                             &null_matched,
                             chunk,
+                            &self.metrics,
                         );
                         #[for_await]
                         for chunk in st1 {
@@ -813,7 +806,7 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
                     }
                 }
                 InternalMessage::Barrier(updates, barrier) => {
-                    if !A {
+                    if !APPEND_ONLY {
                         if wait_first_barrier {
                             wait_first_barrier = false;
                             self.memo_table.as_mut().unwrap().init_epoch(barrier.epoch);
@@ -832,7 +825,6 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
                             self.right_table.cache.clear();
                         }
                     }
-                    self.right_table.cache.update_epoch(barrier.epoch.curr);
                     self.right_table.update(
                         updates,
                         &self.right_join_keys,
@@ -846,8 +838,8 @@ impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool>
     }
 }
 
-impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const A: bool> Execute
-    for TemporalJoinExecutor<K, S, T, A>
+impl<K: HashKey, S: StateStore, const T: JoinTypePrimitive, const APPEND_ONLY: bool> Execute
+    for TemporalJoinExecutor<K, S, T, APPEND_ONLY>
 {
     fn execute(self: Box<Self>) -> super::BoxedMessageStream {
         self.into_stream().boxed()
diff --git a/src/stream/src/executor/top_n/group_top_n.rs b/src/stream/src/executor/top_n/group_top_n.rs
index 3a8a618e3c836..9457e2f7729c6 100644
--- a/src/stream/src/executor/top_n/group_top_n.rs
+++ b/src/stream/src/executor/top_n/group_top_n.rs
@@ -25,8 +25,9 @@ use risingwave_common::util::sort_util::ColumnOrder;
 use super::top_n_cache::TopNCacheTrait;
 use super::utils::*;
 use super::{ManagedTopNState, TopNCache};
-use crate::cache::{new_unbounded, ManagedLruCache};
+use crate::cache::ManagedLruCache;
 use crate::common::metrics::MetricsInfo;
+use crate::executor::monitor::GroupTopNMetrics;
 use crate::executor::prelude::*;
 
 pub type GroupTopNExecutor<K, S, const WITH_TIES: bool> =
@@ -45,20 +46,17 @@ impl<K: HashKey, S: StateStore, const WITH_TIES: bool> GroupTopNExecutor<K, S, W
         state_table: StateTable<S>,
         watermark_epoch: AtomicU64Ref,
     ) -> StreamResult<Self> {
-        Ok(TopNExecutorWrapper {
-            input,
-            ctx: ctx.clone(),
-            inner: InnerGroupTopNExecutor::new(
-                schema,
-                storage_key,
-                offset_and_limit,
-                order_by,
-                group_by,
-                state_table,
-                watermark_epoch,
-                ctx,
-            )?,
-        })
+        let inner = InnerGroupTopNExecutor::new(
+            schema,
+            storage_key,
+            offset_and_limit,
+            order_by,
+            group_by,
+            state_table,
+            watermark_epoch,
+            &ctx,
+        )?;
+        Ok(TopNExecutorWrapper { input, ctx, inner })
     }
 }
 
@@ -85,7 +83,7 @@ pub struct InnerGroupTopNExecutor<K: HashKey, S: StateStore, const WITH_TIES: bo
     /// Used for serializing pk into `CacheKey`.
     cache_key_serde: CacheKeySerde,
 
-    ctx: ActorContextRef,
+    metrics: GroupTopNMetrics,
 }
 
 impl<K: HashKey, S: StateStore, const WITH_TIES: bool> InnerGroupTopNExecutor<K, S, WITH_TIES> {
@@ -98,7 +96,7 @@ impl<K: HashKey, S: StateStore, const WITH_TIES: bool> InnerGroupTopNExecutor<K,
         group_by: Vec<usize>,
         state_table: StateTable<S>,
         watermark_epoch: AtomicU64Ref,
-        ctx: ActorContextRef,
+        ctx: &ActorContext,
     ) -> StreamResult<Self> {
         let metrics_info = MetricsInfo::new(
             ctx.streaming_metrics.clone(),
@@ -106,6 +104,11 @@ impl<K: HashKey, S: StateStore, const WITH_TIES: bool> InnerGroupTopNExecutor<K,
             ctx.id,
             "GroupTopN",
         );
+        let metrics = ctx.streaming_metrics.new_group_top_n_metrics(
+            state_table.table_id(),
+            ctx.id,
+            ctx.fragment_id,
+        );
 
         let cache_key_serde = create_cache_key_serde(&storage_key, &schema, &order_by, &group_by);
         let managed_state = ManagedTopNState::<S>::new(state_table, cache_key_serde.clone());
@@ -119,7 +122,7 @@ impl<K: HashKey, S: StateStore, const WITH_TIES: bool> InnerGroupTopNExecutor<K,
             group_by,
             caches: GroupTopNCache::new(watermark_epoch, metrics_info),
             cache_key_serde,
-            ctx,
+            metrics,
         })
     }
 }
@@ -129,8 +132,8 @@ pub struct GroupTopNCache<K: HashKey, const WITH_TIES: bool> {
 }
 
 impl<K: HashKey, const WITH_TIES: bool> GroupTopNCache<K, WITH_TIES> {
-    pub fn new(watermark_epoch: AtomicU64Ref, metrics_info: MetricsInfo) -> Self {
-        let cache = new_unbounded(watermark_epoch, metrics_info);
+    pub fn new(watermark_sequence: AtomicU64Ref, metrics_info: MetricsInfo) -> Self {
+        let cache = ManagedLruCache::unbounded(watermark_sequence, metrics_info);
         Self { data: cache }
     }
 }
@@ -158,9 +161,6 @@ where
         let mut res_ops = Vec::with_capacity(self.limit);
         let mut res_rows = Vec::with_capacity(self.limit);
         let keys = K::build_many(&self.group_by, chunk.data_chunk());
-        let table_id_str = self.managed_state.table().table_id().to_string();
-        let actor_id_str = self.ctx.id.to_string();
-        let fragment_id_str = self.ctx.fragment_id.to_string();
         for (r, group_cache_key) in chunk.rows_with_holes().zip_eq_debug(keys.iter()) {
             let Some((op, row_ref)) = r else {
                 continue;
@@ -170,19 +170,11 @@ where
             let cache_key = serialize_pk_to_cache_key(pk_row, &self.cache_key_serde);
 
             let group_key = row_ref.project(&self.group_by);
-            self.ctx
-                .streaming_metrics
-                .group_top_n_total_query_cache_count
-                .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-                .inc();
+            self.metrics.group_top_n_total_query_cache_count.inc();
             // If 'self.caches' does not already have a cache for the current group, create a new
             // cache for it and insert it into `self.caches`
             if !self.caches.contains(group_cache_key) {
-                self.ctx
-                    .streaming_metrics
-                    .group_top_n_cache_miss_count
-                    .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-                    .inc();
+                self.metrics.group_top_n_cache_miss_count.inc();
                 let mut topn_cache =
                     TopNCache::new(self.offset, self.limit, self.schema.data_types());
                 self.managed_state
@@ -215,10 +207,8 @@ where
                 }
             }
         }
-        self.ctx
-            .streaming_metrics
+        self.metrics
             .group_top_n_cached_entry_count
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
             .set(self.caches.len() as i64);
         generate_output(res_rows, res_ops, &self.schema)
     }
@@ -231,10 +221,6 @@ where
         self.managed_state.try_flush().await
     }
 
-    fn update_epoch(&mut self, epoch: u64) {
-        self.caches.update_epoch(epoch);
-    }
-
     fn update_vnode_bitmap(&mut self, vnode_bitmap: Arc<Bitmap>) {
         let cache_may_stale = self.managed_state.update_vnode_bitmap(vnode_bitmap);
         if cache_may_stale {
@@ -267,11 +253,9 @@ mod tests {
     use std::sync::atomic::AtomicU64;
 
     use assert_matches::assert_matches;
-    use futures::StreamExt;
     use risingwave_common::array::stream_chunk::StreamChunkTestExt;
-    use risingwave_common::catalog::{Field, Schema};
+    use risingwave_common::catalog::Field;
     use risingwave_common::hash::SerializedKey;
-    use risingwave_common::types::DataType;
     use risingwave_common::util::epoch::test_epoch;
     use risingwave_common::util::sort_util::OrderType;
     use risingwave_storage::memory::MemoryStateStore;
@@ -279,7 +263,6 @@ mod tests {
     use super::*;
     use crate::executor::test_utils::top_n_executor::create_in_memory_state_table;
     use crate::executor::test_utils::MockSource;
-    use crate::executor::{ActorContext, Barrier, Execute, Message};
 
     fn create_schema() -> Schema {
         Schema {
diff --git a/src/stream/src/executor/top_n/group_top_n_appendonly.rs b/src/stream/src/executor/top_n/group_top_n_appendonly.rs
index cbd3f8215c87b..b8e4094486624 100644
--- a/src/stream/src/executor/top_n/group_top_n_appendonly.rs
+++ b/src/stream/src/executor/top_n/group_top_n_appendonly.rs
@@ -25,6 +25,7 @@ use super::top_n_cache::AppendOnlyTopNCacheTrait;
 use super::utils::*;
 use super::{ManagedTopNState, TopNCache};
 use crate::common::metrics::MetricsInfo;
+use crate::executor::monitor::GroupTopNMetrics;
 use crate::executor::prelude::*;
 
 /// If the input is append-only, `AppendOnlyGroupTopNExecutor` does not need
@@ -48,20 +49,17 @@ impl<K: HashKey, S: StateStore, const WITH_TIES: bool>
         state_table: StateTable<S>,
         watermark_epoch: AtomicU64Ref,
     ) -> StreamResult<Self> {
-        Ok(TopNExecutorWrapper {
-            input,
-            ctx: ctx.clone(),
-            inner: InnerAppendOnlyGroupTopNExecutor::new(
-                schema,
-                storage_key,
-                offset_and_limit,
-                order_by,
-                group_by,
-                state_table,
-                watermark_epoch,
-                ctx,
-            )?,
-        })
+        let inner = InnerAppendOnlyGroupTopNExecutor::new(
+            schema,
+            storage_key,
+            offset_and_limit,
+            order_by,
+            group_by,
+            state_table,
+            watermark_epoch,
+            &ctx,
+        )?;
+        Ok(TopNExecutorWrapper { input, ctx, inner })
     }
 }
 
@@ -88,7 +86,7 @@ pub struct InnerAppendOnlyGroupTopNExecutor<K: HashKey, S: StateStore, const WIT
     /// Used for serializing pk into `CacheKey`.
     cache_key_serde: CacheKeySerde,
 
-    ctx: ActorContextRef,
+    metrics: GroupTopNMetrics,
 }
 
 impl<K: HashKey, S: StateStore, const WITH_TIES: bool>
@@ -103,13 +101,18 @@ impl<K: HashKey, S: StateStore, const WITH_TIES: bool>
         group_by: Vec<usize>,
         state_table: StateTable<S>,
         watermark_epoch: AtomicU64Ref,
-        ctx: ActorContextRef,
+        ctx: &ActorContext,
     ) -> StreamResult<Self> {
         let metrics_info = MetricsInfo::new(
             ctx.streaming_metrics.clone(),
             state_table.table_id(),
             ctx.id,
-            "GroupTopN",
+            "AppendOnlyGroupTopN",
+        );
+        let metrics = ctx.streaming_metrics.new_append_only_group_top_n_metrics(
+            state_table.table_id(),
+            ctx.id,
+            ctx.fragment_id,
         );
 
         let cache_key_serde = create_cache_key_serde(&storage_key, &schema, &order_by, &group_by);
@@ -124,7 +127,7 @@ impl<K: HashKey, S: StateStore, const WITH_TIES: bool>
             group_by,
             caches: GroupTopNCache::new(watermark_epoch, metrics_info),
             cache_key_serde,
-            ctx,
+            metrics,
         })
     }
 }
@@ -141,9 +144,6 @@ where
 
         let data_types = self.schema.data_types();
         let row_deserializer = RowDeserializer::new(data_types.clone());
-        let table_id_str = self.managed_state.table().table_id().to_string();
-        let actor_id_str = self.ctx.id.to_string();
-        let fragment_id_str = self.ctx.fragment_id.to_string();
         for (r, group_cache_key) in chunk.rows_with_holes().zip_eq_debug(keys.iter()) {
             let Some((op, row_ref)) = r else {
                 continue;
@@ -153,19 +153,11 @@ where
             let cache_key = serialize_pk_to_cache_key(pk_row, &self.cache_key_serde);
 
             let group_key = row_ref.project(&self.group_by);
-            self.ctx
-                .streaming_metrics
-                .group_top_n_appendonly_total_query_cache_count
-                .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-                .inc();
+            self.metrics.group_top_n_total_query_cache_count.inc();
             // If 'self.caches' does not already have a cache for the current group, create a new
             // cache for it and insert it into `self.caches`
             if !self.caches.contains(group_cache_key) {
-                self.ctx
-                    .streaming_metrics
-                    .group_top_n_appendonly_cache_miss_count
-                    .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
-                    .inc();
+                self.metrics.group_top_n_cache_miss_count.inc();
                 let mut topn_cache = TopNCache::new(self.offset, self.limit, data_types.clone());
                 self.managed_state
                     .init_topn_cache(Some(group_key), &mut topn_cache)
@@ -184,10 +176,8 @@ where
                 &row_deserializer,
             )?;
         }
-        self.ctx
-            .streaming_metrics
-            .group_top_n_appendonly_cached_entry_count
-            .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str])
+        self.metrics
+            .group_top_n_cached_entry_count
             .set(self.caches.len() as i64);
         generate_output(res_rows, res_ops, &self.schema)
     }
@@ -211,10 +201,6 @@ where
         self.caches.evict()
     }
 
-    fn update_epoch(&mut self, epoch: u64) {
-        self.caches.update_epoch(epoch)
-    }
-
     async fn init(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> {
         self.managed_state.init_epoch(epoch);
         Ok(())
diff --git a/src/stream/src/executor/top_n/top_n_appendonly.rs b/src/stream/src/executor/top_n/top_n_appendonly.rs
index c99b911b951ef..1b45c82c9c83a 100644
--- a/src/stream/src/executor/top_n/top_n_appendonly.rs
+++ b/src/stream/src/executor/top_n/top_n_appendonly.rs
@@ -19,7 +19,7 @@ use risingwave_common::util::sort_util::ColumnOrder;
 
 use super::top_n_cache::AppendOnlyTopNCacheTrait;
 use super::utils::*;
-use super::{ManagedTopNState, TopNCache, NO_GROUP_KEY};
+use super::{ManagedTopNState, TopNCache};
 use crate::executor::prelude::*;
 
 /// If the input is append-only, `AppendOnlyGroupTopNExecutor` does not need
diff --git a/src/stream/src/executor/top_n/top_n_plain.rs b/src/stream/src/executor/top_n/top_n_plain.rs
index 30aee860a8fc8..63b9ca94961f8 100644
--- a/src/stream/src/executor/top_n/top_n_plain.rs
+++ b/src/stream/src/executor/top_n/top_n_plain.rs
@@ -185,7 +185,6 @@ where
 #[cfg(test)]
 mod tests {
     use assert_matches::assert_matches;
-    use futures::StreamExt;
     use risingwave_common::array::stream_chunk::StreamChunkTestExt;
     use risingwave_common::catalog::{Field, Schema};
     use risingwave_common::types::DataType;
@@ -201,7 +200,7 @@ mod tests {
         use risingwave_common::util::epoch::test_epoch;
 
         use super::*;
-        use crate::executor::{ActorContext, Execute};
+
         fn create_stream_chunks() -> Vec<StreamChunk> {
             let chunk1 = StreamChunk::from_pretty(
                 "  I I
@@ -685,8 +684,6 @@ mod tests {
 
         use super::*;
         use crate::executor::test_utils::top_n_executor::create_in_memory_state_table_from_state_store;
-        use crate::executor::{ActorContext, Execute};
-
         fn create_source_new() -> Executor {
             let mut chunks = vec![
                 StreamChunk::from_pretty(
@@ -1002,7 +999,6 @@ mod tests {
 
         use super::*;
         use crate::executor::test_utils::top_n_executor::create_in_memory_state_table_from_state_store;
-        use crate::executor::{ActorContext, Execute};
 
         fn create_source() -> Executor {
             let mut chunks = vec![
diff --git a/src/stream/src/executor/top_n/utils.rs b/src/stream/src/executor/top_n/utils.rs
index 4700a8900221e..18409cf3d6db1 100644
--- a/src/stream/src/executor/top_n/utils.rs
+++ b/src/stream/src/executor/top_n/utils.rs
@@ -49,7 +49,6 @@ pub trait TopNExecutorBase: Send + 'static {
     }
 
     fn evict(&mut self) {}
-    fn update_epoch(&mut self, _epoch: u64) {}
 
     fn init(&mut self, epoch: EpochPair) -> impl Future<Output = StreamExecutorResult<()>> + Send;
 
@@ -114,7 +113,6 @@ where
                         self.inner.update_vnode_bitmap(vnode_bitmap);
                     }
 
-                    self.inner.update_epoch(barrier.epoch.curr);
                     yield Message::Barrier(barrier)
                 }
             };
diff --git a/src/stream/src/executor/troublemaker.rs b/src/stream/src/executor/troublemaker.rs
index 7b93392fef419..95b7ad70f7a1f 100644
--- a/src/stream/src/executor/troublemaker.rs
+++ b/src/stream/src/executor/troublemaker.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use rand::Rng;
-use risingwave_common::array::stream_chunk_builder::StreamChunkBuilder;
 use risingwave_common::array::stream_record::{Record, RecordType};
 use risingwave_common::array::Op;
 use risingwave_common::field_generator::{FieldGeneratorImpl, VarcharProperty};
diff --git a/src/stream/src/executor/union.rs b/src/stream/src/executor/union.rs
index ac8f3581dda18..c3279f7e8a344 100644
--- a/src/stream/src/executor/union.rs
+++ b/src/stream/src/executor/union.rs
@@ -15,16 +15,20 @@
 use std::collections::BTreeMap;
 use std::pin::Pin;
 use std::task::{Context, Poll};
+use std::time::Instant;
 
 use futures::stream::{FusedStream, FuturesUnordered};
 use pin_project::pin_project;
 
 use super::watermark::BufferedWatermarks;
 use crate::executor::prelude::*;
+use crate::task::FragmentId;
 
 /// `UnionExecutor` merges data from multiple inputs.
 pub struct UnionExecutor {
     inputs: Vec<Executor>,
+    metrics: Arc<StreamingMetrics>,
+    actor_context: ActorContextRef,
 }
 
 impl std::fmt::Debug for UnionExecutor {
@@ -34,15 +38,29 @@ impl std::fmt::Debug for UnionExecutor {
 }
 
 impl UnionExecutor {
-    pub fn new(inputs: Vec<Executor>) -> Self {
-        Self { inputs }
+    pub fn new(
+        inputs: Vec<Executor>,
+        metrics: Arc<StreamingMetrics>,
+        actor_context: ActorContextRef,
+    ) -> Self {
+        Self {
+            inputs,
+            metrics,
+            actor_context,
+        }
     }
 }
 
 impl Execute for UnionExecutor {
     fn execute(self: Box<Self>) -> BoxedMessageStream {
         let streams = self.inputs.into_iter().map(|e| e.execute()).collect();
-        merge(streams).boxed()
+        merge(
+            streams,
+            self.metrics,
+            self.actor_context.fragment_id,
+            self.actor_context.id,
+        )
+        .boxed()
     }
 }
 
@@ -63,7 +81,12 @@ impl Stream for Input {
 
 /// Merges input streams and aligns with barriers.
 #[try_stream(ok = Message, error = StreamExecutorError)]
-async fn merge(inputs: Vec<BoxedMessageStream>) {
+async fn merge(
+    inputs: Vec<BoxedMessageStream>,
+    metrics: Arc<StreamingMetrics>,
+    fragment_id: FragmentId,
+    actor_id: ActorId,
+) {
     let input_num = inputs.len();
     let mut active: FuturesUnordered<_> = inputs
         .into_iter()
@@ -82,6 +105,13 @@ async fn merge(inputs: Vec<BoxedMessageStream>) {
     // watermark column index -> `BufferedWatermarks`
     let mut watermark_buffers = BTreeMap::<usize, BufferedWatermarks<usize>>::new();
 
+    let mut start_time = Instant::now();
+    let barrier_align = metrics.barrier_align_duration.with_label_values(&[
+        &actor_id.to_string(),
+        &fragment_id.to_string(),
+        "",
+        "Union",
+    ]);
     loop {
         match active.next().await {
             Some((Some(Ok(message)), remaining)) => {
@@ -106,6 +136,9 @@ async fn merge(inputs: Vec<BoxedMessageStream>) {
                     }
                     Message::Barrier(barrier) => {
                         // Block this upstream by pushing it to `blocked`.
+                        if blocked.is_empty() {
+                            start_time = Instant::now();
+                        }
                         blocked.push(remaining);
                         if let Some(cur_barrier) = current_barrier.as_ref() {
                             if barrier.epoch != cur_barrier.epoch {
@@ -131,6 +164,7 @@ async fn merge(inputs: Vec<BoxedMessageStream>) {
             None => {
                 assert!(active.is_terminated());
                 let barrier = current_barrier.take().unwrap();
+                barrier_align.observe(start_time.elapsed().as_secs_f64());
 
                 let upstreams = std::mem::take(&mut blocked);
                 active.extend(upstreams.into_iter().map(|upstream| upstream.into_future()));
@@ -145,12 +179,9 @@ async fn merge(inputs: Vec<BoxedMessageStream>) {
 mod tests {
     use async_stream::try_stream;
     use risingwave_common::array::stream_chunk::StreamChunkTestExt;
-    use risingwave_common::array::StreamChunk;
-    use risingwave_common::types::{DataType, ScalarImpl};
     use risingwave_common::util::epoch::test_epoch;
 
     use super::*;
-    use crate::executor::Watermark;
 
     #[tokio::test]
     async fn union() {
@@ -177,7 +208,7 @@ mod tests {
             .boxed(),
         ];
         let mut output = vec![];
-        let mut merged = merge(streams).boxed();
+        let mut merged = merge(streams, Arc::new(StreamingMetrics::unused()), 0, 0).boxed();
 
         let result = vec![
             Message::Chunk(StreamChunk::from_pretty("I\n + 1")),
diff --git a/src/stream/src/executor/utils.rs b/src/stream/src/executor/utils.rs
index 565866e0d9e95..37ed7803a44c7 100644
--- a/src/stream/src/executor/utils.rs
+++ b/src/stream/src/executor/utils.rs
@@ -12,10 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use risingwave_common::metrics::LabelGuardedIntCounter;
-
 use crate::executor::prelude::*;
-use crate::task::FragmentId;
 
 #[derive(Default)]
 pub struct DummyExecutor;
@@ -25,35 +22,3 @@ impl Execute for DummyExecutor {
         futures::stream::pending().boxed()
     }
 }
-
-pub(crate) struct ActorInputMetrics {
-    pub(crate) actor_in_record_cnt: LabelGuardedIntCounter<3>,
-    pub(crate) actor_input_buffer_blocking_duration_ns: LabelGuardedIntCounter<3>,
-}
-
-impl ActorInputMetrics {
-    pub(crate) fn new(
-        metrics: &StreamingMetrics,
-        actor_id: ActorId,
-        fragment_id: FragmentId,
-        upstream_fragment_id: FragmentId,
-    ) -> Self {
-        let actor_id_str = actor_id.to_string();
-        let fragment_id_str = fragment_id.to_string();
-        let upstream_fragment_id_str = upstream_fragment_id.to_string();
-        Self {
-            actor_in_record_cnt: metrics.actor_in_record_cnt.with_guarded_label_values(&[
-                &actor_id_str,
-                &fragment_id_str,
-                &upstream_fragment_id_str,
-            ]),
-            actor_input_buffer_blocking_duration_ns: metrics
-                .actor_input_buffer_blocking_duration_ns
-                .with_guarded_label_values(&[
-                    &actor_id_str,
-                    &fragment_id_str,
-                    &upstream_fragment_id_str,
-                ]),
-        }
-    }
-}
diff --git a/src/stream/src/executor/watermark_filter.rs b/src/stream/src/executor/watermark_filter.rs
index 813ccbef28920..2aca1251dd058 100644
--- a/src/stream/src/executor/watermark_filter.rs
+++ b/src/stream/src/executor/watermark_filter.rs
@@ -17,7 +17,7 @@ use std::ops::Deref;
 
 use futures::future::{try_join, try_join_all};
 use risingwave_common::hash::VnodeBitmapExt;
-use risingwave_common::types::{DefaultOrd, ScalarImpl};
+use risingwave_common::types::DefaultOrd;
 use risingwave_common::{bail, row};
 use risingwave_expr::expr::{
     build_func_non_strict, ExpressionBoxExt, InputRefExpression, LiteralExpression,
@@ -349,8 +349,7 @@ impl<S: StateStore> WatermarkFilterExecutor<S> {
 #[cfg(test)]
 mod tests {
     use itertools::Itertools;
-    use risingwave_common::array::StreamChunk;
-    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, Schema, TableDesc};
+    use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, TableDesc};
     use risingwave_common::test_prelude::StreamChunkTestExt;
     use risingwave_common::types::Date;
     use risingwave_common::util::epoch::test_epoch;
@@ -359,12 +358,10 @@ mod tests {
     use risingwave_pb::common::ColumnOrder;
     use risingwave_pb::plan_common::PbColumnCatalog;
     use risingwave_storage::memory::MemoryStateStore;
-    use risingwave_storage::table::TableDistribution;
 
     use super::*;
     use crate::executor::test_utils::expr::build_from_pretty;
     use crate::executor::test_utils::{MessageSender, MockSource};
-    use crate::executor::{ActorContext, ExecutorInfo};
 
     const WATERMARK_TYPE: DataType = DataType::Timestamp;
 
diff --git a/src/stream/src/executor/wrapper/epoch_check.rs b/src/stream/src/executor/wrapper/epoch_check.rs
index 0110d8a0d86e6..335c524700f01 100644
--- a/src/stream/src/executor/wrapper/epoch_check.rs
+++ b/src/stream/src/executor/wrapper/epoch_check.rs
@@ -75,7 +75,7 @@ pub async fn epoch_check(info: Arc<ExecutorInfo>, input: impl MessageStream) {
 #[cfg(test)]
 mod tests {
     use assert_matches::assert_matches;
-    use futures::{pin_mut, StreamExt};
+    use futures::pin_mut;
     use risingwave_common::array::StreamChunk;
     use risingwave_common::util::epoch::test_epoch;
 
diff --git a/src/stream/src/executor/wrapper/trace.rs b/src/stream/src/executor/wrapper/trace.rs
index c95809f534728..4c1268cc8698b 100644
--- a/src/stream/src/executor/wrapper/trace.rs
+++ b/src/stream/src/executor/wrapper/trace.rs
@@ -33,6 +33,11 @@ pub async fn trace(
     let actor_id_str = actor_ctx.id.to_string();
     let fragment_id_str = actor_ctx.fragment_id.to_string();
 
+    let executor_row_count = actor_ctx
+        .streaming_metrics
+        .executor_row_count
+        .with_guarded_label_values(&[&actor_id_str, &fragment_id_str, &info.identity]);
+
     let new_span = || {
         tracing::info_span!(
             "executor",
@@ -50,11 +55,7 @@ pub async fn trace(
         match &message {
             Message::Chunk(chunk) => {
                 if enable_executor_row_count {
-                    actor_ctx
-                        .streaming_metrics
-                        .executor_row_count
-                        .with_label_values(&[&actor_id_str, &fragment_id_str, &info.identity])
-                        .inc_by(chunk.cardinality() as u64);
+                    executor_row_count.inc_by(chunk.cardinality() as u64);
                 }
                 tracing::debug!(
                     target: "events::stream::message::chunk",
diff --git a/src/stream/src/from_proto/batch_query.rs b/src/stream/src/from_proto/batch_query.rs
index 5e86d7e6d5b30..a1659008724e7 100644
--- a/src/stream/src/from_proto/batch_query.rs
+++ b/src/stream/src/from_proto/batch_query.rs
@@ -16,7 +16,6 @@ use risingwave_common::catalog::ColumnId;
 use risingwave_pb::plan_common::StorageTableDesc;
 use risingwave_pb::stream_plan::BatchPlanNode;
 use risingwave_storage::table::batch_table::storage_table::StorageTable;
-use risingwave_storage::StateStore;
 
 use super::*;
 use crate::executor::{BatchQueryExecutor, DummyExecutor};
diff --git a/src/stream/src/from_proto/group_top_n.rs b/src/stream/src/from_proto/group_top_n.rs
index 51512cb9d94d3..fe5dfc1042cc8 100644
--- a/src/stream/src/from_proto/group_top_n.rs
+++ b/src/stream/src/from_proto/group_top_n.rs
@@ -22,7 +22,7 @@ use risingwave_pb::stream_plan::GroupTopNNode;
 
 use super::*;
 use crate::common::table::state_table::StateTable;
-use crate::executor::{ActorContextRef, AppendOnlyGroupTopNExecutor, Executor, GroupTopNExecutor};
+use crate::executor::{ActorContextRef, AppendOnlyGroupTopNExecutor, GroupTopNExecutor};
 use crate::task::AtomicU64Ref;
 
 pub struct GroupTopNExecutorBuilder<const APPEND_ONLY: bool>;
diff --git a/src/stream/src/from_proto/hash_join.rs b/src/stream/src/from_proto/hash_join.rs
index ad69fd1ecd5ac..2d421274cec39 100644
--- a/src/stream/src/from_proto/hash_join.rs
+++ b/src/stream/src/from_proto/hash_join.rs
@@ -27,7 +27,7 @@ use super::*;
 use crate::common::table::state_table::StateTable;
 use crate::executor::hash_join::*;
 use crate::executor::monitor::StreamingMetrics;
-use crate::executor::{ActorContextRef, Executor, JoinType};
+use crate::executor::{ActorContextRef, JoinType};
 use crate::task::AtomicU64Ref;
 
 pub struct HashJoinExecutorBuilder;
@@ -155,6 +155,11 @@ impl ExecutorBuilder for HashJoinExecutorBuilder {
             join_type_proto: node.get_join_type()?,
             join_key_data_types,
             chunk_size: params.env.config().developer.chunk_size,
+            high_join_amplification_threshold: params
+                .env
+                .config()
+                .developer
+                .high_join_amplification_threshold,
         };
 
         let exec = args.dispatch()?;
@@ -183,6 +188,7 @@ struct HashJoinExecutorDispatcherArgs<S: StateStore> {
     join_type_proto: JoinTypeProto,
     join_key_data_types: Vec<DataType>,
     chunk_size: usize,
+    high_join_amplification_threshold: usize,
 }
 
 impl<S: StateStore> HashKeyDispatcher for HashJoinExecutorDispatcherArgs<S> {
@@ -211,6 +217,7 @@ impl<S: StateStore> HashKeyDispatcher for HashJoinExecutorDispatcherArgs<S> {
                     self.is_append_only,
                     self.metrics,
                     self.chunk_size,
+                    self.high_join_amplification_threshold,
                 )
                 .boxed())
             };
diff --git a/src/stream/src/from_proto/mod.rs b/src/stream/src/from_proto/mod.rs
index 4c3323e864210..ba433595718a3 100644
--- a/src/stream/src/from_proto/mod.rs
+++ b/src/stream/src/from_proto/mod.rs
@@ -46,7 +46,6 @@ mod source_backfill;
 mod stateless_simple_agg;
 mod stream_cdc_scan;
 mod stream_scan;
-mod subscription;
 mod temporal_join;
 mod top_n;
 mod union;
@@ -96,7 +95,6 @@ use self::union::*;
 use self::watermark_filter::WatermarkFilterBuilder;
 use crate::error::StreamResult;
 use crate::executor::{Execute, Executor, ExecutorInfo};
-use crate::from_proto::subscription::SubscriptionExecutorBuilder;
 use crate::from_proto::values::ValuesExecutorBuilder;
 use crate::task::ExecutorParams;
 
@@ -149,7 +147,6 @@ pub async fn create_executor(
         NodeBody::BatchPlan => BatchQueryExecutorBuilder,
         NodeBody::Merge => MergeExecutorBuilder,
         NodeBody::Materialize => MaterializeExecutorBuilder,
-        NodeBody::Subscription => SubscriptionExecutorBuilder,
         NodeBody::Filter => FilterExecutorBuilder,
         NodeBody::CdcFilter => CdcFilterExecutorBuilder,
         NodeBody::Arrange => ArrangeExecutorBuilder,
diff --git a/src/stream/src/from_proto/project.rs b/src/stream/src/from_proto/project.rs
index d7f96c4dffcbf..177045b3eba7d 100644
--- a/src/stream/src/from_proto/project.rs
+++ b/src/stream/src/from_proto/project.rs
@@ -67,6 +67,7 @@ impl ExecutorBuilder for ProjectExecutorBuilder {
             watermark_derivations,
             nondecreasing_expr_indices,
             materialize_selectivity_threshold,
+            node.noop_update_hint,
         );
         Ok((params.info, exec).into())
     }
diff --git a/src/stream/src/from_proto/project_set.rs b/src/stream/src/from_proto/project_set.rs
index c2338394b33ef..ad6158c91dff0 100644
--- a/src/stream/src/from_proto/project_set.rs
+++ b/src/stream/src/from_proto/project_set.rs
@@ -14,11 +14,10 @@
 
 use multimap::MultiMap;
 use risingwave_common::util::iter_util::ZipEqFast;
-use risingwave_expr::table_function::ProjectSetSelectItem;
 use risingwave_pb::stream_plan::ProjectSetNode;
 
 use super::*;
-use crate::executor::ProjectSetExecutor;
+use crate::executor::{ProjectSetExecutor, ProjectSetSelectItem};
 
 pub struct ProjectSetExecutorBuilder;
 
@@ -35,7 +34,11 @@ impl ExecutorBuilder for ProjectSetExecutorBuilder {
             .get_select_list()
             .iter()
             .map(|proto| {
-                ProjectSetSelectItem::from_prost(proto, params.env.config().developer.chunk_size)
+                ProjectSetSelectItem::from_prost(
+                    proto,
+                    params.eval_error_report.clone(),
+                    params.env.config().developer.chunk_size,
+                )
             })
             .try_collect()?;
         let watermark_derivations = MultiMap::from_iter(
@@ -62,6 +65,7 @@ impl ExecutorBuilder for ProjectSetExecutorBuilder {
             chunk_size,
             watermark_derivations,
             nondecreasing_expr_indices,
+            params.eval_error_report,
         );
         Ok((params.info, exec).into())
     }
diff --git a/src/stream/src/from_proto/sink.rs b/src/stream/src/from_proto/sink.rs
index b583c553ce637..5e77be7beb7a0 100644
--- a/src/stream/src/from_proto/sink.rs
+++ b/src/stream/src/from_proto/sink.rs
@@ -180,7 +180,6 @@ impl ExecutorBuilder for SinkExecutorBuilder {
         );
 
         let sink_write_param = SinkWriterParam {
-            connector_params: params.env.connector_params(),
             executor_id: params.executor_id,
             vnode_bitmap: params.vnode_bitmap.clone(),
             meta_client: params.env.meta_client().map(SinkMetaClient::MetaClient),
@@ -215,7 +214,7 @@ impl ExecutorBuilder for SinkExecutorBuilder {
             SinkLogStoreType::KvLogStore => {
                 let metrics = KvLogStoreMetrics::new(
                     &params.executor_stats,
-                    &sink_write_param,
+                    &params.info.identity,
                     &sink_param,
                     connector,
                 );
diff --git a/src/stream/src/from_proto/source/fs_fetch.rs b/src/stream/src/from_proto/source/fs_fetch.rs
index 04baf944b552f..1951365a47eed 100644
--- a/src/stream/src/from_proto/source/fs_fetch.rs
+++ b/src/stream/src/from_proto/source/fs_fetch.rs
@@ -19,7 +19,7 @@ use risingwave_connector::source::filesystem::opendal_source::{
     OpendalGcs, OpendalPosixFs, OpendalS3,
 };
 use risingwave_connector::source::reader::desc::SourceDescBuilder;
-use risingwave_connector::source::{ConnectorProperties, SourceCtrlOpts};
+use risingwave_connector::source::ConnectorProperties;
 use risingwave_pb::stream_plan::StreamFsFetchNode;
 use risingwave_storage::StateStore;
 
@@ -53,14 +53,9 @@ impl ExecutorBuilder for FsFetchExecutorBuilder {
             source.row_id_index.map(|x| x as _),
             source.with_properties.clone(),
             source_info.clone(),
-            params.env.connector_params(),
             params.env.config().developer.connector_message_buffer_size,
             params.info.pk_indices.clone(),
         );
-        let source_ctrl_opts = SourceCtrlOpts {
-            chunk_size: params.env.config().developer.chunk_size,
-            rate_limit: source.rate_limit.map(|x| x as _),
-        };
 
         let source_column_ids: Vec<_> = source_desc_builder
             .column_catalogs_to_source_column_descs()
@@ -93,7 +88,7 @@ impl ExecutorBuilder for FsFetchExecutorBuilder {
                     params.actor_context.clone(),
                     stream_source_core,
                     upstream,
-                    source_ctrl_opts,
+                    source.rate_limit,
                 )
                 .boxed()
             }
@@ -102,7 +97,7 @@ impl ExecutorBuilder for FsFetchExecutorBuilder {
                     params.actor_context.clone(),
                     stream_source_core,
                     upstream,
-                    source_ctrl_opts,
+                    source.rate_limit,
                 )
                 .boxed()
             }
@@ -111,7 +106,7 @@ impl ExecutorBuilder for FsFetchExecutorBuilder {
                     params.actor_context.clone(),
                     stream_source_core,
                     upstream,
-                    source_ctrl_opts,
+                    source.rate_limit,
                 )
                 .boxed()
             }
diff --git a/src/stream/src/from_proto/source/trad_source.rs b/src/stream/src/from_proto/source/trad_source.rs
index ac305e53d386a..667f2c5d49bc9 100644
--- a/src/stream/src/from_proto/source/trad_source.rs
+++ b/src/stream/src/from_proto/source/trad_source.rs
@@ -12,15 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 
 use risingwave_common::catalog::{
     default_key_column_name_version_mapping, TableId, KAFKA_TIMESTAMP_COLUMN_NAME,
 };
 use risingwave_connector::source::reader::desc::SourceDescBuilder;
-use risingwave_connector::source::{
-    should_copy_to_format_encode_options, SourceCtrlOpts, UPSTREAM_SOURCE_KEY,
-};
+use risingwave_connector::source::{should_copy_to_format_encode_options, UPSTREAM_SOURCE_KEY};
 use risingwave_connector::WithPropertiesExt;
 use risingwave_pb::catalog::PbStreamSourceInfo;
 use risingwave_pb::data::data_type::TypeName as PbTypeName;
@@ -48,7 +46,7 @@ pub fn create_source_desc_builder(
     params: &ExecutorParams,
     source_info: PbStreamSourceInfo,
     row_id_index: Option<u32>,
-    with_properties: HashMap<String, String>,
+    with_properties: BTreeMap<String, String>,
 ) -> SourceDescBuilder {
     {
         // compatible code: introduced in https://github.com/risingwavelabs/risingwave/pull/13707
@@ -120,7 +118,6 @@ pub fn create_source_desc_builder(
         row_id_index.map(|x| x as _),
         with_properties,
         source_info,
-        params.env.connector_params(),
         params.env.config().developer.connector_message_buffer_size,
         // `pk_indices` is used to ensure that a message will be skipped instead of parsed
         // with null pk when the pk column is missing.
@@ -179,11 +176,6 @@ impl ExecutorBuilder for SourceExecutorBuilder {
                     source.with_properties.clone(),
                 );
 
-                let source_ctrl_opts = SourceCtrlOpts {
-                    chunk_size: params.env.config().developer.chunk_size,
-                    rate_limit: source.rate_limit.map(|x| x as _),
-                };
-
                 let source_column_ids: Vec<_> = source_desc_builder
                     .column_catalogs_to_source_column_descs()
                     .iter()
@@ -219,7 +211,7 @@ impl ExecutorBuilder for SourceExecutorBuilder {
                         params.executor_stats,
                         barrier_receiver,
                         system_params,
-                        source_ctrl_opts,
+                        source.rate_limit,
                     )?
                     .boxed()
                 } else if is_fs_v2_connector {
@@ -229,17 +221,19 @@ impl ExecutorBuilder for SourceExecutorBuilder {
                         params.executor_stats.clone(),
                         barrier_receiver,
                         system_params,
-                        source_ctrl_opts.clone(),
+                        source.rate_limit,
                     )
                     .boxed()
                 } else {
+                    let is_shared = source.info.as_ref().is_some_and(|info| info.is_shared());
                     SourceExecutor::new(
                         params.actor_context.clone(),
                         Some(stream_source_core),
                         params.executor_stats.clone(),
                         barrier_receiver,
                         system_params,
-                        source_ctrl_opts.clone(),
+                        source.rate_limit,
+                        is_shared,
                     )
                     .boxed()
                 }
@@ -268,11 +262,8 @@ impl ExecutorBuilder for SourceExecutorBuilder {
                 params.executor_stats,
                 barrier_receiver,
                 system_params,
-                // we don't expect any data in, so no need to set chunk_sizes
-                SourceCtrlOpts {
-                    chunk_size: 0,
-                    rate_limit: None,
-                },
+                None,
+                false,
             );
             Ok((params.info, exec).into())
         }
diff --git a/src/stream/src/from_proto/source_backfill.rs b/src/stream/src/from_proto/source_backfill.rs
index 12b159b6fdd7c..84f4bf7adab86 100644
--- a/src/stream/src/from_proto/source_backfill.rs
+++ b/src/stream/src/from_proto/source_backfill.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use risingwave_common::catalog::TableId;
-use risingwave_connector::source::SourceCtrlOpts;
 use risingwave_pb::stream_plan::SourceBackfillNode;
 
 use super::*;
@@ -44,11 +43,6 @@ impl ExecutorBuilder for SourceBackfillExecutorBuilder {
             node.with_properties.clone(),
         );
 
-        let source_ctrl_opts = SourceCtrlOpts {
-            chunk_size: params.env.config().developer.chunk_size,
-            rate_limit: node.rate_limit,
-        };
-
         let source_column_ids: Vec<_> = source_desc_builder
             .column_catalogs_to_source_column_descs()
             .iter()
@@ -80,8 +74,8 @@ impl ExecutorBuilder for SourceBackfillExecutorBuilder {
             stream_source_core,
             params.executor_stats.clone(),
             params.env.system_params_manager_ref().get_params(),
-            source_ctrl_opts.clone(),
             backfill_state_table,
+            node.rate_limit,
         );
         let [input]: [_; 1] = params.input.try_into().unwrap();
 
diff --git a/src/stream/src/from_proto/stream_cdc_scan.rs b/src/stream/src/from_proto/stream_cdc_scan.rs
index 2f035fc4f270a..150812c57a1c4 100644
--- a/src/stream/src/from_proto/stream_cdc_scan.rs
+++ b/src/stream/src/from_proto/stream_cdc_scan.rs
@@ -15,15 +15,18 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
+use anyhow::anyhow;
 use risingwave_common::catalog::{Schema, TableId};
 use risingwave_common::util::sort_util::OrderType;
-use risingwave_connector::source::cdc::external::{CdcTableType, SchemaTableName};
+use risingwave_connector::source::cdc::external::{
+    CdcTableType, ExternalTableConfig, SchemaTableName,
+};
 use risingwave_pb::plan_common::ExternalTableDesc;
 use risingwave_pb::stream_plan::StreamCdcScanNode;
 
 use super::*;
 use crate::common::table::state_table::StateTable;
-use crate::executor::{CdcBackfillExecutor, Executor, ExternalStorageTable};
+use crate::executor::{CdcBackfillExecutor, CdcScanOptions, ExternalStorageTable};
 
 pub struct StreamCdcScanExecutorBuilder;
 
@@ -44,21 +47,16 @@ impl ExecutorBuilder for StreamCdcScanExecutorBuilder {
             .collect_vec();
 
         let table_desc: &ExternalTableDesc = node.get_cdc_table_desc()?;
-        let disable_backfill = node.disable_backfill;
 
-        let table_schema: Schema = table_desc.columns.iter().map(Into::into).collect();
-        assert_eq!(output_indices, (0..table_schema.len()).collect_vec());
-        assert_eq!(table_schema.data_types(), params.info.schema.data_types());
+        let output_schema: Schema = table_desc.columns.iter().map(Into::into).collect();
+        assert_eq!(output_indices, (0..output_schema.len()).collect_vec());
+        assert_eq!(output_schema.data_types(), params.info.schema.data_types());
 
         let properties: HashMap<String, String> = table_desc
             .connect_properties
             .iter()
             .map(|(k, v)| (k.clone(), v.clone()))
             .collect();
-        let table_type = CdcTableType::from_properties(&properties);
-        let table_reader = table_type
-            .create_table_reader(properties.clone(), table_schema.clone())
-            .await?;
 
         let table_pk_order_types = table_desc
             .pk
@@ -71,15 +69,51 @@ impl ExecutorBuilder for StreamCdcScanExecutorBuilder {
             .map(|k| k.column_index as usize)
             .collect_vec();
 
+        let scan_options = node
+            .options
+            .as_ref()
+            .map(CdcScanOptions::from_proto)
+            .unwrap_or(CdcScanOptions {
+                disable_backfill: node.disable_backfill,
+                ..Default::default()
+            });
+        let table_type = CdcTableType::from_properties(&properties);
+
+        // Filter out additional columns to construct the external table schema
+        let table_schema: Schema = table_desc
+            .columns
+            .iter()
+            .filter(|col| {
+                !col.additional_column
+                    .as_ref()
+                    .is_some_and(|a_col| a_col.column_type.is_some())
+            })
+            .map(Into::into)
+            .collect();
+
         let schema_table_name = SchemaTableName::from_properties(&properties);
+        let table_config = serde_json::from_value::<ExternalTableConfig>(
+            serde_json::to_value(properties).unwrap(),
+        )
+        .map_err(|e| anyhow!("failed to parse external table config").context(e))?;
+        let database_name = table_config.database.clone();
+        let table_reader = table_type
+            .create_table_reader(
+                table_config,
+                table_schema.clone(),
+                table_pk_indices.clone(),
+                scan_options.snapshot_batch_size,
+            )
+            .await?;
+
         let external_table = ExternalStorageTable::new(
             TableId::new(table_desc.table_id),
             schema_table_name,
+            database_name,
             table_reader,
             table_schema,
             table_pk_order_types,
             table_pk_indices,
-            output_indices.clone(),
         );
 
         let vnodes = params.vnode_bitmap.map(Arc::new);
@@ -88,23 +122,18 @@ impl ExecutorBuilder for StreamCdcScanExecutorBuilder {
         let state_table =
             StateTable::from_table_catalog(node.get_state_table()?, state_store, vnodes).await;
 
-        // adjust backfill chunk size if rate limit is set.
-        let chunk_size = params.env.config().developer.chunk_size;
-        let backfill_chunk_size = node
-            .rate_limit
-            .map(|x| std::cmp::min(x as usize, chunk_size))
-            .unwrap_or(chunk_size);
-
+        let output_columns = table_desc.columns.iter().map(Into::into).collect_vec();
         let exec = CdcBackfillExecutor::new(
             params.actor_context.clone(),
             external_table,
             upstream,
             output_indices,
+            output_columns,
             None,
             params.executor_stats,
             state_table,
-            backfill_chunk_size,
-            disable_backfill,
+            node.rate_limit,
+            scan_options,
         );
         Ok((params.info, exec).into())
     }
diff --git a/src/stream/src/from_proto/subscription.rs b/src/stream/src/from_proto/subscription.rs
deleted file mode 100644
index a7e8b79b16c10..0000000000000
--- a/src/stream/src/from_proto/subscription.rs
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use risingwave_common::catalog::{TableId, TableOption};
-use risingwave_pb::stream_plan::SubscriptionNode;
-use risingwave_storage::store::{NewLocalOptions, OpConsistencyLevel};
-
-use super::ExecutorBuilder;
-use crate::common::log_store_impl::kv_log_store::serde::LogStoreRowSerde;
-use crate::common::log_store_impl::kv_log_store::KV_LOG_STORE_V2_INFO;
-use crate::common::log_store_impl::subscription_log_store::SubscriptionLogStoreWriter;
-use crate::error::StreamResult;
-use crate::executor::{Executor, SubscriptionExecutor};
-
-pub struct SubscriptionExecutorBuilder;
-
-impl ExecutorBuilder for SubscriptionExecutorBuilder {
-    type Node = SubscriptionNode;
-
-    async fn new_boxed_executor(
-        params: crate::task::ExecutorParams,
-        node: &Self::Node,
-        state_store: impl risingwave_storage::StateStore,
-    ) -> StreamResult<Executor> {
-        let [input]: [_; 1] = params.input.try_into().unwrap();
-        let table_id = TableId::new(node.log_store_table.as_ref().unwrap().id);
-        let vnodes = std::sync::Arc::new(
-            params
-                .vnode_bitmap
-                .expect("vnodes not set for subscription"),
-        );
-        let serde = LogStoreRowSerde::new(
-            node.log_store_table.as_ref().unwrap(),
-            Some(vnodes),
-            &KV_LOG_STORE_V2_INFO,
-        );
-
-        let local_state_store = state_store
-            .new_local(NewLocalOptions {
-                table_id: TableId {
-                    table_id: node.log_store_table.as_ref().unwrap().id,
-                },
-                op_consistency_level: OpConsistencyLevel::Inconsistent,
-                table_option: TableOption {
-                    retention_seconds: None,
-                },
-                is_replicated: false,
-                vnodes: serde.vnodes().clone(),
-            })
-            .await;
-
-        let log_store_identity = format!(
-            "subscription[{}]-executor[{}]",
-            node.subscription_catalog.as_ref().unwrap().id,
-            params.executor_id
-        );
-        let log_store =
-            SubscriptionLogStoreWriter::new(table_id, local_state_store, serde, log_store_identity);
-        let exec = SubscriptionExecutor::new(
-            params.actor_context,
-            input,
-            log_store,
-            node.subscription_catalog
-                .as_ref()
-                .unwrap()
-                .properties
-                .clone(),
-        )
-        .await?;
-        Ok((params.info, exec).into())
-    }
-}
diff --git a/src/stream/src/from_proto/union.rs b/src/stream/src/from_proto/union.rs
index 1360f92c1c83b..ee06b1068491c 100644
--- a/src/stream/src/from_proto/union.rs
+++ b/src/stream/src/from_proto/union.rs
@@ -27,6 +27,10 @@ impl ExecutorBuilder for UnionExecutorBuilder {
         _node: &Self::Node,
         _store: impl StateStore,
     ) -> StreamResult<Executor> {
-        Ok((params.info, UnionExecutor::new(params.input)).into())
+        Ok((
+            params.info,
+            UnionExecutor::new(params.input, params.executor_stats, params.actor_context),
+        )
+            .into())
     }
 }
diff --git a/src/stream/src/lib.rs b/src/stream/src/lib.rs
index 1a78320de8cd1..51a735af0f5a7 100644
--- a/src/stream/src/lib.rs
+++ b/src/stream/src/lib.rs
@@ -32,7 +32,6 @@
 #![feature(lazy_cell)]
 #![feature(error_generic_member_access)]
 #![feature(btree_extract_if)]
-#![feature(bound_map)]
 #![feature(iter_order_by)]
 #![feature(exact_size_is_empty)]
 #![feature(impl_trait_in_assoc_type)]
@@ -63,6 +62,18 @@ tokio::task_local! {
     pub(crate) static CONFIG: Arc<StreamingConfig>;
 }
 
+mod config {
+    use risingwave_common::config::default;
+
+    pub(crate) fn chunk_size() -> usize {
+        let res = crate::CONFIG.try_with(|config| config.developer.chunk_size);
+        if res.is_err() && cfg!(not(test)) {
+            tracing::warn!("streaming CONFIG is not set, which is probably a bug")
+        }
+        res.unwrap_or_else(|_| default::developer::stream_chunk_size())
+    }
+}
+
 mod consistency {
     //! This module contains global variables and methods to access the stream consistency settings.
 
@@ -82,12 +93,10 @@ mod consistency {
     /// Check if strict consistency is required.
     pub(crate) fn enable_strict_consistency() -> bool {
         let res = crate::CONFIG.try_with(|config| config.unsafe_enable_strict_consistency);
-        if cfg!(test) {
-            // use default value in tests
-            res.unwrap_or_else(|_| default::streaming::unsafe_enable_strict_consistency())
-        } else {
-            res.expect("streaming CONFIG is not set, which is highly probably a bug")
+        if res.is_err() && cfg!(not(test)) {
+            tracing::warn!("streaming CONFIG is not set, which is probably a bug");
         }
+        res.unwrap_or_else(|_| default::streaming::unsafe_enable_strict_consistency())
     }
 
     /// Log an error message for breaking consistency. Must only be called in non-strict mode.
diff --git a/src/stream/src/task/barrier_manager.rs b/src/stream/src/task/barrier_manager.rs
index 4549552bab607..c5d564d9c6ab8 100644
--- a/src/stream/src/task/barrier_manager.rs
+++ b/src/stream/src/task/barrier_manager.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeSet, HashMap, HashSet};
 use std::future::pending;
 use std::sync::Arc;
 use std::time::Duration;
@@ -50,18 +50,18 @@ use risingwave_common::util::runtime::BackgroundShutdownRuntime;
 use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map;
 use risingwave_hummock_sdk::{LocalSstableInfo, SyncResult};
 use risingwave_pb::common::ActorInfo;
-use risingwave_pb::stream_plan;
 use risingwave_pb::stream_plan::barrier::BarrierKind;
 use risingwave_pb::stream_service::streaming_control_stream_request::{InitRequest, Request};
 use risingwave_pb::stream_service::streaming_control_stream_response::InitResponse;
 use risingwave_pb::stream_service::{
-    streaming_control_stream_response, BarrierCompleteResponse, StreamingControlStreamRequest,
-    StreamingControlStreamResponse,
+    streaming_control_stream_response, BarrierCompleteResponse, BuildActorInfo,
+    StreamingControlStreamRequest, StreamingControlStreamResponse,
 };
 
 use crate::executor::exchange::permit::Receiver;
 use crate::executor::monitor::StreamingMetrics;
 use crate::executor::{Actor, Barrier, DispatchExecutor, Mutation, StreamExecutorError};
+use crate::task::barrier_manager::managed_state::ManagedBarrierStateDebugInfo;
 use crate::task::barrier_manager::progress::BackfillState;
 
 /// If enabled, all actors will be grouped in the same tracing span within one epoch.
@@ -100,6 +100,10 @@ impl ControlStreamHandle {
         }
     }
 
+    pub(super) fn connected(&self) -> bool {
+        self.pair.is_some()
+    }
+
     fn reset_stream_with_err(&mut self, err: Status) {
         if let Some((sender, _)) = self.pair.take() {
             // Note: `TonicStatusWrapper` provides a better error report.
@@ -209,7 +213,7 @@ pub(super) enum LocalActorOperation {
         result_sender: oneshot::Sender<()>,
     },
     UpdateActors {
-        actors: Vec<stream_plan::StreamActor>,
+        actors: Vec<BuildActorInfo>,
         result_sender: oneshot::Sender<StreamResult<()>>,
     },
     BuildActors {
@@ -232,6 +236,9 @@ pub(super) enum LocalActorOperation {
         senders: Vec<UnboundedSender<Barrier>>,
         result_sender: oneshot::Sender<()>,
     },
+    InspectState {
+        result_sender: oneshot::Sender<String>,
+    },
 }
 
 pub(super) struct CreateActorOutput {
@@ -246,7 +253,7 @@ pub(crate) struct StreamActorManagerState {
     pub(super) handles: HashMap<ActorId, ActorHandle>,
 
     /// Stores all actor information, taken after actor built.
-    pub(super) actors: HashMap<ActorId, stream_plan::StreamActor>,
+    pub(super) actors: HashMap<ActorId, BuildActorInfo>,
 
     /// Stores all actor tokio runtime monitoring tasks.
     pub(super) actor_monitor_tasks: HashMap<ActorId, ActorHandle>,
@@ -255,7 +262,7 @@ pub(crate) struct StreamActorManagerState {
     pub(super) creating_actors: FuturesUnordered<
         AttachedFuture<
             JoinHandle<StreamResult<CreateActorOutput>>,
-            oneshot::Sender<StreamResult<()>>,
+            (BTreeSet<ActorId>, oneshot::Sender<StreamResult<()>>),
         >,
     >,
 }
@@ -276,7 +283,7 @@ impl StreamActorManagerState {
         oneshot::Sender<StreamResult<()>>,
         StreamResult<CreateActorOutput>,
     ) {
-        let (join_result, sender) = pending_on_none(self.creating_actors.next()).await;
+        let (join_result, (_, sender)) = pending_on_none(self.creating_actors.next()).await;
         (
             sender,
             try { join_result.context("failed to join creating actors futures")?? },
@@ -298,6 +305,16 @@ pub(crate) struct StreamActorManager {
     pub(super) runtime: BackgroundShutdownRuntime,
 }
 
+#[derive(Debug)]
+#[expect(dead_code)]
+pub(super) struct LocalBarrierWorkerDebugInfo<'a> {
+    actor_to_send: BTreeSet<ActorId>,
+    running_actors: BTreeSet<ActorId>,
+    creating_actors: Vec<BTreeSet<ActorId>>,
+    managed_barrier_state: ManagedBarrierStateDebugInfo<'a>,
+    has_control_stream_connected: bool,
+}
+
 /// [`LocalBarrierWorker`] manages barrier control flow, used by local stream manager.
 /// Specifically, [`LocalBarrierWorker`] serve barrier injection from meta server, send the
 /// barriers to and collect them from all actors, and finally report the progress.
@@ -356,6 +373,21 @@ impl LocalBarrierWorker {
         }
     }
 
+    fn to_debug_info(&self) -> LocalBarrierWorkerDebugInfo<'_> {
+        LocalBarrierWorkerDebugInfo {
+            actor_to_send: self.barrier_senders.keys().cloned().collect(),
+            running_actors: self.actor_manager_state.handles.keys().cloned().collect(),
+            creating_actors: self
+                .actor_manager_state
+                .creating_actors
+                .iter()
+                .map(|fut| fut.item().0.clone())
+                .collect(),
+            managed_barrier_state: self.state.to_debug_info(),
+            has_control_stream_connected: self.control_stream_handle.connected(),
+        }
+    }
+
     async fn run(mut self, mut actor_op_rx: UnboundedReceiver<LocalActorOperation>) {
         loop {
             select! {
@@ -505,6 +537,9 @@ impl LocalBarrierWorker {
                 self.register_sender(actor_id, senders);
                 let _ = result_sender.send(());
             }
+            LocalActorOperation::InspectState { result_sender } => {
+                let _ = result_sender.send(format!("{:#?}", self.to_debug_info()));
+            }
         }
     }
 }
@@ -523,8 +558,14 @@ impl LocalBarrierWorker {
             sync_result,
         } = result;
 
-        let (synced_sstables, table_watermarks) = sync_result
-            .map(|sync_result| (sync_result.uncommitted_ssts, sync_result.table_watermarks))
+        let (synced_sstables, table_watermarks, old_value_ssts) = sync_result
+            .map(|sync_result| {
+                (
+                    sync_result.uncommitted_ssts,
+                    sync_result.table_watermarks,
+                    sync_result.old_value_ssts,
+                )
+            })
             .unwrap_or_default();
 
         let result = StreamingControlStreamResponse {
@@ -553,6 +594,10 @@ impl LocalBarrierWorker {
                             .into_iter()
                             .map(|(key, value)| (key.table_id, value.to_protobuf()))
                             .collect(),
+                        old_value_sstables: old_value_ssts
+                            .into_iter()
+                            .map(|sst| sst.sst_info)
+                            .collect(),
                     },
                 ),
             ),
@@ -843,12 +888,28 @@ impl LocalBarrierManager {
 pub fn try_find_root_actor_failure<'a>(
     actor_errors: impl IntoIterator<Item = &'a StreamError>,
 ) -> Option<StreamError> {
+    // Explicitly list all error kinds here to notice developers to update this function when
+    // there are changes in error kinds.
+
     fn stream_executor_error_score(e: &StreamExecutorError) -> i32 {
         use crate::executor::error::ErrorKind;
         match e.inner() {
-            ErrorKind::ChannelClosed(_) | ErrorKind::ExchangeChannelClosed(_) => 0,
-            ErrorKind::Internal(_) => 1,
-            _ => 999,
+            // `ChannelClosed` or `ExchangeChannelClosed` is likely to be caused by actor exit
+            // and not the root cause.
+            ErrorKind::ChannelClosed(_) | ErrorKind::ExchangeChannelClosed(_) => 1,
+
+            // Normal errors.
+            ErrorKind::Uncategorized(_)
+            | ErrorKind::Storage(_)
+            | ErrorKind::ArrayError(_)
+            | ErrorKind::ExprError(_)
+            | ErrorKind::SerdeError(_)
+            | ErrorKind::SinkError(_)
+            | ErrorKind::RpcError(_)
+            | ErrorKind::AlignBarrier(_, _)
+            | ErrorKind::ConnectorError(_)
+            | ErrorKind::DmlError(_)
+            | ErrorKind::NotImplemented(_) => 999,
         }
     }
 
@@ -858,9 +919,18 @@ pub fn try_find_root_actor_failure<'a>(
             // `UnexpectedExit` wraps the original error. Score on the inner error.
             ErrorKind::UnexpectedExit { source, .. } => stream_error_score(source),
 
-            ErrorKind::Internal(_) => 1000,
+            // `BarrierSend` is likely to be caused by actor exit and not the root cause.
+            ErrorKind::BarrierSend { .. } => 1,
+
+            // Executor errors first.
             ErrorKind::Executor(ee) => 2000 + stream_executor_error_score(ee),
-            _ => 3000,
+
+            // Then other errors.
+            ErrorKind::Uncategorized(_)
+            | ErrorKind::Storage(_)
+            | ErrorKind::Expression(_)
+            | ErrorKind::Array(_)
+            | ErrorKind::Sink(_) => 1000,
         }
     }
 
diff --git a/src/stream/src/task/barrier_manager/managed_state.rs b/src/stream/src/task/barrier_manager/managed_state.rs
index 4bb39ddabff1e..00ff5606bde2e 100644
--- a/src/stream/src/task/barrier_manager/managed_state.rs
+++ b/src/stream/src/task/barrier_manager/managed_state.rs
@@ -14,7 +14,8 @@
 
 use std::assert_matches::assert_matches;
 use std::collections::btree_map::Entry;
-use std::collections::{BTreeMap, HashMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+use std::fmt::{Debug, Formatter};
 use std::future::Future;
 use std::mem::replace;
 use std::sync::Arc;
@@ -22,7 +23,7 @@ use std::sync::Arc;
 use anyhow::anyhow;
 use await_tree::InstrumentAwait;
 use futures::stream::FuturesOrdered;
-use futures::{FutureExt, StreamExt};
+use futures::{FutureExt, StreamExt, TryFutureExt};
 use prometheus::HistogramTimer;
 use risingwave_common::must_match;
 use risingwave_hummock_sdk::SyncResult;
@@ -40,6 +41,23 @@ use crate::executor::monitor::StreamingMetrics;
 use crate::executor::{Barrier, Mutation};
 use crate::task::{await_tree_key, ActorId};
 
+struct IssuedState {
+    pub mutation: Option<Arc<Mutation>>,
+    /// Actor ids remaining to be collected.
+    pub remaining_actors: BTreeSet<ActorId>,
+
+    pub barrier_inflight_latency: HistogramTimer,
+}
+
+impl Debug for IssuedState {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("IssuedState")
+            .field("mutation", &self.mutation)
+            .field("remaining_actors", &self.remaining_actors)
+            .finish()
+    }
+}
+
 /// The state machine of local barrier manager.
 #[derive(Debug)]
 enum ManagedBarrierStateInner {
@@ -51,13 +69,7 @@ enum ManagedBarrierStateInner {
     },
 
     /// Meta service has issued a `send_barrier` request. We're collecting barriers now.
-    Issued {
-        mutation: Option<Arc<Mutation>>,
-        /// Actor ids remaining to be collected.
-        remaining_actors: HashSet<ActorId>,
-
-        barrier_inflight_latency: HistogramTimer,
-    },
+    Issued(IssuedState),
 
     /// The barrier has been collected by all remaining actors
     AllCollected,
@@ -84,46 +96,60 @@ fn sync_epoch(
     kind: BarrierKind,
 ) -> impl Future<Output = StreamResult<Option<SyncResult>>> + 'static {
     let barrier_sync_latency = streaming_metrics.barrier_sync_latency.clone();
-    let state_store = state_store.clone();
 
-    async move {
-        let sync_result = match kind {
-            BarrierKind::Unspecified => unreachable!(),
-            BarrierKind::Initial => {
-                if let Some(hummock) = state_store.as_hummock() {
-                    let mce = hummock.get_pinned_version().max_committed_epoch();
-                    assert_eq!(
-                        mce, prev_epoch,
-                        "first epoch should match with the current version",
-                    );
-                }
-                tracing::info!(?prev_epoch, "ignored syncing data for the first barrier");
-                None
-            }
-            BarrierKind::Barrier => None,
-            BarrierKind::Checkpoint => {
-                let timer = barrier_sync_latency.start_timer();
-                let sync_result = dispatch_state_store!(state_store, store, {
-                    store
-                        .sync(prev_epoch)
-                        .instrument_await(format!("sync_epoch (epoch {})", prev_epoch))
-                        .await
-                        .inspect_err(|e| {
-                            tracing::error!(
-                                prev_epoch,
-                                error = %e.as_report(),
-                                "Failed to sync state store",
-                            );
-                        })
-                })?;
-                timer.observe_duration();
-                Some(sync_result)
+    let sync_result_future = match kind {
+        BarrierKind::Unspecified => unreachable!(),
+        BarrierKind::Initial => {
+            if let Some(hummock) = state_store.as_hummock() {
+                let mce = hummock.get_pinned_version().max_committed_epoch();
+                assert_eq!(
+                    mce, prev_epoch,
+                    "first epoch should match with the current version",
+                );
             }
-        };
-        Ok(sync_result)
+            tracing::info!(?prev_epoch, "ignored syncing data for the first barrier");
+            None
+        }
+        BarrierKind::Barrier => None,
+        BarrierKind::Checkpoint => {
+            let timer = barrier_sync_latency.start_timer();
+            let sync_result_future = dispatch_state_store!(state_store, store, {
+                store
+                    .sync(prev_epoch)
+                    .instrument_await(format!("sync_epoch (epoch {})", prev_epoch))
+                    .inspect_ok(move |_| {
+                        timer.observe_duration();
+                    })
+                    .inspect_err(move |e| {
+                        tracing::error!(
+                            prev_epoch,
+                            error = %e.as_report(),
+                            "Failed to sync state store",
+                        );
+                    })
+                    .boxed()
+            });
+            Some(sync_result_future)
+        }
+    };
+    async move {
+        if let Some(sync_result_future) = sync_result_future {
+            Ok(Some(sync_result_future.await?))
+        } else {
+            Ok(None)
+        }
     }
 }
 
+#[derive(Debug)]
+pub(super) struct ManagedBarrierStateDebugInfo<'a> {
+    #[expect(dead_code)]
+    epoch_barrier_state_map: &'a BTreeMap<u64, BarrierState>,
+
+    #[expect(dead_code)]
+    create_mview_progress: &'a HashMap<u64, HashMap<ActorId, BackfillState>>,
+}
+
 pub(super) struct ManagedBarrierState {
     /// Record barrier state for each epoch of concurrent checkpoints.
     ///
@@ -170,6 +196,13 @@ impl ManagedBarrierState {
         }
     }
 
+    pub(super) fn to_debug_info(&self) -> ManagedBarrierStateDebugInfo<'_> {
+        ManagedBarrierStateDebugInfo {
+            epoch_barrier_state_map: &self.epoch_barrier_state_map,
+            create_mview_progress: &self.create_mview_progress,
+        }
+    }
+
     pub fn read_barrier_mutation(
         &mut self,
         barrier: &Barrier,
@@ -193,7 +226,7 @@ impl ManagedBarrierState {
                     } => {
                         mutation_senders.push(sender);
                     }
-                    ManagedBarrierStateInner::Issued { mutation, .. } => {
+                    ManagedBarrierStateInner::Issued(IssuedState { mutation, .. }) => {
                         let _ = sender.send(mutation.clone());
                     }
                     _ => {
@@ -219,14 +252,13 @@ impl ManagedBarrierState {
         for (prev_epoch, barrier_state) in &mut self.epoch_barrier_state_map {
             let prev_epoch = *prev_epoch;
             match &barrier_state.inner {
-                ManagedBarrierStateInner::Issued {
+                ManagedBarrierStateInner::Issued(IssuedState {
                     remaining_actors, ..
-                } if remaining_actors.is_empty() => {}
+                }) if remaining_actors.is_empty() => {}
                 ManagedBarrierStateInner::AllCollected | ManagedBarrierStateInner::Completed(_) => {
                     continue;
                 }
-                ManagedBarrierStateInner::Stashed { .. }
-                | ManagedBarrierStateInner::Issued { .. } => {
+                ManagedBarrierStateInner::Stashed { .. } | ManagedBarrierStateInner::Issued(_) => {
                     break;
                 }
             }
@@ -236,10 +268,10 @@ impl ManagedBarrierState {
                 ManagedBarrierStateInner::AllCollected,
             );
 
-            must_match!(prev_state, ManagedBarrierStateInner::Issued {
+            must_match!(prev_state, ManagedBarrierStateInner::Issued(IssuedState {
                 barrier_inflight_latency: timer,
                 ..
-            } => {
+            }) => {
                 timer.observe_duration();
             });
 
@@ -317,10 +349,10 @@ impl ManagedBarrierState {
             .filter_map(move |(prev_epoch, barrier_state)| {
                 #[allow(clippy::single_match)]
                 match barrier_state.inner {
-                    ManagedBarrierStateInner::Issued {
+                    ManagedBarrierStateInner::Issued(IssuedState {
                         ref remaining_actors,
                         ..
-                    } => {
+                    }) => {
                         if remaining_actors.contains(&actor_id) {
                             Some(*prev_epoch)
                         } else {
@@ -357,10 +389,10 @@ impl ManagedBarrierState {
             Some(&mut BarrierState {
                 curr_epoch,
                 inner:
-                    ManagedBarrierStateInner::Issued {
+                    ManagedBarrierStateInner::Issued(IssuedState {
                         ref mut remaining_actors,
                         ..
-                    },
+                    }),
                 ..
             }) => {
                 let exist = remaining_actors.remove(&actor_id);
@@ -416,11 +448,11 @@ impl ManagedBarrierState {
             barrier.epoch.prev,
             BarrierState {
                 curr_epoch: barrier.epoch.curr,
-                inner: ManagedBarrierStateInner::Issued {
-                    remaining_actors: actor_ids_to_collect,
+                inner: ManagedBarrierStateInner::Issued(IssuedState {
+                    remaining_actors: BTreeSet::from_iter(actor_ids_to_collect),
                     mutation: barrier.mutation.clone(),
                     barrier_inflight_latency: timer,
-                },
+                }),
                 kind: barrier.kind,
             },
         );
diff --git a/src/stream/src/task/barrier_manager/tests.rs b/src/stream/src/task/barrier_manager/tests.rs
index 99efb00269b8a..5fbedcdfd0dcc 100644
--- a/src/stream/src/task/barrier_manager/tests.rs
+++ b/src/stream/src/task/barrier_manager/tests.rs
@@ -20,10 +20,8 @@ use std::task::Poll;
 use assert_matches::assert_matches;
 use futures::future::join_all;
 use futures::FutureExt;
-use itertools::Itertools;
 use risingwave_common::util::epoch::test_epoch;
 use risingwave_pb::stream_service::{streaming_control_stream_request, InjectBarrierRequest};
-use tokio::sync::mpsc::unbounded_channel;
 use tokio_stream::wrappers::UnboundedReceiverStream;
 
 use super::*;
diff --git a/src/stream/src/task/env.rs b/src/stream/src/task/env.rs
index 9a0b26f25f0c5..a47eb8279224c 100644
--- a/src/stream/src/task/env.rs
+++ b/src/stream/src/task/env.rs
@@ -19,7 +19,6 @@ use risingwave_common::config::StreamingConfig;
 use risingwave_common::system_param::local_manager::LocalSystemParamsManagerRef;
 use risingwave_common::util::addr::HostAddr;
 use risingwave_connector::source::monitor::SourceMetrics;
-use risingwave_connector::ConnectorParams;
 use risingwave_dml::dml_manager::DmlManagerRef;
 use risingwave_rpc_client::MetaClient;
 use risingwave_storage::StateStoreImpl;
@@ -33,9 +32,6 @@ pub struct StreamEnvironment {
     /// Endpoint the stream manager listens on.
     server_addr: HostAddr,
 
-    /// Parameters used by connector nodes.
-    connector_params: ConnectorParams,
-
     /// Streaming related configurations.
     config: Arc<StreamingConfig>,
 
@@ -65,7 +61,6 @@ impl StreamEnvironment {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
         server_addr: HostAddr,
-        connector_params: ConnectorParams,
         config: Arc<StreamingConfig>,
         worker_id: WorkerNodeId,
         state_store: StateStoreImpl,
@@ -76,7 +71,6 @@ impl StreamEnvironment {
     ) -> Self {
         StreamEnvironment {
             server_addr,
-            connector_params,
             config,
             worker_id,
             state_store,
@@ -93,11 +87,9 @@ impl StreamEnvironment {
     pub fn for_test() -> Self {
         use risingwave_common::system_param::local_manager::LocalSystemParamsManager;
         use risingwave_dml::dml_manager::DmlManager;
-        use risingwave_pb::connector_service::SinkPayloadFormat;
         use risingwave_storage::monitor::MonitoredStorageMetrics;
         StreamEnvironment {
             server_addr: "127.0.0.1:5688".parse().unwrap(),
-            connector_params: ConnectorParams::new(SinkPayloadFormat::Json),
             config: Arc::new(StreamingConfig::default()),
             worker_id: WorkerNodeId::default(),
             state_store: StateStoreImpl::shared_in_memory_store(Arc::new(
@@ -127,10 +119,6 @@ impl StreamEnvironment {
         self.state_store.clone()
     }
 
-    pub fn connector_params(&self) -> ConnectorParams {
-        self.connector_params.clone()
-    }
-
     pub fn dml_manager_ref(&self) -> DmlManagerRef {
         self.dml_manager.clone()
     }
diff --git a/src/stream/src/task/mod.rs b/src/stream/src/task/mod.rs
index 7a6fd40f9231a..77f21b52406f5 100644
--- a/src/stream/src/task/mod.rs
+++ b/src/stream/src/task/mod.rs
@@ -57,9 +57,9 @@ pub struct SharedContext {
     /// There are three cases when we need local channels to pass around messages:
     /// 1. pass `Message` between two local actors
     /// 2. The RPC client at the downstream actor forwards received `Message` to one channel in
-    /// `ReceiverExecutor` or `MergerExecutor`.
+    ///    `ReceiverExecutor` or `MergerExecutor`.
     /// 3. The RPC `Output` at the upstream actor forwards received `Message` to
-    /// `ExchangeServiceImpl`.
+    ///    `ExchangeServiceImpl`.
     ///
     /// The channel serves as a buffer because `ExchangeServiceImpl`
     /// is on the server-side and we will also introduce backpressure.
diff --git a/src/stream/src/task/stream_manager.rs b/src/stream/src/task/stream_manager.rs
index a275a6eaeb0f4..f9fa061993a79 100644
--- a/src/stream/src/task/stream_manager.rs
+++ b/src/stream/src/task/stream_manager.rs
@@ -13,10 +13,12 @@
 // limitations under the License.
 
 use core::time::Duration;
+use std::collections::HashSet;
 use std::fmt::Debug;
 use std::mem::take;
 use std::sync::atomic::AtomicU64;
 use std::sync::Arc;
+use std::time::Instant;
 
 use anyhow::anyhow;
 use async_recursion::async_recursion;
@@ -25,15 +27,15 @@ use futures::FutureExt;
 use itertools::Itertools;
 use risingwave_common::bail;
 use risingwave_common::buffer::Bitmap;
-use risingwave_common::catalog::{Field, Schema};
+use risingwave_common::catalog::{Field, Schema, TableId};
 use risingwave_common::config::MetricLevel;
 use risingwave_pb::common::ActorInfo;
 use risingwave_pb::stream_plan;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
-use risingwave_pb::stream_plan::{StreamActor, StreamNode};
+use risingwave_pb::stream_plan::StreamNode;
 use risingwave_pb::stream_service::streaming_control_stream_request::InitRequest;
 use risingwave_pb::stream_service::{
-    StreamingControlStreamRequest, StreamingControlStreamResponse,
+    BuildActorInfo, StreamingControlStreamRequest, StreamingControlStreamResponse,
 };
 use risingwave_storage::monitor::HummockTraceFutureExt;
 use risingwave_storage::{dispatch_state_store, StateStore};
@@ -171,6 +173,12 @@ impl LocalStreamManager {
         await_tree_config: Option<await_tree::Config>,
         watermark_epoch: AtomicU64Ref,
     ) -> Self {
+        if !env.config().unsafe_enable_strict_consistency {
+            // If strict consistency is disabled, should disable storage sanity check.
+            // Since this is a special config, we have to check it here.
+            risingwave_storage::hummock::utils::disable_sanity_check();
+        }
+
         let await_tree_reg = await_tree_config.clone().map(await_tree::Registry::new);
 
         let (actor_op_tx, actor_op_rx) = unbounded_channel();
@@ -219,7 +227,7 @@ impl LocalStreamManager {
             .await
     }
 
-    pub async fn update_actors(&self, actors: Vec<stream_plan::StreamActor>) -> StreamResult<()> {
+    pub async fn update_actors(&self, actors: Vec<BuildActorInfo>) -> StreamResult<()> {
         self.actor_op_tx
             .send_and_await(|result_sender| LocalActorOperation::UpdateActors {
                 actors,
@@ -254,6 +262,21 @@ impl LocalStreamManager {
             })
             .await?
     }
+
+    pub async fn inspect_barrier_state(&self) -> StreamResult<String> {
+        info!("start inspecting barrier state");
+        let start = Instant::now();
+        self.actor_op_tx
+            .send_and_await(|result_sender| LocalActorOperation::InspectState { result_sender })
+            .inspect(|result| {
+                info!(
+                    ok = result.is_ok(),
+                    time = ?start.elapsed(),
+                    "finish inspecting barrier state"
+                );
+            })
+            .await
+    }
 }
 
 impl LocalBarrierWorker {
@@ -298,10 +321,7 @@ impl LocalBarrierWorker {
         self.actor_manager.env.dml_manager_ref().clear();
     }
 
-    pub(super) fn update_actors(
-        &mut self,
-        actors: Vec<stream_plan::StreamActor>,
-    ) -> StreamResult<()> {
+    pub(super) fn update_actors(&mut self, actors: Vec<BuildActorInfo>) -> StreamResult<()> {
         self.actor_manager_state.update_actors(actors)
     }
 
@@ -312,7 +332,7 @@ impl LocalBarrierWorker {
         actors: &[ActorId],
         result_sender: oneshot::Sender<StreamResult<()>>,
     ) {
-        let actors = {
+        let actors: Vec<_> = {
             let actor_result = actors
                 .iter()
                 .map(|actor_id| {
@@ -330,6 +350,10 @@ impl LocalBarrierWorker {
                 }
             }
         };
+        let actor_ids = actors
+            .iter()
+            .map(|actor| actor.actor.as_ref().unwrap().actor_id)
+            .collect();
         let actor_manager = self.actor_manager.clone();
         let create_actors_fut = crate::CONFIG.scope(
             self.actor_manager.env.config().clone(),
@@ -338,7 +362,7 @@ impl LocalBarrierWorker {
         let join_handle = self.actor_manager.runtime.spawn(create_actors_fut);
         self.actor_manager_state
             .creating_actors
-            .push(AttachedFuture::new(join_handle, result_sender));
+            .push(AttachedFuture::new(join_handle, (actor_ids, result_sender)));
     }
 }
 
@@ -529,18 +553,32 @@ impl StreamActorManager {
 
     async fn create_actors(
         self: Arc<Self>,
-        actors: Vec<StreamActor>,
+        actors: Vec<BuildActorInfo>,
         shared_context: Arc<SharedContext>,
     ) -> StreamResult<CreateActorOutput> {
         let mut ret = Vec::with_capacity(actors.len());
         let create_actor_context = CreateActorContext::default();
         for actor in actors {
+            let BuildActorInfo {
+                actor,
+                related_subscriptions,
+            } = actor;
+            let actor = actor.unwrap();
             let actor_id = actor.actor_id;
             let actor_context = ActorContext::create(
                 &actor,
                 self.env.total_mem_usage(),
                 self.streaming_metrics.clone(),
                 actor.dispatcher.len(),
+                related_subscriptions
+                    .into_iter()
+                    .map(|(table_id, subscription_ids)| {
+                        (
+                            TableId::new(table_id),
+                            HashSet::from_iter(subscription_ids.subscription_ids),
+                        )
+                    })
+                    .collect(),
             );
             let vnode_bitmap = actor.vnode_bitmap.as_ref().map(|b| b.into());
             let expr_context = actor.expr_context.clone().unwrap();
@@ -620,54 +658,43 @@ impl LocalBarrierWorker {
 
             if self.actor_manager.streaming_metrics.level >= MetricLevel::Debug {
                 tracing::info!("Tokio metrics are enabled because metrics_level >= Debug");
-                let actor_id_str = actor_id.to_string();
-                let metrics = self.actor_manager.streaming_metrics.clone();
+                let streaming_metrics = self.actor_manager.streaming_metrics.clone();
                 let actor_monitor_task = self.actor_manager.runtime.spawn(async move {
+                    let metrics = streaming_metrics.new_actor_metrics(actor_id);
                     loop {
                         let task_metrics = monitor.cumulative();
                         metrics
                             .actor_execution_time
-                            .with_label_values(&[&actor_id_str])
                             .set(task_metrics.total_poll_duration.as_secs_f64());
                         metrics
                             .actor_fast_poll_duration
-                            .with_label_values(&[&actor_id_str])
                             .set(task_metrics.total_fast_poll_duration.as_secs_f64());
                         metrics
                             .actor_fast_poll_cnt
-                            .with_label_values(&[&actor_id_str])
                             .set(task_metrics.total_fast_poll_count as i64);
                         metrics
                             .actor_slow_poll_duration
-                            .with_label_values(&[&actor_id_str])
                             .set(task_metrics.total_slow_poll_duration.as_secs_f64());
                         metrics
                             .actor_slow_poll_cnt
-                            .with_label_values(&[&actor_id_str])
                             .set(task_metrics.total_slow_poll_count as i64);
                         metrics
                             .actor_poll_duration
-                            .with_label_values(&[&actor_id_str])
                             .set(task_metrics.total_poll_duration.as_secs_f64());
                         metrics
                             .actor_poll_cnt
-                            .with_label_values(&[&actor_id_str])
                             .set(task_metrics.total_poll_count as i64);
                         metrics
                             .actor_idle_duration
-                            .with_label_values(&[&actor_id_str])
                             .set(task_metrics.total_idle_duration.as_secs_f64());
                         metrics
                             .actor_idle_cnt
-                            .with_label_values(&[&actor_id_str])
                             .set(task_metrics.total_idled_count as i64);
                         metrics
                             .actor_scheduled_duration
-                            .with_label_values(&[&actor_id_str])
                             .set(task_metrics.total_scheduled_duration.as_secs_f64());
                         metrics
                             .actor_scheduled_cnt
-                            .with_label_values(&[&actor_id_str])
                             .set(task_metrics.total_scheduled_count as i64);
                         tokio::time::sleep(Duration::from_secs(1)).await;
                     }
@@ -726,9 +753,9 @@ impl StreamActorManagerState {
         self.actor_monitor_tasks.clear();
     }
 
-    fn update_actors(&mut self, actors: Vec<stream_plan::StreamActor>) -> StreamResult<()> {
+    fn update_actors(&mut self, actors: Vec<BuildActorInfo>) -> StreamResult<()> {
         for actor in actors {
-            let actor_id = actor.get_actor_id();
+            let actor_id = actor.actor.as_ref().unwrap().get_actor_id();
             self.actors
                 .try_insert(actor_id, actor)
                 .map_err(|_| anyhow!("duplicated actor {}", actor_id))?;
diff --git a/src/stream/tests/integration_tests/eowc_over_window.rs b/src/stream/tests/integration_tests/eowc_over_window.rs
index cc674e556ab5e..72e3dade4c993 100644
--- a/src/stream/tests/integration_tests/eowc_over_window.rs
+++ b/src/stream/tests/integration_tests/eowc_over_window.rs
@@ -86,14 +86,14 @@ async fn test_over_window() {
         // lag(x, 1)
         WindowFuncCall {
             kind: WindowFuncKind::Aggregate(AggKind::FirstValue),
-            args: AggArgs::Unary(DataType::Int32, 3),
+            args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int32,
             frame: Frame::rows(FrameBound::Preceding(1), FrameBound::Preceding(1)),
         },
         // lead(x, 1)
         WindowFuncCall {
             kind: WindowFuncKind::Aggregate(AggKind::FirstValue),
-            args: AggArgs::Unary(DataType::Int32, 3),
+            args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int32,
             frame: Frame::rows(FrameBound::Following(1), FrameBound::Following(1)),
         },
@@ -186,7 +186,7 @@ async fn test_over_window_aggregate() {
     let store = MemoryStateStore::new();
     let calls = vec![WindowFuncCall {
         kind: WindowFuncKind::Aggregate(AggKind::Sum),
-        args: AggArgs::Unary(DataType::Int32, 3),
+        args: AggArgs::from_iter([(DataType::Int32, 3)]),
         return_type: DataType::Int64,
         frame: Frame::rows(FrameBound::Preceding(1), FrameBound::Following(1)),
     }];
diff --git a/src/stream/tests/integration_tests/over_window.rs b/src/stream/tests/integration_tests/over_window.rs
index 0be8e1848e9cd..f1ebba752b167 100644
--- a/src/stream/tests/integration_tests/over_window.rs
+++ b/src/stream/tests/integration_tests/over_window.rs
@@ -105,14 +105,14 @@ async fn test_over_window_lag_lead_append_only() {
         // lag(x, 1)
         WindowFuncCall {
             kind: WindowFuncKind::Aggregate(AggKind::FirstValue),
-            args: AggArgs::Unary(DataType::Int32, 3),
+            args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int32,
             frame: Frame::rows(FrameBound::Preceding(1), FrameBound::Preceding(1)),
         },
         // lead(x, 1)
         WindowFuncCall {
             kind: WindowFuncKind::Aggregate(AggKind::FirstValue),
-            args: AggArgs::Unary(DataType::Int32, 3),
+            args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int32,
             frame: Frame::rows(FrameBound::Following(1), FrameBound::Following(1)),
         },
@@ -216,14 +216,14 @@ async fn test_over_window_lag_lead_with_updates() {
         // lag(x, 1)
         WindowFuncCall {
             kind: WindowFuncKind::Aggregate(AggKind::FirstValue),
-            args: AggArgs::Unary(DataType::Int32, 3),
+            args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int32,
             frame: Frame::rows(FrameBound::Preceding(1), FrameBound::Preceding(1)),
         },
         // lead(x, 1)
         WindowFuncCall {
             kind: WindowFuncKind::Aggregate(AggKind::FirstValue),
-            args: AggArgs::Unary(DataType::Int32, 3),
+            args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int32,
             frame: Frame::rows(FrameBound::Following(1), FrameBound::Following(1)),
         },
@@ -391,7 +391,7 @@ async fn test_over_window_sum() {
         // )
         WindowFuncCall {
             kind: WindowFuncKind::Aggregate(AggKind::Sum),
-            args: AggArgs::Unary(DataType::Int32, 3),
+            args: AggArgs::from_iter([(DataType::Int32, 3)]),
             return_type: DataType::Int64,
             frame: Frame::rows_with_exclusion(
                 FrameBound::Preceding(1),
diff --git a/src/stream/tests/integration_tests/project_set.rs b/src/stream/tests/integration_tests/project_set.rs
index 5cff03284f4b6..543f710b61b75 100644
--- a/src/stream/tests/integration_tests/project_set.rs
+++ b/src/stream/tests/integration_tests/project_set.rs
@@ -15,6 +15,7 @@
 use multimap::MultiMap;
 use risingwave_expr::table_function::repeat;
 use risingwave_stream::executor::ProjectSetExecutor;
+use risingwave_stream::task::ActorEvalErrorReport;
 
 use crate::prelude::*;
 
@@ -30,8 +31,8 @@ fn create_executor() -> (MessageSender, BoxedMessageStream) {
     let (tx, source) = MockSource::channel();
     let source = source.into_executor(schema, PkIndices::new());
 
-    let test_expr = build_from_pretty("(add:int8 $0:int8 $1:int8)").into_inner();
-    let test_expr_watermark = build_from_pretty("(add:int8 $0:int8 1:int8)").into_inner();
+    let test_expr = build_from_pretty("(add:int8 $0:int8 $1:int8)");
+    let test_expr_watermark = build_from_pretty("(add:int8 $0:int8 1:int8)");
     let tf1 = repeat(build_from_pretty("1:int4").into_inner(), 1);
     let tf2 = repeat(build_from_pretty("2:int4").into_inner(), 2);
 
@@ -47,6 +48,10 @@ fn create_executor() -> (MessageSender, BoxedMessageStream) {
         CHUNK_SIZE,
         MultiMap::from_iter(std::iter::once((0, 1))),
         vec![],
+        ActorEvalErrorReport {
+            actor_context: ActorContext::for_test(123),
+            identity: "ProjectSetExecutor".into(),
+        },
     );
     (tx, project_set.boxed().execute())
 }
diff --git a/src/tests/compaction_test/src/compaction_test_runner.rs b/src/tests/compaction_test/src/compaction_test_runner.rs
index 48e8311816d0a..e492b64979070 100644
--- a/src/tests/compaction_test/src/compaction_test_runner.rs
+++ b/src/tests/compaction_test/src/compaction_test_runner.rs
@@ -23,7 +23,7 @@ use std::time::Duration;
 use anyhow::anyhow;
 use bytes::{BufMut, Bytes, BytesMut};
 use clap::Parser;
-use foyer::memory::CacheContext;
+use foyer::CacheContext;
 use risingwave_common::catalog::TableId;
 use risingwave_common::config::{
     extract_storage_memory_config, load_config, MetaConfig, NoOverride,
@@ -69,9 +69,9 @@ impl CompactionTestMetrics {
 /// 1. Start the cluster with ci-compaction-test config: `./risedev d ci-compaction-test`
 /// 2. Ingest enough L0 SSTs, for example we can use the tpch-bench tool
 /// 3. Disable hummock manager commit new epochs: `./risedev ctl hummock disable-commit-epoch`, and
-/// it will print the current max committed epoch in Meta.
+///    it will print the current max committed epoch in Meta.
 /// 4. Use the test tool to replay hummock version deltas and trigger compactions:
-/// `./risedev compaction-test --state-store hummock+s3://your-bucket -t <table_id>`
+///    `./risedev compaction-test --state-store hummock+s3://your-bucket -t <table_id>`
 pub async fn compaction_test_main(
     _listen_addr: SocketAddr,
     advertise_addr: HostAddr,
diff --git a/src/tests/compaction_test/src/delete_range_runner.rs b/src/tests/compaction_test/src/delete_range_runner.rs
index 341e89a58af89..7183ee7e2e851 100644
--- a/src/tests/compaction_test/src/delete_range_runner.rs
+++ b/src/tests/compaction_test/src/delete_range_runner.rs
@@ -15,18 +15,16 @@
 use std::future::Future;
 use std::ops::{Bound, RangeBounds};
 use std::pin::{pin, Pin};
-use std::sync::atomic::AtomicU32;
 use std::sync::Arc;
 use std::time::{Duration, SystemTime};
 
 use bytes::Bytes;
-use foyer::memory::CacheContext;
+use foyer::{CacheContext, HybridCacheBuilder};
 use rand::rngs::StdRng;
 use rand::{RngCore, SeedableRng};
 use risingwave_common::catalog::TableId;
 use risingwave_common::config::{
-    extract_storage_memory_config, load_config, EvictionConfig, NoOverride, ObjectStoreConfig,
-    RwConfig,
+    extract_storage_memory_config, load_config, NoOverride, ObjectStoreConfig, RwConfig,
 };
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::util::epoch::{test_epoch, EpochExt};
@@ -53,7 +51,7 @@ use risingwave_storage::hummock::compactor::{
 use risingwave_storage::hummock::sstable_store::SstableStoreRef;
 use risingwave_storage::hummock::utils::cmp_delete_range_left_bounds;
 use risingwave_storage::hummock::{
-    CachePolicy, FileCache, HummockStorage, MemoryLimiter, SstableObjectIdManager, SstableStore,
+    CachePolicy, HummockStorage, MemoryLimiter, SstableObjectIdManager, SstableStore,
     SstableStoreConfig,
 };
 use risingwave_storage::monitor::{CompactorMetrics, HummockStateStoreMetrics};
@@ -207,24 +205,30 @@ async fn compaction_test(
         state_store_type.strip_prefix("hummock+").unwrap(),
         object_store_metrics.clone(),
         "Hummock",
-        ObjectStoreConfig::default(),
+        Arc::new(ObjectStoreConfig::default()),
     )
     .await;
+    let meta_cache = HybridCacheBuilder::new()
+        .memory(storage_memory_config.meta_cache_capacity_mb * (1 << 20))
+        .with_shards(storage_memory_config.meta_cache_shard_num)
+        .storage()
+        .build()
+        .await?;
+    let block_cache = HybridCacheBuilder::new()
+        .memory(storage_memory_config.block_cache_capacity_mb * (1 << 20))
+        .with_shards(storage_memory_config.block_cache_shard_num)
+        .storage()
+        .build()
+        .await?;
     let sstable_store = Arc::new(SstableStore::new(SstableStoreConfig {
         store: Arc::new(remote_object_store),
         path: system_params.data_directory().to_string(),
-        block_cache_capacity: storage_memory_config.block_cache_capacity_mb * (1 << 20),
-        meta_cache_capacity: storage_memory_config.meta_cache_capacity_mb * (1 << 20),
-        block_cache_shard_num: storage_memory_config.block_cache_shard_num,
-        meta_cache_shard_num: storage_memory_config.meta_cache_shard_num,
-        block_cache_eviction: EvictionConfig::for_test(),
-        meta_cache_eviction: EvictionConfig::for_test(),
         prefetch_buffer_capacity: storage_memory_config.prefetch_buffer_capacity_mb * (1 << 20),
         max_prefetch_block_number: storage_opts.max_prefetch_block_number,
-        data_file_cache: FileCache::none(),
-        meta_file_cache: FileCache::none(),
         recent_filter: None,
         state_store_metrics: state_store_metrics.clone(),
+        meta_cache,
+        block_cache,
     }));
 
     let store = HummockStorage::new(
@@ -600,24 +604,15 @@ fn run_compactor_thread(
 ) {
     let filter_key_extractor_manager =
         FilterKeyExtractorManager::RpcFilterKeyExtractorManager(filter_key_extractor_manager);
-
-    let compaction_executor = Arc::new(CompactionExecutor::new(Some(1)));
-    let max_task_parallelism = Arc::new(AtomicU32::new(
-        (compaction_executor.worker_num() as f32 * storage_opts.compactor_max_task_multiplier)
-            .ceil() as u32,
-    ));
     let compactor_context = CompactorContext {
         storage_opts,
         sstable_store,
         compactor_metrics,
         is_share_buffer_compact: false,
         compaction_executor: Arc::new(CompactionExecutor::new(None)),
-
         memory_limiter: MemoryLimiter::unlimit(),
         task_progress_manager: Default::default(),
         await_tree_reg: None,
-        running_task_parallelism: Arc::new(AtomicU32::new(0)),
-        max_task_parallelism,
     };
 
     start_compactor(
diff --git a/src/tests/compaction_test/src/lib.rs b/src/tests/compaction_test/src/lib.rs
index 1e2207fc99a9b..e5fd10b10b176 100644
--- a/src/tests/compaction_test/src/lib.rs
+++ b/src/tests/compaction_test/src/lib.rs
@@ -22,7 +22,6 @@
 #![warn(clippy::map_flatten)]
 #![warn(clippy::await_holding_lock)]
 #![deny(rustdoc::broken_intra_doc_links)]
-#![feature(bound_map)]
 #![feature(register_tool)]
 #![register_tool(rw)]
 #![allow(rw::format_error)] // test code
@@ -45,7 +44,7 @@ pub struct CompactionTestOpts {
     #[clap(long)]
     pub client_address: Option<String>,
 
-    /// The state store string e.g. hummock+s3://test-bucket
+    /// The state store string e.g. `hummock+s3://test-bucket`
     #[clap(short, long)]
     pub state_store: String,
 
diff --git a/src/tests/e2e_extended_mode/Cargo.toml b/src/tests/e2e_extended_mode/Cargo.toml
index ecbf751fe1004..cbc7831ea7a03 100644
--- a/src/tests/e2e_extended_mode/Cargo.toml
+++ b/src/tests/e2e_extended_mode/Cargo.toml
@@ -19,7 +19,7 @@ chrono = { version = "0.4", features = ['serde'] }
 clap = { workspace = true }
 pg_interval = "0.4"
 rust_decimal = { version = "1.35", features = ["db-postgres"] }
-tokio = { version = "0.2.24", package = "madsim-tokio", features = [
+tokio = { version = "0.2", package = "madsim-tokio", features = [
     "rt",
     "macros",
     "rt-multi-thread",
diff --git a/src/tests/regress/Cargo.toml b/src/tests/regress/Cargo.toml
index 8c5478d13d52e..65248877adf60 100644
--- a/src/tests/regress/Cargo.toml
+++ b/src/tests/regress/Cargo.toml
@@ -26,6 +26,7 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [
     "time",
     "signal",
     "process",
+    "io-util",
 ] }
 tracing = "0.1"
 tracing-subscriber = "0.3.17"
diff --git a/src/tests/regress/data/sql/jsonb.sql b/src/tests/regress/data/sql/jsonb.sql
index b8d1e24654af2..71ce107ef270c 100644
--- a/src/tests/regress/data/sql/jsonb.sql
+++ b/src/tests/regress/data/sql/jsonb.sql
@@ -1114,18 +1114,18 @@ select '{"a":1 , "b":2, "c":3}'::jsonb - '{b}'::text[];
 select '{"a":1 , "b":2, "c":3}'::jsonb - '{c,b}'::text[];
 select '{"a":1 , "b":2, "c":3}'::jsonb - '{}'::text[];
 
---@ select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{n}', '[1,2,3]');
---@ select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{b,-1}', '[1,2,3]');
---@ select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{d,1,0}', '[1,2,3]');
---@ select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{d,NULL,0}', '[1,2,3]');
---@ 
---@ select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{n}', '{"1": 2}');
---@ select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{b,-1}', '{"1": 2}');
---@ select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{d,1,0}', '{"1": 2}');
---@ select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{d,NULL,0}', '{"1": 2}');
---@ 
---@ select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{b,-1}', '"test"');
---@ select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{b,-1}', '{"f": "test"}');
+select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{n}', '[1,2,3]');
+select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{b,-1}', '[1,2,3]');
+select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{d,1,0}', '[1,2,3]');
+select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{d,NULL,0}', '[1,2,3]');
+
+select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{n}', '{"1": 2}');
+select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{b,-1}', '{"1": 2}');
+select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{d,1,0}', '{"1": 2}');
+select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{d,NULL,0}', '{"1": 2}');
+
+select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{b,-1}', '"test"');
+select jsonb_set('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}'::jsonb, '{b,-1}', '{"f": "test"}');
 
 select jsonb_delete_path('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}', '{n}');
 select jsonb_delete_path('{"n":null, "a":1, "b":[1,2], "c":{"1":2}, "d":{"1":[2,3]}}', '{b,-1}');
@@ -1148,33 +1148,33 @@ select '[]'::jsonb - 1;
 select '"a"'::jsonb #- '{a}'; -- error
 select '{}'::jsonb #- '{a}';
 select '[]'::jsonb #- '{a}';
---@ select jsonb_set('"a"','{a}','"b"'); --error
---@ select jsonb_set('{}','{a}','"b"', false);
---@ select jsonb_set('[]','{1}','"b"', false);
---@ select jsonb_set('[{"f1":1,"f2":null},2,null,3]', '{0}','[2,3,4]', false);
---@ 
---@ -- jsonb_set adding instead of replacing
---@ 
---@ -- prepend to array
---@ select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{b,-33}','{"foo":123}');
---@ -- append to array
---@ select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{b,33}','{"foo":123}');
---@ -- check nesting levels addition
---@ select jsonb_set('{"a":1,"b":[4,5,[0,1,2],6,7],"c":{"d":4}}','{b,2,33}','{"foo":123}');
---@ -- add new key
---@ select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{c,e}','{"foo":123}');
---@ -- adding doesn't do anything if elements before last aren't present
---@ select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{x,-33}','{"foo":123}');
---@ select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{x,y}','{"foo":123}');
---@ -- add to empty object
---@ select jsonb_set('{}','{x}','{"foo":123}');
---@ --add to empty array
---@ select jsonb_set('[]','{0}','{"foo":123}');
---@ select jsonb_set('[]','{99}','{"foo":123}');
---@ select jsonb_set('[]','{-99}','{"foo":123}');
---@ select jsonb_set('{"a": [1, 2, 3]}', '{a, non_integer}', '"new_value"');
---@ select jsonb_set('{"a": {"b": [1, 2, 3]}}', '{a, b, non_integer}', '"new_value"');
---@ select jsonb_set('{"a": {"b": [1, 2, 3]}}', '{a, b, NULL}', '"new_value"');
+select jsonb_set('"a"','{a}','"b"'); --error
+select jsonb_set('{}','{a}','"b"', false);
+select jsonb_set('[]','{1}','"b"', false);
+select jsonb_set('[{"f1":1,"f2":null},2,null,3]', '{0}','[2,3,4]', false);
+
+-- jsonb_set adding instead of replacing
+
+-- prepend to array
+select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{b,-33}','{"foo":123}');
+-- append to array
+select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{b,33}','{"foo":123}');
+-- check nesting levels addition
+select jsonb_set('{"a":1,"b":[4,5,[0,1,2],6,7],"c":{"d":4}}','{b,2,33}','{"foo":123}');
+-- add new key
+select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{c,e}','{"foo":123}');
+-- adding doesn't do anything if elements before last aren't present
+select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{x,-33}','{"foo":123}');
+select jsonb_set('{"a":1,"b":[0,1,2],"c":{"d":4}}','{x,y}','{"foo":123}');
+-- add to empty object
+select jsonb_set('{}','{x}','{"foo":123}');
+--add to empty array
+select jsonb_set('[]','{0}','{"foo":123}');
+select jsonb_set('[]','{99}','{"foo":123}');
+select jsonb_set('[]','{-99}','{"foo":123}');
+select jsonb_set('{"a": [1, 2, 3]}', '{a, non_integer}', '"new_value"');
+select jsonb_set('{"a": {"b": [1, 2, 3]}}', '{a, b, non_integer}', '"new_value"');
+select jsonb_set('{"a": {"b": [1, 2, 3]}}', '{a, b, NULL}', '"new_value"');
 
 -- jsonb_set_lax
 
diff --git a/src/tests/regress/data/sql/text.sql b/src/tests/regress/data/sql/text.sql
index d9e7af6030921..62372488a651b 100644
--- a/src/tests/regress/data/sql/text.sql
+++ b/src/tests/regress/data/sql/text.sql
@@ -43,9 +43,9 @@ select concat_ws('',10,20,null,30);
 select concat_ws(NULL,10,20,null,30) is null;
 select reverse('abcde');
 select i, left('ahoj', i), right('ahoj', i) from generate_series(-5, 5) t(i) order by i;
---@ select quote_literal('');
---@ select quote_literal('abc''');
---@ select quote_literal(e'\\');
+select quote_literal('');
+select quote_literal('abc''');
+select quote_literal(e'\\');
 -- check variadic labeled argument
 --@ select concat(variadic array[1,2,3]);
 --@ select concat_ws(',', variadic array[1,2,3]);
diff --git a/src/tests/simulation/Cargo.toml b/src/tests/simulation/Cargo.toml
index fc55e6356f4c5..5fbfc0e19f6fd 100644
--- a/src/tests/simulation/Cargo.toml
+++ b/src/tests/simulation/Cargo.toml
@@ -24,7 +24,7 @@ futures = { version = "0.3", default-features = false, features = ["alloc"] }
 glob = "0.3"
 itertools = { workspace = true }
 lru = { workspace = true }
-madsim = "0.2.22"
+madsim = "0.2.27"
 paste = "1"
 pin-project = "1.1"
 pretty_assertions = "1"
@@ -38,7 +38,7 @@ risingwave_compute = { workspace = true }
 risingwave_connector = { workspace = true }
 risingwave_ctl = { workspace = true }
 risingwave_e2e_extended_mode_test = { path = "../e2e_extended_mode" }
-risingwave_expr_impl = { workspace = true }
+risingwave_expr_impl = { workspace = true, features = ["js-udf"] }
 risingwave_frontend = { workspace = true }
 risingwave_hummock_sdk = { workspace = true }
 risingwave_meta_node = { workspace = true }
@@ -53,7 +53,7 @@ serde_json = "1.0.107"
 sqllogictest = "0.20"
 tempfile = "3"
 tikv-jemallocator = { workspace = true }
-tokio = { version = "0.2.24", package = "madsim-tokio" }
+tokio = { version = "0.2", package = "madsim-tokio" }
 tokio-postgres = "0.7"
 tokio-stream = "0.1"
 tracing = "0.1"
diff --git a/src/tests/simulation/src/slt.rs b/src/tests/simulation/src/slt.rs
index ec1aca82d36cc..89e05b974e275 100644
--- a/src/tests/simulation/src/slt.rs
+++ b/src/tests/simulation/src/slt.rs
@@ -488,6 +488,7 @@ fn hack_kafka_test(path: &Path) -> tempfile::NamedTempFile {
             .expect("failed to get schema path");
     let content = content
         .replace("127.0.0.1:29092", "192.168.11.1:29092")
+        .replace("localhost:29092", "192.168.11.1:29092")
         .replace(
             "/risingwave/avro-simple-schema.avsc",
             simple_avsc_full_path.to_str().unwrap(),
diff --git a/src/tests/simulation/tests/integration_tests/backfill_tests.rs b/src/tests/simulation/tests/integration_tests/backfill_tests.rs
index ba34f45e6af67..1d8f7bb727101 100644
--- a/src/tests/simulation/tests/integration_tests/backfill_tests.rs
+++ b/src/tests/simulation/tests/integration_tests/backfill_tests.rs
@@ -256,7 +256,8 @@ async fn test_arrangement_backfill_progress() -> Result<()> {
         .run("CREATE MATERIALIZED VIEW m1 AS SELECT * FROM t")
         .await?;
 
-    // Verify arrangement backfill progress after 10s, it should be 1% at least.
+    // Verify arrangement backfill progress after 10s, it should be around 1%,
+    // since 10s = 10 records processed.
     sleep(Duration::from_secs(10)).await;
     let progress = session
         .run("SELECT progress FROM rw_catalog.rw_ddl_progress")
@@ -264,7 +265,7 @@ async fn test_arrangement_backfill_progress() -> Result<()> {
     let progress = progress.replace('%', "");
     let progress = progress.parse::<f64>().unwrap();
     assert!(
-        (1.0..10.0).contains(&progress),
+        (0.5..1.5).contains(&progress),
         "progress not within bounds {}",
         progress
     );
diff --git a/src/tests/simulation/tests/integration_tests/batch/mod.rs b/src/tests/simulation/tests/integration_tests/batch/mod.rs
index ef530efa9f46a..25c690cfdcbf8 100644
--- a/src/tests/simulation/tests/integration_tests/batch/mod.rs
+++ b/src/tests/simulation/tests/integration_tests/batch/mod.rs
@@ -54,7 +54,7 @@ fn cluster_config_no_compute_nodes() -> Configuration {
         file.write_all(
             "\
 [meta]
-max_heartbeat_interval_secs = 60
+max_heartbeat_interval_secs = 300
 
 [system]
 barrier_interval_ms = 1000
@@ -63,6 +63,9 @@ checkpoint_frequency = 1
 [server]
 telemetry_enabled = false
 metrics_level = \"Disabled\"
+
+[batch]
+mask_worker_temporary_secs = 30
         "
             .as_bytes(),
         )
@@ -163,7 +166,7 @@ async fn test_serving_cluster_availability() {
     tokio::time::sleep(Duration::from_secs(15)).await;
     query_and_assert(session.clone()).await;
     // wait for mask expire
-    tokio::time::sleep(Duration::from_secs(30)).await;
+    tokio::time::sleep(Duration::from_secs(60)).await;
     session.run(select).await.unwrap_err();
     // wait for previous nodes expire
     tokio::time::sleep(Duration::from_secs(300)).await;
diff --git a/src/tests/simulation/tests/integration_tests/scale/plan.rs b/src/tests/simulation/tests/integration_tests/scale/plan.rs
index e1d8148632c62..d39e159fc61d9 100644
--- a/src/tests/simulation/tests/integration_tests/scale/plan.rs
+++ b/src/tests/simulation/tests/integration_tests/scale/plan.rs
@@ -13,7 +13,6 @@
 // limitations under the License.
 
 use std::collections::HashMap;
-use std::default::Default;
 
 use anyhow::Result;
 use itertools::Itertools;
diff --git a/src/tests/sqlsmith/tests/frontend/mod.rs b/src/tests/sqlsmith/tests/frontend/mod.rs
index 5c58f028b933d..6eedb4e41876b 100644
--- a/src/tests/sqlsmith/tests/frontend/mod.rs
+++ b/src/tests/sqlsmith/tests/frontend/mod.rs
@@ -186,26 +186,24 @@ fn run_batch_query(
         .bind(stmt)
         .map_err(|e| Failed::from(format!("Failed to bind:\nReason:\n{}", e.as_report())))?;
     let mut planner = Planner::new(context);
-    let mut logical_plan = planner.plan(bound).map_err(|e| {
+    let mut plan_root = planner.plan(bound).map_err(|e| {
         Failed::from(format!(
             "Failed to generate logical plan:\nReason:\n{}",
             e.as_report()
         ))
     })?;
-    let batch_plan = logical_plan.gen_batch_plan().map_err(|e| {
+    plan_root.gen_batch_plan().map_err(|e| {
         Failed::from(format!(
             "Failed to generate batch plan:\nReason:\n{}",
             e.as_report()
         ))
     })?;
-    logical_plan
-        .gen_batch_distributed_plan(batch_plan)
-        .map_err(|e| {
-            Failed::from(format!(
-                "Failed to generate batch distributed plan:\nReason:\n{}",
-                e.as_report()
-            ))
-        })?;
+    plan_root.gen_batch_distributed_plan().map_err(|e| {
+        Failed::from(format!(
+            "Failed to generate batch distributed plan:\nReason:\n{}",
+            e.as_report()
+        ))
+    })?;
     Ok(())
 }
 
diff --git a/src/utils/delta_btree_map/Cargo.toml b/src/utils/delta_btree_map/Cargo.toml
index 48534ffad1867..274a028489395 100644
--- a/src/utils/delta_btree_map/Cargo.toml
+++ b/src/utils/delta_btree_map/Cargo.toml
@@ -15,7 +15,7 @@ ignored = ["workspace-hack"]
 normal = ["workspace-hack"]
 
 [dependencies]
-educe = "0.5"
+educe = "0.6"
 enum-as-inner = "0.6"
 
 [lints]
diff --git a/src/utils/delta_btree_map/src/lib.rs b/src/utils/delta_btree_map/src/lib.rs
index c300955af567b..1f7e1a77190a0 100644
--- a/src/utils/delta_btree_map/src/lib.rs
+++ b/src/utils/delta_btree_map/src/lib.rs
@@ -76,16 +76,18 @@ impl<'a, K: Ord, V> DeltaBTreeMap<'a, K, V> {
     pub fn find(&self, key: &K) -> Option<CursorWithDelta<'a, K, V>> {
         let ss_cursor = self.snapshot.lower_bound(Bound::Included(key));
         let dt_cursor = self.delta.lower_bound(Bound::Included(key));
-        let curr_key_value = if dt_cursor.key() == Some(key) {
-            match dt_cursor.key_value().unwrap() {
+        let ss_cursor_kv = ss_cursor.peek_next();
+        let dt_cursor_kv = dt_cursor.peek_next();
+        let curr_key_value = if dt_cursor_kv.map(|(k, _)| k) == Some(key) {
+            match dt_cursor_kv.unwrap() {
                 (key, Change::Insert(value)) => (key, value),
                 (_key, Change::Delete) => {
                     // the key is deleted
                     return None;
                 }
             }
-        } else if ss_cursor.key() == Some(key) {
-            ss_cursor.key_value().unwrap()
+        } else if ss_cursor_kv.map(|(k, _)| k) == Some(key) {
+            ss_cursor_kv.unwrap()
         } else {
             // the key doesn't exist
             return None;
@@ -102,16 +104,8 @@ impl<'a, K: Ord, V> DeltaBTreeMap<'a, K, V> {
         // the implementation is very similar to `CursorWithDelta::peek_next`
         let mut ss_cursor = self.snapshot.lower_bound(bound);
         let mut dt_cursor = self.delta.lower_bound(bound);
-        let next_ss_entry = || {
-            let tmp = ss_cursor.key_value();
-            ss_cursor.move_next();
-            tmp
-        };
-        let next_dt_entry = || {
-            let tmp = dt_cursor.key_value();
-            dt_cursor.move_next();
-            tmp
-        };
+        let next_ss_entry = || ss_cursor.next();
+        let next_dt_entry = || dt_cursor.next();
         let curr_key_value =
             CursorWithDelta::peek_impl(PeekDirection::Next, next_ss_entry, next_dt_entry);
         CursorWithDelta {
@@ -126,16 +120,8 @@ impl<'a, K: Ord, V> DeltaBTreeMap<'a, K, V> {
         // the implementation is very similar to `CursorWithDelta::peek_prev`
         let mut ss_cursor = self.snapshot.upper_bound(bound);
         let mut dt_cursor = self.delta.upper_bound(bound);
-        let prev_ss_entry = || {
-            let tmp = ss_cursor.key_value();
-            ss_cursor.move_prev();
-            tmp
-        };
-        let prev_dt_entry = || {
-            let tmp = dt_cursor.key_value();
-            dt_cursor.move_prev();
-            tmp
-        };
+        let prev_ss_entry = || ss_cursor.prev();
+        let prev_dt_entry = || dt_cursor.prev();
         let curr_key_value =
             CursorWithDelta::peek_impl(PeekDirection::Prev, prev_ss_entry, prev_dt_entry);
         CursorWithDelta {
@@ -244,22 +230,14 @@ impl<'a, K: Ord, V> CursorWithDelta<'a, K, V> {
             let mut ss_cursor = self.snapshot.lower_bound(Bound::Included(key));
             let mut dt_cursor = self.delta.lower_bound(Bound::Included(key));
             // either one of `ss_cursor.key()` and `dt_cursor.key()` == `Some(key)`, or both are
-            if ss_cursor.key() == Some(key) {
-                ss_cursor.move_next();
+            if ss_cursor.peek_next().map(|(k, _)| k) == Some(key) {
+                ss_cursor.next();
             }
-            if dt_cursor.key() == Some(key) {
-                dt_cursor.move_next();
+            if dt_cursor.peek_next().map(|(k, _)| k) == Some(key) {
+                dt_cursor.next();
             }
-            let next_ss_entry = || {
-                let tmp = ss_cursor.key_value();
-                ss_cursor.move_next();
-                tmp
-            };
-            let next_dt_entry = || {
-                let tmp = dt_cursor.key_value();
-                dt_cursor.move_next();
-                tmp
-            };
+            let next_ss_entry = || ss_cursor.next();
+            let next_dt_entry = || dt_cursor.next();
             Self::peek_impl(PeekDirection::Next, next_ss_entry, next_dt_entry)
         } else {
             // we are at the ghost position, now let's go back to the beginning
@@ -275,22 +253,14 @@ impl<'a, K: Ord, V> CursorWithDelta<'a, K, V> {
             let mut ss_cursor = self.snapshot.upper_bound(Bound::Included(key));
             let mut dt_cursor = self.delta.upper_bound(Bound::Included(key));
             // either one of `ss_cursor.key()` and `dt_cursor.key()` == `Some(key)`, or both are
-            if ss_cursor.key() == Some(key) {
-                ss_cursor.move_prev();
+            if ss_cursor.peek_prev().map(|(k, _)| k) == Some(key) {
+                ss_cursor.prev();
             }
-            if dt_cursor.key() == Some(key) {
-                dt_cursor.move_prev();
+            if dt_cursor.peek_prev().map(|(k, _)| k) == Some(key) {
+                dt_cursor.prev();
             }
-            let next_ss_entry = || {
-                let tmp = ss_cursor.key_value();
-                ss_cursor.move_prev();
-                tmp
-            };
-            let next_dt_entry = || {
-                let tmp = dt_cursor.key_value();
-                dt_cursor.move_prev();
-                tmp
-            };
+            let next_ss_entry = || ss_cursor.prev();
+            let next_dt_entry = || dt_cursor.prev();
             Self::peek_impl(PeekDirection::Prev, next_ss_entry, next_dt_entry)
         } else {
             // we are at the ghost position, now let's go back to the end
@@ -317,8 +287,6 @@ impl<'a, K: Ord, V> CursorWithDelta<'a, K, V> {
 
 #[cfg(test)]
 mod tests {
-    use std::collections::BTreeMap;
-
     use super::*;
 
     #[test]
diff --git a/src/utils/futures_util/src/misc.rs b/src/utils/futures_util/src/misc.rs
index f224b58214ddf..c6f5bcf07d23d 100644
--- a/src/utils/futures_util/src/misc.rs
+++ b/src/utils/futures_util/src/misc.rs
@@ -103,6 +103,12 @@ impl<F, T> AttachedFuture<F, T> {
             self.item.expect("should not be called after polled ready"),
         )
     }
+
+    pub fn item(&self) -> &T {
+        self.item
+            .as_ref()
+            .expect("should not be called after polled ready")
+    }
 }
 
 impl<F: Future + Unpin, T: Unpin> Future for AttachedFuture<F, T> {
diff --git a/src/utils/pgwire/src/pg_field_descriptor.rs b/src/utils/pgwire/src/pg_field_descriptor.rs
index 0b33c5743c107..82d75c78f7956 100644
--- a/src/utils/pgwire/src/pg_field_descriptor.rs
+++ b/src/utils/pgwire/src/pg_field_descriptor.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct PgFieldDescriptor {
     name: String,
     table_oid: i32,
diff --git a/src/utils/pgwire/src/pg_message.rs b/src/utils/pgwire/src/pg_message.rs
index 4898faf89b5d9..00c9a09b2eda6 100644
--- a/src/utils/pgwire/src/pg_message.rs
+++ b/src/utils/pgwire/src/pg_message.rs
@@ -697,7 +697,7 @@ macro_rules! from_usize {
         impl FromUsize for $t {
             #[inline]
             fn from_usize(x: usize) -> Result<$t> {
-                if x > <$t>::max_value() as usize {
+                if x > <$t>::MAX as usize {
                     Err(Error::new(ErrorKind::InvalidInput, "value too large to transmit").into())
                 } else {
                     Ok(x as $t)
diff --git a/src/utils/pgwire/src/pg_protocol.rs b/src/utils/pgwire/src/pg_protocol.rs
index 38eba6e92d5b6..d700e39757df1 100644
--- a/src/utils/pgwire/src/pg_protocol.rs
+++ b/src/utils/pgwire/src/pg_protocol.rs
@@ -29,7 +29,7 @@ use risingwave_common::types::DataType;
 use risingwave_common::util::panic::FutureCatchUnwindExt;
 use risingwave_common::util::query_log::*;
 use risingwave_common::{PG_VERSION, SERVER_ENCODING, STANDARD_CONFORMING_STRINGS};
-use risingwave_sqlparser::ast::Statement;
+use risingwave_sqlparser::ast::{RedactSqlOptionKeywordsRef, Statement};
 use risingwave_sqlparser::parser::Parser;
 use thiserror_ext::AsReport;
 use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
@@ -101,6 +101,8 @@ where
 
     // Client Address
     peer_addr: AddressRef,
+
+    redact_sql_option_keywords: Option<RedactSqlOptionKeywordsRef>,
 }
 
 /// Configures TLS encryption for connections.
@@ -152,16 +154,31 @@ pub fn cstr_to_str(b: &Bytes) -> Result<&str, Utf8Error> {
 }
 
 /// Record `sql` in the current tracing span.
-fn record_sql_in_span(sql: &str) {
+fn record_sql_in_span(sql: &str, redact_sql_option_keywords: Option<RedactSqlOptionKeywordsRef>) {
+    let redacted_sql = if let Some(keywords) = redact_sql_option_keywords {
+        redact_sql(sql, keywords)
+    } else {
+        sql.to_owned()
+    };
     tracing::Span::current().record(
         "sql",
         tracing::field::display(truncated_fmt::TruncatedFmt(
-            &sql,
+            &redacted_sql,
             *RW_QUERY_LOG_TRUNCATE_LEN,
         )),
     );
 }
 
+fn redact_sql(sql: &str, keywords: RedactSqlOptionKeywordsRef) -> String {
+    match Parser::parse_sql(sql) {
+        Ok(sqls) => sqls
+            .into_iter()
+            .map(|sql| sql.to_redacted_string(keywords.clone()))
+            .join(";"),
+        Err(_) => sql.to_owned(),
+    }
+}
+
 impl<S, SM> PgProtocol<S, SM>
 where
     S: AsyncWrite + AsyncRead + Unpin,
@@ -172,6 +189,7 @@ where
         session_mgr: Arc<SM>,
         tls_config: Option<TlsConfig>,
         peer_addr: AddressRef,
+        redact_sql_option_keywords: Option<RedactSqlOptionKeywordsRef>,
     ) -> Self {
         Self {
             stream: Conn::Unencrypted(PgStream {
@@ -193,6 +211,7 @@ where
             statement_portal_dependency: Default::default(),
             ignore_util_sync: false,
             peer_addr,
+            redact_sql_option_keywords,
         }
     }
 
@@ -555,7 +574,7 @@ where
     async fn process_query_msg(&mut self, query_string: io::Result<&str>) -> PsqlResult<()> {
         let sql: Arc<str> =
             Arc::from(query_string.map_err(|err| PsqlError::SimpleQueryError(Box::new(err)))?);
-        record_sql_in_span(&sql);
+        record_sql_in_span(&sql, self.redact_sql_option_keywords.clone());
         let session = self.session.clone().unwrap();
 
         session.check_idle_in_transaction_timeout()?;
@@ -664,7 +683,7 @@ where
 
     fn process_parse_msg(&mut self, msg: FeParseMessage) -> PsqlResult<()> {
         let sql = cstr_to_str(&msg.sql_bytes).unwrap();
-        record_sql_in_span(sql);
+        record_sql_in_span(sql, self.redact_sql_option_keywords.clone());
         let session = self.session.clone().unwrap();
         let statement_name = cstr_to_str(&msg.statement_name).unwrap().to_string();
 
@@ -767,7 +786,7 @@ where
             self.unnamed_portal.replace(portal);
         } else {
             assert!(
-                self.result_cache.get(&portal_name).is_none(),
+                !self.result_cache.contains_key(&portal_name),
                 "Named portal never can be overridden."
             );
             self.portal_store.insert(portal_name.clone(), portal);
@@ -798,7 +817,7 @@ where
         } else {
             let portal = self.get_portal(&portal_name)?;
             let sql: Arc<str> = Arc::from(format!("{}", portal));
-            record_sql_in_span(&sql);
+            record_sql_in_span(&sql, self.redact_sql_option_keywords.clone());
 
             session.check_idle_in_transaction_timeout()?;
             let _exec_context_guard = session.init_exec_context(sql.clone());
@@ -1205,3 +1224,25 @@ pub mod truncated_fmt {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashSet;
+
+    use super::*;
+
+    #[test]
+    fn test_redact_parsable_sql() {
+        let keywords = Arc::new(HashSet::from(["v2".into(), "v4".into(), "b".into()]));
+        let sql = r"
+        create source temp (k bigint, v varchar) with (
+            connector = 'datagen',
+            v1 = 123,
+            v2 = 'with',
+            v3 = false,
+            v4 = '',
+        ) FORMAT plain ENCODE json (a='1',b='2')
+        ";
+        assert_eq!(redact_sql(sql, keywords), "CREATE SOURCE temp (k BIGINT, v CHARACTER VARYING) WITH (connector = 'datagen', v1 = 123, v2 = [REDACTED], v3 = false, v4 = [REDACTED]) FORMAT PLAIN ENCODE JSON (a = '1', b = [REDACTED])");
+    }
+}
diff --git a/src/utils/pgwire/src/pg_response.rs b/src/utils/pgwire/src/pg_response.rs
index 8e17423228af0..4f55c524942bc 100644
--- a/src/utils/pgwire/src/pg_response.rs
+++ b/src/utils/pgwire/src/pg_response.rs
@@ -26,6 +26,7 @@ use crate::types::Row;
 
 pub type RowSet = Vec<Row>;
 pub type RowSetResult = Result<RowSet, BoxedError>;
+
 pub trait ValuesStream = Stream<Item = RowSetResult> + Unpin + Send;
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
@@ -53,8 +54,10 @@ pub enum StatementType {
     CREATE_SCHEMA,
     CREATE_USER,
     CREATE_INDEX,
+    CREATE_AGGREGATE,
     CREATE_FUNCTION,
     CREATE_CONNECTION,
+    CREATE_SECRET,
     COMMENT,
     DECLARE_CURSOR,
     DESCRIBE,
@@ -65,6 +68,7 @@ pub enum StatementType {
     DROP_VIEW,
     DROP_INDEX,
     DROP_FUNCTION,
+    DROP_AGGREGATE,
     DROP_SOURCE,
     DROP_SINK,
     DROP_SUBSCRIPTION,
@@ -72,6 +76,7 @@ pub enum StatementType {
     DROP_DATABASE,
     DROP_USER,
     DROP_CONNECTION,
+    DROP_SECRET,
     ALTER_DATABASE,
     ALTER_SCHEMA,
     ALTER_INDEX,
@@ -116,6 +121,7 @@ impl std::fmt::Display for StatementType {
 }
 
 pub trait Callback = Future<Output = Result<(), BoxedError>> + Send;
+
 pub type BoxedCallback = Pin<Box<dyn Callback>>;
 
 pub struct PgResponse<VS> {
@@ -291,6 +297,7 @@ impl StatementType {
                 risingwave_sqlparser::ast::ObjectType::Connection => {
                     Ok(StatementType::DROP_CONNECTION)
                 }
+                risingwave_sqlparser::ast::ObjectType::Secret => Ok(StatementType::DROP_SECRET),
                 risingwave_sqlparser::ast::ObjectType::Subscription => {
                     Ok(StatementType::DROP_SUBSCRIPTION)
                 }
diff --git a/src/utils/pgwire/src/pg_server.rs b/src/utils/pgwire/src/pg_server.rs
index 0538217de9acd..d2882fdceafbb 100644
--- a/src/utils/pgwire/src/pg_server.rs
+++ b/src/utils/pgwire/src/pg_server.rs
@@ -15,7 +15,6 @@
 use std::collections::HashMap;
 use std::future::Future;
 use std::io;
-use std::result::Result;
 use std::str::FromStr;
 use std::sync::Arc;
 use std::time::Instant;
@@ -24,7 +23,7 @@ use bytes::Bytes;
 use jsonwebtoken::{decode, decode_header, Algorithm, DecodingKey, Validation};
 use parking_lot::Mutex;
 use risingwave_common::types::DataType;
-use risingwave_sqlparser::ast::Statement;
+use risingwave_sqlparser::ast::{RedactSqlOptionKeywordsRef, Statement};
 use serde::Deserialize;
 use thiserror_ext::AsReport;
 use tokio::io::{AsyncRead, AsyncWrite};
@@ -252,28 +251,49 @@ pub async fn pg_serve(
     addr: &str,
     session_mgr: Arc<impl SessionManager>,
     tls_config: Option<TlsConfig>,
+    redact_sql_option_keywords: Option<RedactSqlOptionKeywordsRef>,
 ) -> io::Result<()> {
     let listener = Listener::bind(addr).await?;
     tracing::info!(addr, "server started");
 
-    loop {
-        let conn_ret = listener.accept().await;
-        match conn_ret {
-            Ok((stream, peer_addr)) => {
-                tracing::info!(%peer_addr, "accept connection");
-                tokio::spawn(handle_connection(
-                    stream,
-                    session_mgr.clone(),
-                    tls_config.clone(),
-                    Arc::new(peer_addr),
-                ));
-            }
+    let acceptor_runtime = {
+        let mut builder = tokio::runtime::Builder::new_multi_thread();
+        builder.worker_threads(1);
+        builder
+            .thread_name("rw-acceptor")
+            .enable_all()
+            .build()
+            .unwrap()
+    };
 
-            Err(e) => {
-                tracing::error!(error = %e.as_report(), "failed to accept connection",);
+    #[cfg(not(madsim))]
+    let worker_runtime = tokio::runtime::Handle::current();
+    #[cfg(madsim)]
+    let worker_runtime = tokio::runtime::Builder::new_multi_thread().build().unwrap();
+
+    let f = async move {
+        loop {
+            let conn_ret = listener.accept().await;
+            match conn_ret {
+                Ok((stream, peer_addr)) => {
+                    tracing::info!(%peer_addr, "accept connection");
+                    worker_runtime.spawn(handle_connection(
+                        stream,
+                        session_mgr.clone(),
+                        tls_config.clone(),
+                        Arc::new(peer_addr),
+                        redact_sql_option_keywords.clone(),
+                    ));
+                }
+
+                Err(e) => {
+                    tracing::error!(error = %e.as_report(), "failed to accept connection",);
+                }
             }
         }
-    }
+    };
+    acceptor_runtime.spawn(f).await?;
+    Ok(())
 }
 
 pub async fn handle_connection<S, SM>(
@@ -281,11 +301,18 @@ pub async fn handle_connection<S, SM>(
     session_mgr: Arc<SM>,
     tls_config: Option<TlsConfig>,
     peer_addr: AddressRef,
+    redact_sql_option_keywords: Option<RedactSqlOptionKeywordsRef>,
 ) where
     S: AsyncWrite + AsyncRead + Unpin,
     SM: SessionManager,
 {
-    let mut pg_proto = PgProtocol::new(stream, session_mgr, tls_config, peer_addr);
+    let mut pg_proto = PgProtocol::new(
+        stream,
+        session_mgr,
+        tls_config,
+        peer_addr,
+        redact_sql_option_keywords,
+    );
     loop {
         let msg = match pg_proto.read_message().await {
             Ok(msg) => msg,
@@ -468,7 +495,7 @@ mod tests {
         let pg_config = pg_config.into();
 
         let session_mgr = Arc::new(MockSessionManager {});
-        tokio::spawn(async move { pg_serve(&bind_addr, session_mgr, None).await });
+        tokio::spawn(async move { pg_serve(&bind_addr, session_mgr, None, None).await });
         // wait for server to start
         tokio::time::sleep(std::time::Duration::from_millis(100)).await;
 
diff --git a/src/utils/runtime/src/logger.rs b/src/utils/runtime/src/logger.rs
index 7cc4194600b00..f48c0b5d9ea65 100644
--- a/src/utils/runtime/src/logger.rs
+++ b/src/utils/runtime/src/logger.rs
@@ -18,6 +18,7 @@ use std::path::PathBuf;
 use either::Either;
 use risingwave_common::metrics::MetricsLayer;
 use risingwave_common::util::deployment::Deployment;
+use risingwave_common::util::env_var::env_var_is_true;
 use risingwave_common::util::query_log::*;
 use risingwave_common::util::tracing::layer::set_toggle_otel_layer_fn;
 use thiserror_ext::AsReport;
@@ -170,6 +171,17 @@ fn disabled_filter() -> filter::Targets {
 ///
 /// `RW_QUERY_LOG_TRUNCATE_LEN` configures the max length of the SQLs logged in the query log,
 /// to avoid the log file growing too large. The default value is 1024 in production.
+///
+/// ### `RW_ENABLE_PRETTY_LOG`
+///
+/// If it is set to `true`, enable pretty log output, which contains line numbers and prints spans in multiple lines.
+/// This can be helpful for development and debugging.
+///
+/// Hint: Also turn off other uninteresting logs to make the most of the pretty log.
+/// e.g.,
+/// ```bash
+/// RUST_LOG="risingwave_storage::hummock::event_handler=off,batch_execute=off,risingwave_batch::task=off" RW_ENABLE_PRETTY_LOG=true risedev d
+/// ```
 pub fn init_risingwave_logger(settings: LoggerSettings) {
     let deployment = Deployment::current();
 
@@ -201,7 +213,7 @@ pub fn init_risingwave_logger(settings: LoggerSettings) {
 
         // Configure levels for external crates.
         filter = filter
-            .with_target("foyer", Level::WARN)
+            .with_target("foyer", Level::INFO)
             .with_target("aws", Level::INFO)
             .with_target("aws_config", Level::WARN)
             .with_target("aws_endpoint", Level::WARN)
@@ -276,7 +288,13 @@ pub fn init_risingwave_logger(settings: LoggerSettings) {
                 .json()
                 .map_event_format(|e| e.with_current_span(false)) // avoid duplication as there's a span list field
                 .boxed(),
-            Deployment::Other => fmt_layer.boxed(),
+            Deployment::Other => {
+                if env_var_is_true("RW_ENABLE_PRETTY_LOG") {
+                    fmt_layer.pretty().boxed()
+                } else {
+                    fmt_layer.boxed()
+                }
+            }
         };
 
         layers.push(
diff --git a/src/utils/workspace-config/Cargo.toml b/src/utils/workspace-config/Cargo.toml
index e15dd774b7e3d..8972cb7c08475 100644
--- a/src/utils/workspace-config/Cargo.toml
+++ b/src/utils/workspace-config/Cargo.toml
@@ -11,7 +11,7 @@ repository = { workspace = true }
 [features]
 # some crates opt-in static linking, while some opt-in dynamic linking,
 # so they are two features :)
-rw-static-link = ["static-libz-sys", "static-lzma-sys", "static-openssl-sys", "static-sasl2-sys"]
+rw-static-link = ["static-libz-sys", "static-lzma-sys", "static-sasl2-sys"]
 rw-dynamic-link = ["dynamic-zstd-sys"]
 
 [dependencies]
@@ -22,7 +22,6 @@ tracing = { version = "0.1", features = ["release_max_level_debug"] }
 # Static linking
 static-libz-sys = { package = "libz-sys", version = "1", optional = true, features = ["static"] }
 static-lzma-sys = { package = "lzma-sys", version = "0.1", optional = true, features = ["static"] }
-static-openssl-sys = { package = "openssl-sys", version = "0.9.96", optional = true, features = ["vendored"] }
 static-sasl2-sys = { package = "sasl2-sys", version = "0.1", optional = true, features = ["gssapi-vendored"] }
 
 # Dynamic linking
diff --git a/standalone/grafana.ini b/standalone/grafana.ini
new file mode 100644
index 0000000000000..c05dee52e831a
--- /dev/null
+++ b/standalone/grafana.ini
@@ -0,0 +1,10 @@
+[server]
+http_addr = 0.0.0.0
+http_port = 3001
+
+[users]
+default_theme = light
+
+[auth.anonymous]
+enabled = true
+org_role = Admin
\ No newline at end of file
diff --git a/standalone/prometheus.yml b/standalone/prometheus.yml
new file mode 100644
index 0000000000000..b354503515509
--- /dev/null
+++ b/standalone/prometheus.yml
@@ -0,0 +1,40 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 60s
+  external_labels:
+    rw_cluster: 20240506-185437
+
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ["127.0.0.1:9500"]
+
+  - job_name: compute
+    static_configs:
+      - targets: ["127.0.0.1:1222"]
+
+  - job_name: meta
+    static_configs:
+      - targets: ["127.0.0.1:1250"]
+
+  - job_name: minio
+    metrics_path: /minio/v2/metrics/cluster
+    static_configs:
+    - targets: ["127.0.0.1:9301"]
+
+  - job_name: compactor
+    static_configs:
+      - targets: ["127.0.0.1:1260"]
+
+  - job_name: etcd
+    static_configs:
+      - targets: ["127.0.0.1:2379"]
+
+  - job_name: frontend
+    static_configs:
+      - targets: ["127.0.0.1:2222"]
+
+  - job_name: redpanda
+    static_configs:
+      - targets: []